diff --git a/docs/Contributors_Guide/basic_components.rst b/docs/Contributors_Guide/basic_components.rst index e2a29ab574..86d3d57323 100644 --- a/docs/Contributors_Guide/basic_components.rst +++ b/docs/Contributors_Guide/basic_components.rst @@ -274,7 +274,7 @@ should be set. Add Support for MET Dictionary ------------------------------ -The handle_met_config_dict function can be used to easily set a MET config +The add_met_config_dict function can be used to easily set a MET config dictionary variable. The function takes 2 arguments: * dict_name: Name of the MET dictionary variable, i.e. distance_map. @@ -285,7 +285,7 @@ dictionary variable. The function takes 2 arguments: :: - self.handle_met_config_dict('fcst_genesis', { + self.add_met_config_dict('fcst_genesis', { 'vmax_thresh': 'thresh', 'mslp_thresh': 'thresh', }) @@ -319,7 +319,7 @@ a function is typically used to handle it. For example, this function is in CompareGriddedWrapper and is used by GridStat, PointStat, and EnsembleStat:: def handle_climo_cdf_dict(self): - self.handle_met_config_dict('climo_cdf', { + self.add_met_config_dict('climo_cdf', { 'cdf_bins': ('float', None, None, ['CLIMO_CDF_BINS']), 'center_bins': 'bool', 'write_bins': 'bool', @@ -333,26 +333,26 @@ the nickname 'CLIMO_CDF_BINS' allows the user to set the variable GRID_STAT_CLIMO_CDF_BINS instead. There are many MET config dictionaries that only contain beg and end to define -a window. A function in CommandBuilder called handle_met_config_window can be +a window. A function in CommandBuilder called add_met_config_window can be used to easily set these variable by only supplying the name of the MET dictionary variable. :: - def handle_met_config_window(self, dict_name): + def add_met_config_window(self, dict_name): """! Handle a MET config window dictionary. It is assumed that the dictionary only contains 'beg' and 'end' entries that are integers. @param dict_name name of MET dictionary """ - self.handle_met_config_dict(dict_name, { + self.add_met_config_dict(dict_name, { 'beg': 'int', 'end': 'int', }) This can be called from any wrapper, i.e. TCGen:: - self.handle_met_config_window('fcst_hr_window') + self.add_met_config_window('fcst_hr_window') This will check if TC_GEN_FCST_HR_WINDOW_BEGIN (or TC_GEN_FCST_HR_WINDOW_BEG) and TC_GEN_FCST_HR_WINDOW_END are set and override fcst_hr_window.beg and/or @@ -383,5 +383,5 @@ handle_climo_dict, handle_mask, and handle_interp_dict. if uses_field: items['field'] = ('string', 'remove_quotes') - self.handle_met_config_dict('interp', items) + self.add_met_config_dict('interp', items) diff --git a/internal_tests/pytests/ascii2nc/test_ascii2nc_wrapper.py b/internal_tests/pytests/ascii2nc/test_ascii2nc_wrapper.py index 8db28911a4..b7c3b72767 100644 --- a/internal_tests/pytests/ascii2nc/test_ascii2nc_wrapper.py +++ b/internal_tests/pytests/ascii2nc/test_ascii2nc_wrapper.py @@ -173,7 +173,12 @@ def test_ascii2nc_wrapper(metplus_config, config_overrides, assert(all_commands[0][0] == expected_cmd) env_vars = all_commands[0][1] - for env_var_key in wrapper.WRAPPER_ENV_VAR_KEYS: + # check that environment variables were set properly + # including deprecated env vars (not in wrapper env var keys) + env_var_keys = (wrapper.WRAPPER_ENV_VAR_KEYS + + [name for name in env_var_values + if name not in wrapper.WRAPPER_ENV_VAR_KEYS]) + for env_var_key in env_var_keys: match = next((item for item in env_vars if item.startswith(env_var_key)), None) assert (match is not None) diff --git a/internal_tests/pytests/command_builder/test_command_builder.py b/internal_tests/pytests/command_builder/test_command_builder.py index 29fede8943..355cb66b87 100644 --- a/internal_tests/pytests/command_builder/test_command_builder.py +++ b/internal_tests/pytests/command_builder/test_command_builder.py @@ -10,7 +10,7 @@ import datetime from metplus.wrappers.command_builder import CommandBuilder from metplus.util import time_util -from metplus.util import METConfigInfo as met_config +from metplus.util import METConfig @pytest.mark.parametrize( @@ -380,19 +380,6 @@ def test_handle_description(metplus_config, config_overrides, expected_value): cbw.handle_description() assert cbw.env_var_dict.get('METPLUS_DESC', '') == expected_value -@pytest.mark.parametrize( - 'input, output', [ - ('', 'NONE'), - ('NONE', 'NONE'), - ('FCST', 'FCST'), - ('OBS', 'OBS'), - ('G002', '"G002"'), - ] -) -def test_format_regrid_to_grid(metplus_config, input, output): - cbw = CommandBuilder(metplus_config()) - assert cbw.format_regrid_to_grid(input) == output - @pytest.mark.parametrize( 'config_overrides, set_to_grid, expected_dict', [ ({}, True, {'REGRID_TO_GRID': 'NONE'}), @@ -499,25 +486,29 @@ def test_handle_regrid_new(metplus_config, config_overrides, expected_output): True, 'test_string_1 = value_1;'), ] ) -def test_set_met_config_string(metplus_config, mp_config_name, met_config_name, +def test_add_met_config_string(metplus_config, mp_config_name, met_config_name, c_dict_key, remove_quotes, expected_output): cbw = CommandBuilder(metplus_config()) # set some config variables to test cbw.config.set('config', 'TEST_STRING_1', 'value_1') - c_dict = {} + extra_args = {} + if remove_quotes: + extra_args['remove_quotes'] = True - cbw.set_met_config_string(c_dict, - mp_config_name, - met_config_name, - c_dict_key=c_dict_key, - remove_quotes=remove_quotes) key = c_dict_key if key is None: - key = met_config_name.upper() + key = met_config_name + key = key.upper() + + cbw.add_met_config(name=met_config_name, + data_type='string', + env_var_name=key, + metplus_configs=[mp_config_name], + extra_args=extra_args) - assert c_dict.get(key, '') == expected_output + assert cbw.env_var_dict.get(f'METPLUS_{key}', '') == expected_output @pytest.mark.parametrize( 'mp_config_name,met_config_name,c_dict_key,uppercase,expected_output, is_ok', [ @@ -547,7 +538,7 @@ def test_set_met_config_string(metplus_config, mp_config_name, met_config_name, True, '', False), ] ) -def test_set_met_config_bool(metplus_config, mp_config_name, met_config_name, +def test_add_met_config_bool(metplus_config, mp_config_name, met_config_name, c_dict_key, uppercase, expected_output, is_ok): cbw = CommandBuilder(metplus_config()) @@ -556,18 +547,22 @@ def test_set_met_config_bool(metplus_config, mp_config_name, met_config_name, cbw.config.set('config', 'TEST_BOOL_3', False) cbw.config.set('config', 'TEST_BOOL_4', 'chicken') - c_dict = {} + extra_args = {} + if not uppercase: + extra_args['uppercase'] = False - cbw.set_met_config_bool(c_dict, - mp_config_name, - met_config_name, - c_dict_key=c_dict_key, - uppercase=uppercase) key = c_dict_key if key is None: - key = met_config_name.upper() + key = met_config_name + key = key.upper() + + cbw.add_met_config(name=met_config_name, + data_type='bool', + env_var_name=key, + metplus_configs=[mp_config_name], + extra_args=extra_args) - assert c_dict.get(key, '') == expected_output + assert cbw.env_var_dict.get(f'METPLUS_{key}', '') == expected_output assert cbw.isOK == is_ok # int @@ -590,7 +585,7 @@ def test_set_met_config_bool(metplus_config, mp_config_name, met_config_name, '', False), ] ) -def test_set_met_config_int(metplus_config, mp_config_name, met_config_name, +def test_add_met_config_int(metplus_config, mp_config_name, met_config_name, c_dict_key, expected_output, is_ok): cbw = CommandBuilder(metplus_config()) @@ -599,17 +594,17 @@ def test_set_met_config_int(metplus_config, mp_config_name, met_config_name, cbw.config.set('config', 'TEST_INT_3', -4) cbw.config.set('config', 'TEST_INT_4', 'chicken') - c_dict = {} - - cbw.set_met_config_int(c_dict, - mp_config_name, - met_config_name, - c_dict_key=c_dict_key) key = c_dict_key if key is None: - key = met_config_name.upper() + key = met_config_name + key = key.upper() - assert c_dict.get(key, '') == expected_output + cbw.add_met_config(name=met_config_name, + data_type='int', + env_var_name=key, + metplus_configs=[mp_config_name]) + + assert cbw.env_var_dict.get(f'METPLUS_{key}', '') == expected_output assert cbw.isOK == is_ok @pytest.mark.parametrize( @@ -631,7 +626,7 @@ def test_set_met_config_int(metplus_config, mp_config_name, met_config_name, '', False), ] ) -def test_set_met_config_float(metplus_config, mp_config_name, met_config_name, +def test_add_met_config_float(metplus_config, mp_config_name, met_config_name, c_dict_key, expected_output, is_ok): cbw = CommandBuilder(metplus_config()) @@ -640,17 +635,17 @@ def test_set_met_config_float(metplus_config, mp_config_name, met_config_name, cbw.config.set('config', 'TEST_FLOAT_3', 4) cbw.config.set('config', 'TEST_FLOAT_4', 'chicken') - c_dict = {} - - cbw.set_met_config_float(c_dict, - mp_config_name, - met_config_name, - c_dict_key=c_dict_key) key = c_dict_key if key is None: - key = met_config_name.upper() + key = met_config_name + key = key.upper() + + cbw.add_met_config(name=met_config_name, + data_type='float', + env_var_name=key, + metplus_configs=[mp_config_name]) - assert c_dict.get(key, '') == expected_output + assert cbw.env_var_dict.get(f'METPLUS_{key}', '') == expected_output assert cbw.isOK == is_ok @pytest.mark.parametrize( @@ -678,7 +673,7 @@ def test_set_met_config_float(metplus_config, mp_config_name, met_config_name, 'test_thresh_6 = NA;', True), ] ) -def test_set_met_config_thresh(metplus_config, mp_config_name, met_config_name, +def test_add_met_config_thresh(metplus_config, mp_config_name, met_config_name, c_dict_key, expected_output, is_ok): cbw = CommandBuilder(metplus_config()) @@ -689,17 +684,18 @@ def test_set_met_config_thresh(metplus_config, mp_config_name, met_config_name, cbw.config.set('config', 'TEST_THRESH_5', '>CDP40&&<=CDP50') cbw.config.set('config', 'TEST_THRESH_6', 'NA') - c_dict = {} - - cbw.set_met_config_thresh(c_dict, - mp_config_name, - met_config_name, - c_dict_key=c_dict_key) key = c_dict_key if key is None: - key = met_config_name.upper() + key = met_config_name + key = key.upper() - assert c_dict.get(key, '') == expected_output + cbw.add_met_config(name=met_config_name, + env_var_name=key, + data_type='thresh', + metplus_configs=[mp_config_name]) + + print(f"KEY: {key}, ENV VARS: {cbw.env_var_dict}") + assert cbw.env_var_dict.get(f'METPLUS_{key}', '') == expected_output assert cbw.isOK == is_ok @pytest.mark.parametrize( @@ -727,7 +723,7 @@ def test_set_met_config_thresh(metplus_config, mp_config_name, met_config_name, True, 'test_list_4 = [value_1, value2];'), ] ) -def test_set_met_config_list(metplus_config, mp_config_name, met_config_name, +def test_add_met_config_list(metplus_config, mp_config_name, met_config_name, c_dict_key, remove_quotes, expected_output): cbw = CommandBuilder(metplus_config()) @@ -736,18 +732,23 @@ def test_set_met_config_list(metplus_config, mp_config_name, met_config_name, cbw.config.set('config', 'TEST_LIST_3', "'value_1', 'value2'") cbw.config.set('config', 'TEST_LIST_4', '"value_1", "value2"') - c_dict = {} + extra_args = {} + if remove_quotes: + extra_args['remove_quotes'] = True - cbw.set_met_config_list(c_dict, - mp_config_name, - met_config_name, - c_dict_key=c_dict_key, - remove_quotes=remove_quotes) key = c_dict_key if key is None: - key = met_config_name.upper() + key = met_config_name + + key = key.upper() - assert c_dict.get(key, '') == expected_output + cbw.add_met_config(name=met_config_name, + data_type='list', + env_var_name=key, + metplus_configs=[mp_config_name], + extra_args=extra_args) + print(f"KEY: {key}, ENV VARS: {cbw.env_var_dict}") + assert cbw.env_var_dict.get(f'METPLUS_{key}', '') == expected_output @pytest.mark.parametrize( 'mp_config_name,allow_empty,expected_output', [ @@ -761,42 +762,28 @@ def test_set_met_config_list(metplus_config, mp_config_name, met_config_name, ('TEST_LIST_2', True, ''), ] ) -def test_set_met_config_list_allow_empty(metplus_config, mp_config_name, +def test_add_met_config_list_allow_empty(metplus_config, mp_config_name, allow_empty, expected_output): cbw = CommandBuilder(metplus_config()) # set some config variables to test cbw.config.set('config', 'TEST_LIST_1', '') - c_dict = {} + extra_args = {} + if allow_empty: + extra_args['allow_empty'] = True met_config_name = mp_config_name.lower() - cbw.set_met_config_list(c_dict, - mp_config_name, - met_config_name, - allow_empty=allow_empty) - - assert c_dict.get(mp_config_name, '') == expected_output + cbw.add_met_config(name=met_config_name, + data_type='list', + metplus_configs=[mp_config_name], + extra_args=extra_args) -@pytest.mark.parametrize( - 'data_type, expected_function', [ - ('int', 'set_met_config_int'), - ('float', 'set_met_config_float'), - ('list', 'set_met_config_list'), - ('string', 'set_met_config_string'), - ('thresh', 'set_met_config_thresh'), - ('bool', 'set_met_config_bool'), - ('bad_name', None), - ] -) -def test_set_met_config_function(metplus_config, data_type, expected_function): - cbw = CommandBuilder(metplus_config()) - function_found = cbw.set_met_config_function(data_type) - function_name = function_found.__name__ if function_found else None - assert(function_name == expected_function) + assert cbw.env_var_dict.get(f'METPLUS_{mp_config_name}', '') == expected_output + #assert c_dict.get(mp_config_name, '') == expected_output -def test_handle_met_config_dict(metplus_config): +def test_add_met_config_dict(metplus_config): dict_name = 'fcst_hr_window' beg = -3 end = 5 @@ -813,12 +800,12 @@ def test_handle_met_config_dict(metplus_config): 'end': 'int', } - cbw.handle_met_config_dict(dict_name, items) + cbw.add_met_config_dict(dict_name, items) print(f"env_var_dict: {cbw.env_var_dict}") actual_value = cbw.env_var_dict.get('METPLUS_FCST_HR_WINDOW_DICT') assert actual_value == expected_value -def test_handle_met_config_window(metplus_config): +def test_add_met_config_window(metplus_config): dict_name = 'fcst_hr_window' beg = -3 end = 5 @@ -830,7 +817,7 @@ def test_handle_met_config_window(metplus_config): cbw = CommandBuilder(config) cbw.app_name = 'tc_gen' - cbw.handle_met_config_window(dict_name) + cbw.add_met_config_window(dict_name) print(f"env_var_dict: {cbw.env_var_dict}") actual_value = cbw.env_var_dict.get('METPLUS_FCST_HR_WINDOW_DICT') assert actual_value == expected_value @@ -848,7 +835,7 @@ def test_add_met_config(metplus_config): expected_value = f'valid_freq = {value};' assert cbw.env_var_dict['METPLUS_VALID_FREQ'] == expected_value -def test_handle_met_config_dict_nested(metplus_config): +def test_add_met_config_dict_nested(metplus_config): dict_name = 'outer' beg = -3 end = 5 @@ -876,6 +863,6 @@ def test_handle_met_config_dict_nested(metplus_config): }), } - cbw.handle_met_config_dict(dict_name, items) + cbw.add_met_config_dict(dict_name, items) print(f"env_var_dict: {cbw.env_var_dict}") assert cbw.env_var_dict.get('METPLUS_OUTER_DICT') == expected_value diff --git a/internal_tests/pytests/config_metplus/test_config_metplus.py b/internal_tests/pytests/config_metplus/test_config_metplus.py index 3fef44ce23..1c55fbc973 100644 --- a/internal_tests/pytests/config_metplus/test_config_metplus.py +++ b/internal_tests/pytests/config_metplus/test_config_metplus.py @@ -1,7 +1,9 @@ #!/usr/bin/env python3 import pytest +import pprint import os +from datetime import datetime from metplus.util import config_metplus @@ -28,9 +30,965 @@ def test_get_default_config_list(): expected_new = [os.path.join(new_parm_base, item) for item in new_list] expected_both = [os.path.join(both_parm_base, item) for item in both_list] - actual_old = config_metplus.get_default_config_list(old_parm_base) - actual_new = config_metplus.get_default_config_list(new_parm_base) - actual_both = config_metplus.get_default_config_list(both_parm_base) + actual_old = config_metplus._get_default_config_list(old_parm_base) + actual_new = config_metplus._get_default_config_list(new_parm_base) + actual_both = config_metplus._get_default_config_list(both_parm_base) assert actual_old == expected_old assert actual_new == expected_new assert actual_both == expected_both + +@pytest.mark.parametrize( + 'regex,index,id,expected_result', [ + # 0: No ID + (r'^FCST_VAR(\d+)_NAME$', 1, None, + {'1': [None], + '2': [None], + '4': [None]}), + # 1: ID and index 2 + (r'(\w+)_VAR(\d+)_NAME', 2, 1, + {'1': ['FCST'], + '2': ['FCST'], + '4': ['FCST']}), + # 2: index 1, ID 2, multiple identifiers + (r'^FCST_VAR(\d+)_(\w+)$', 1, 2, + {'1': ['NAME', 'LEVELS'], + '2': ['NAME'], + '4': ['NAME']}), + # 3: command that StatAnalysis wrapper uses + (r'MODEL(\d+)$', 1, None, + {'1': [None], + '2': [None],}), + # 4: TCPairs conensus logic + (r'^TC_PAIRS_CONSENSUS(\d+)_(\w+)$', 1, 2, + {'1': ['NAME', 'MEMBERS', 'REQUIRED', 'MIN_REQ'], + '2': ['NAME', 'MEMBERS', 'REQUIRED', 'MIN_REQ']}), + ] +) +def test_find_indices_in_config_section(metplus_config, regex, index, + id, expected_result): + config = metplus_config() + config.set('config', 'FCST_VAR1_NAME', 'name1') + config.set('config', 'FCST_VAR1_LEVELS', 'level1') + config.set('config', 'FCST_VAR2_NAME', 'name2') + config.set('config', 'FCST_VAR4_NAME', 'name4') + config.set('config', 'MODEL1', 'model1') + config.set('config', 'MODEL2', 'model2') + + config.set('config', 'TC_PAIRS_CONSENSUS1_NAME', 'name1') + config.set('config', 'TC_PAIRS_CONSENSUS1_MEMBERS', 'member1') + config.set('config', 'TC_PAIRS_CONSENSUS1_REQUIRED', 'True') + config.set('config', 'TC_PAIRS_CONSENSUS1_MIN_REQ', '1') + config.set('config', 'TC_PAIRS_CONSENSUS2_NAME', 'name2') + config.set('config', 'TC_PAIRS_CONSENSUS2_MEMBERS', 'member2') + config.set('config', 'TC_PAIRS_CONSENSUS2_REQUIRED', 'True') + config.set('config', 'TC_PAIRS_CONSENSUS2_MIN_REQ', '2') + + + indices = config_metplus.find_indices_in_config_section(regex, config, + index_index=index, + id_index=id) + + pp = pprint.PrettyPrinter() + print(f'Indices:') + pp.pprint(indices) + + assert indices == expected_result + +@pytest.mark.parametrize( + 'conf_items, met_tool, expected_result', [ + ({'CUSTOM_LOOP_LIST': "one, two, three"}, '', ['one', 'two', 'three']), + ({'CUSTOM_LOOP_LIST': "one, two, three", + 'GRID_STAT_CUSTOM_LOOP_LIST': "four, five",}, 'grid_stat', ['four', 'five']), + ({'CUSTOM_LOOP_LIST': "one, two, three", + 'GRID_STAT_CUSTOM_LOOP_LIST': "four, five",}, 'point_stat', ['one', 'two', 'three']), + ({'CUSTOM_LOOP_LIST': "one, two, three", + 'ASCII2NC_CUSTOM_LOOP_LIST': "four, five",}, 'ascii2nc', ['four', 'five']), + # fails to read custom loop list for point2grid because there are underscores in name + ({'CUSTOM_LOOP_LIST': "one, two, three", + 'POINT_2_GRID_CUSTOM_LOOP_LIST': "four, five",}, 'point2grid', ['one', 'two', 'three']), + ({'CUSTOM_LOOP_LIST': "one, two, three", + 'POINT2GRID_CUSTOM_LOOP_LIST': "four, five",}, 'point2grid', ['four', 'five']), + ] +) +def test_get_custom_string_list(metplus_config, conf_items, met_tool, expected_result): + config = metplus_config() + for conf_key, conf_value in conf_items.items(): + config.set('config', conf_key, conf_value) + + assert(config_metplus.get_custom_string_list(config, met_tool) == expected_result) + +@pytest.mark.parametrize( + 'config_var_name, expected_indices, set_met_tool', [ + ('FCST_GRID_STAT_VAR1_NAME', ['1'], True), + ('FCST_GRID_STAT_VAR2_INPUT_FIELD_NAME', ['2'], True), + ('FCST_GRID_STAT_VAR3_FIELD_NAME', ['3'], True), + ('BOTH_GRID_STAT_VAR4_NAME', ['4'], True), + ('BOTH_GRID_STAT_VAR5_INPUT_FIELD_NAME', ['5'], True), + ('BOTH_GRID_STAT_VAR6_FIELD_NAME', ['6'], True), + ('FCST_VAR7_NAME', ['7'], False), + ('FCST_VAR8_INPUT_FIELD_NAME', ['8'], False), + ('FCST_VAR9_FIELD_NAME', ['9'], False), + ('BOTH_VAR10_NAME', ['10'], False), + ('BOTH_VAR11_INPUT_FIELD_NAME', ['11'], False), + ('BOTH_VAR12_FIELD_NAME', ['12'], False), + ] +) +def test_find_var_indices_fcst(metplus_config, + config_var_name, + expected_indices, + set_met_tool): + config = metplus_config() + data_types = ['FCST'] + config.set('config', config_var_name, "NAME1") + met_tool = 'grid_stat' if set_met_tool else None + var_name_indices = config_metplus.find_var_name_indices(config, + data_types=data_types, + met_tool=met_tool) + + assert(len(var_name_indices) == len(expected_indices)) + for actual_index in var_name_indices: + assert(actual_index in expected_indices) + +@pytest.mark.parametrize( + 'data_type, met_tool, expected_out', [ + ('FCST', None, ['FCST_', + 'BOTH_',]), + ('OBS', None, ['OBS_', + 'BOTH_',]), + ('FCST', 'grid_stat', ['FCST_GRID_STAT_', + 'BOTH_GRID_STAT_', + 'FCST_', + 'BOTH_', + ]), + ('OBS', 'extract_tiles', ['OBS_EXTRACT_TILES_', + 'BOTH_EXTRACT_TILES_', + 'OBS_', + 'BOTH_', + ]), + ('ENS', None, ['ENS_']), + ('DATA', None, ['DATA_']), + ('DATA', 'tc_gen', ['DATA_TC_GEN_', + 'DATA_']), + + ] +) +def test_get_field_search_prefixes(data_type, met_tool, expected_out): + assert(config_metplus.get_field_search_prefixes(data_type, + met_tool) == expected_out) + +@pytest.mark.parametrize( + 'item_list, extension, is_valid', [ + (['FCST'], 'NAME', False), + (['OBS'], 'NAME', False), + (['FCST', 'OBS'], 'NAME', True), + (['BOTH'], 'NAME', True), + (['FCST', 'OBS', 'BOTH'], 'NAME', False), + (['FCST', 'ENS'], 'NAME', False), + (['OBS', 'ENS'], 'NAME', False), + (['FCST', 'OBS', 'ENS'], 'NAME', True), + (['BOTH', 'ENS'], 'NAME', True), + (['FCST', 'OBS', 'BOTH', 'ENS'], 'NAME', False), + + (['FCST', 'OBS'], 'THRESH', True), + (['BOTH'], 'THRESH', True), + (['FCST', 'OBS', 'BOTH'], 'THRESH', False), + (['FCST', 'OBS', 'ENS'], 'THRESH', True), + (['BOTH', 'ENS'], 'THRESH', True), + (['FCST', 'OBS', 'BOTH', 'ENS'], 'THRESH', False), + + (['FCST'], 'OPTIONS', True), + (['OBS'], 'OPTIONS', True), + (['FCST', 'OBS'], 'OPTIONS', True), + (['BOTH'], 'OPTIONS', True), + (['FCST', 'OBS', 'BOTH'], 'OPTIONS', False), + (['FCST', 'ENS'], 'OPTIONS', True), + (['OBS', 'ENS'], 'OPTIONS', True), + (['FCST', 'OBS', 'ENS'], 'OPTIONS', True), + (['BOTH', 'ENS'], 'OPTIONS', True), + (['FCST', 'OBS', 'BOTH', 'ENS'], 'OPTIONS', False), + + (['FCST', 'OBS', 'BOTH'], 'LEVELS', False), + (['FCST', 'OBS'], 'LEVELS', True), + (['BOTH'], 'LEVELS', True), + (['FCST', 'OBS', 'ENS'], 'LEVELS', True), + (['BOTH', 'ENS'], 'LEVELS', True), + + ] +) +def test_is_var_item_valid(metplus_config, item_list, extension, is_valid): + conf = metplus_config() + assert(config_metplus.is_var_item_valid(item_list, '1', extension, conf)[0] == is_valid) + +@pytest.mark.parametrize( + 'item_list, configs_to_set, is_valid', [ + + (['FCST'], {'FCST_VAR1_LEVELS': 'A06', + 'OBS_VAR1_NAME': 'script_name.py something else'}, True), + (['FCST'], {'FCST_VAR1_LEVELS': 'A06', + 'OBS_VAR1_NAME': 'APCP'}, False), + (['OBS'], {'OBS_VAR1_LEVELS': '"(*,*)"', + 'FCST_VAR1_NAME': 'script_name.py something else'}, True), + (['OBS'], {'OBS_VAR1_LEVELS': '"(*,*)"', + 'FCST_VAR1_NAME': 'APCP'}, False), + + (['FCST', 'ENS'], {'FCST_VAR1_LEVELS': 'A06', + 'OBS_VAR1_NAME': 'script_name.py something else'}, True), + (['FCST', 'ENS'], {'FCST_VAR1_LEVELS': 'A06', + 'OBS_VAR1_NAME': 'APCP'}, False), + (['OBS', 'ENS'], {'OBS_VAR1_LEVELS': '"(*,*)"', + 'FCST_VAR1_NAME': 'script_name.py something else'}, True), + (['OBS', 'ENS'], {'OBS_VAR1_LEVELS': '"(*,*)"', + 'FCST_VAR1_NAME': 'APCP'}, False), + + (['FCST'], {'FCST_VAR1_LEVELS': 'A06, A12', + 'OBS_VAR1_NAME': 'script_name.py something else'}, False), + (['FCST'], {'FCST_VAR1_LEVELS': 'A06, A12', + 'OBS_VAR1_NAME': 'APCP'}, False), + (['OBS'], {'OBS_VAR1_LEVELS': '"(0,*,*)", "(1,*,*)"', + 'FCST_VAR1_NAME': 'script_name.py something else'}, False), + + (['FCST', 'ENS'], {'FCST_VAR1_LEVELS': 'A06, A12', + 'OBS_VAR1_NAME': 'script_name.py something else'}, False), + (['FCST', 'ENS'], {'FCST_VAR1_LEVELS': 'A06, A12', + 'OBS_VAR1_NAME': 'APCP'}, False), + (['OBS', 'ENS'], {'OBS_VAR1_LEVELS': '"(0,*,*)", "(1,*,*)"', + 'FCST_VAR1_NAME': 'script_name.py something else'}, False), + + ] +) +def test_is_var_item_valid_levels(metplus_config, item_list, configs_to_set, is_valid): + conf = metplus_config() + for key, value in configs_to_set.items(): + conf.set('config', key, value) + + assert(config_metplus.is_var_item_valid(item_list, '1', 'LEVELS', conf)[0] == is_valid) + +# search prefixes are valid prefixes to append to field info variables +# config_overrides are a dict of config vars and their values +# search_key is the key of the field config item to check +# expected_value is the variable that search_key is set to +@pytest.mark.parametrize( + 'search_prefixes, config_overrides, expected_value', [ + (['BOTH_', 'FCST_'], + {'FCST_VAR1_': 'fcst_var1'}, + 'fcst_var1' + ), + (['BOTH_', 'FCST_'], {}, None), + + (['BOTH_', 'FCST_'], + {'FCST_VAR1_': 'fcst_var1', + 'BOTH_VAR1_': 'both_var1'}, + 'both_var1' + ), + + (['BOTH_GRID_STAT_', 'FCST_GRID_STAT_'], + {'FCST_GRID_STAT_VAR1_': 'fcst_grid_stat_var1'}, + 'fcst_grid_stat_var1' + ), + (['BOTH_GRID_STAT_', 'FCST_GRID_STAT_'], {}, None), + (['BOTH_GRID_STAT_', 'FCST_GRID_STAT_'], + {'FCST_GRID_STAT_VAR1_': 'fcst_grid_stat_var1', + 'BOTH_GRID_STAT_VAR1_': 'both_grid_stat_var1'}, + 'both_grid_stat_var1' + ), + + (['ENS_'], + {'ENS_VAR1_': 'env_var1'}, + 'env_var1' + ), + (['ENS_'], {}, None), + + ] +) +def test_get_field_config_variables(metplus_config, + search_prefixes, + config_overrides, + expected_value): + config = metplus_config() + index = '1' + field_info_types = ['name', 'levels', 'thresh', 'options', 'output_names'] + for field_info_type in field_info_types: + for key, value in config_overrides.items(): + config.set('config', + f'{key}{field_info_type.upper()}', + value) + + field_configs = config_metplus.get_field_config_variables(config, + index, + search_prefixes) + + assert(field_configs.get(field_info_type) == expected_value) + +@pytest.mark.parametrize( + 'config_keys, field_key, expected_value', [ + (['NAME', + ], + 'name', 'NAME' + ), + (['NAME', + 'INPUT_FIELD_NAME', + ], + 'name', 'NAME' + ), + (['INPUT_FIELD_NAME', + ], + 'name', 'INPUT_FIELD_NAME' + ), + ([], 'name', None), + (['LEVELS', + ], + 'levels', 'LEVELS' + ), + (['LEVELS', + 'FIELD_LEVEL', + ], + 'levels', 'LEVELS' + ), + (['FIELD_LEVEL', + ], + 'levels', 'FIELD_LEVEL' + ), + ([], 'levels', None), + (['OUTPUT_NAMES', + ], + 'output_names', 'OUTPUT_NAMES' + ), + (['OUTPUT_NAMES', + 'OUTPUT_FIELD_NAME', + ], + 'output_names', 'OUTPUT_NAMES' + ), + (['OUTPUT_FIELD_NAME', + ], + 'output_names', 'OUTPUT_FIELD_NAME' + ), + ([], 'output_names', None), + ] +) +def test_get_field_config_variables_synonyms(metplus_config, + config_keys, + field_key, + expected_value): + config = metplus_config() + index = '1' + prefix = 'BOTH_REGRID_DATA_PLANE_' + for key in config_keys: + config.set('config', f'{prefix}VAR{index}_{key}', key) + + field_configs = config_metplus.get_field_config_variables(config, + index, + [prefix]) + + assert(field_configs.get(field_key) == expected_value) + +# field info only defined in the FCST_* variables +@pytest.mark.parametrize( + 'data_type, list_created', [ + (None, False), + ('FCST', True), + ('OBS', False), + ] +) +def test_parse_var_list_fcst_only(metplus_config, data_type, list_created): + conf = metplus_config() + conf.set('config', 'FCST_VAR1_NAME', "NAME1") + conf.set('config', 'FCST_VAR1_LEVELS', "LEVELS11, LEVELS12") + conf.set('config', 'FCST_VAR2_NAME', "NAME2") + conf.set('config', 'FCST_VAR2_LEVELS', "LEVELS21, LEVELS22") + + # this should not occur because OBS variables are missing + if config_metplus.validate_configuration_variables(conf, force_check=True)[1]: + assert(False) + + var_list = config_metplus.parse_var_list(conf, time_info=None, data_type=data_type) + + # list will be created if requesting just OBS, but it should not be created if + # nothing was requested because FCST values are missing + if list_created: + assert(var_list[0]['fcst_name'] == "NAME1" and \ + var_list[1]['fcst_name'] == "NAME1" and \ + var_list[2]['fcst_name'] == "NAME2" and \ + var_list[3]['fcst_name'] == "NAME2" and \ + var_list[0]['fcst_level'] == "LEVELS11" and \ + var_list[1]['fcst_level'] == "LEVELS12" and \ + var_list[2]['fcst_level'] == "LEVELS21" and \ + var_list[3]['fcst_level'] == "LEVELS22") + else: + assert(not var_list) + +# field info only defined in the OBS_* variables +@pytest.mark.parametrize( + 'data_type, list_created', [ + (None, False), + ('OBS', True), + ('FCST', False), + ] +) +def test_parse_var_list_obs(metplus_config, data_type, list_created): + conf = metplus_config() + conf.set('config', 'OBS_VAR1_NAME', "NAME1") + conf.set('config', 'OBS_VAR1_LEVELS', "LEVELS11, LEVELS12") + conf.set('config', 'OBS_VAR2_NAME', "NAME2") + conf.set('config', 'OBS_VAR2_LEVELS', "LEVELS21, LEVELS22") + + # this should not occur because FCST variables are missing + if config_metplus.validate_configuration_variables(conf, force_check=True)[1]: + assert(False) + + var_list = config_metplus.parse_var_list(conf, time_info=None, data_type=data_type) + + # list will be created if requesting just OBS, but it should not be created if + # nothing was requested because FCST values are missing + if list_created: + assert(var_list[0]['obs_name'] == "NAME1" and \ + var_list[1]['obs_name'] == "NAME1" and \ + var_list[2]['obs_name'] == "NAME2" and \ + var_list[3]['obs_name'] == "NAME2" and \ + var_list[0]['obs_level'] == "LEVELS11" and \ + var_list[1]['obs_level'] == "LEVELS12" and \ + var_list[2]['obs_level'] == "LEVELS21" and \ + var_list[3]['obs_level'] == "LEVELS22") + else: + assert(not var_list) + + +# field info only defined in the BOTH_* variables +@pytest.mark.parametrize( + 'data_type, list_created', [ + (None, 'fcst:obs'), + ('FCST', 'fcst'), + ('OBS', 'obs'), + ] +) +def test_parse_var_list_both(metplus_config, data_type, list_created): + conf = metplus_config() + conf.set('config', 'BOTH_VAR1_NAME', "NAME1") + conf.set('config', 'BOTH_VAR1_LEVELS', "LEVELS11, LEVELS12") + conf.set('config', 'BOTH_VAR2_NAME', "NAME2") + conf.set('config', 'BOTH_VAR2_LEVELS', "LEVELS21, LEVELS22") + + # this should not occur because BOTH variables are used + if not config_metplus.validate_configuration_variables(conf, force_check=True)[1]: + assert(False) + + var_list = config_metplus.parse_var_list(conf, time_info=None, data_type=data_type) + print(f'var_list:{var_list}') + for list_to_check in list_created.split(':'): + if not var_list[0][f'{list_to_check}_name'] == "NAME1" or \ + not var_list[1][f'{list_to_check}_name'] == "NAME1" or \ + not var_list[2][f'{list_to_check}_name'] == "NAME2" or \ + not var_list[3][f'{list_to_check}_name'] == "NAME2" or \ + not var_list[0][f'{list_to_check}_level'] == "LEVELS11" or \ + not var_list[1][f'{list_to_check}_level'] == "LEVELS12" or \ + not var_list[2][f'{list_to_check}_level'] == "LEVELS21" or \ + not var_list[3][f'{list_to_check}_level'] == "LEVELS22": + assert(False) + +# field info defined in both FCST_* and OBS_* variables +def test_parse_var_list_fcst_and_obs(metplus_config): + conf = metplus_config() + conf.set('config', 'FCST_VAR1_NAME', "FNAME1") + conf.set('config', 'FCST_VAR1_LEVELS', "FLEVELS11, FLEVELS12") + conf.set('config', 'FCST_VAR2_NAME', "FNAME2") + conf.set('config', 'FCST_VAR2_LEVELS', "FLEVELS21, FLEVELS22") + conf.set('config', 'OBS_VAR1_NAME', "ONAME1") + conf.set('config', 'OBS_VAR1_LEVELS', "OLEVELS11, OLEVELS12") + conf.set('config', 'OBS_VAR2_NAME', "ONAME2") + conf.set('config', 'OBS_VAR2_LEVELS', "OLEVELS21, OLEVELS22") + + # this should not occur because FCST and OBS variables are found + if not config_metplus.validate_configuration_variables(conf, force_check=True)[1]: + assert(False) + + var_list = config_metplus.parse_var_list(conf) + + assert(var_list[0]['fcst_name'] == "FNAME1" and \ + var_list[0]['obs_name'] == "ONAME1" and \ + var_list[1]['fcst_name'] == "FNAME1" and \ + var_list[1]['obs_name'] == "ONAME1" and \ + var_list[2]['fcst_name'] == "FNAME2" and \ + var_list[2]['obs_name'] == "ONAME2" and \ + var_list[3]['fcst_name'] == "FNAME2" and \ + var_list[3]['obs_name'] == "ONAME2" and \ + var_list[0]['fcst_level'] == "FLEVELS11" and \ + var_list[0]['obs_level'] == "OLEVELS11" and \ + var_list[1]['fcst_level'] == "FLEVELS12" and \ + var_list[1]['obs_level'] == "OLEVELS12" and \ + var_list[2]['fcst_level'] == "FLEVELS21" and \ + var_list[2]['obs_level'] == "OLEVELS21" and \ + var_list[3]['fcst_level'] == "FLEVELS22" and \ + var_list[3]['obs_level'] == "OLEVELS22") + +# VAR1 defined by FCST, VAR2 defined by OBS +def test_parse_var_list_fcst_and_obs_alternate(metplus_config): + conf = metplus_config() + conf.set('config', 'FCST_VAR1_NAME', "FNAME1") + conf.set('config', 'FCST_VAR1_LEVELS', "FLEVELS11, FLEVELS12") + conf.set('config', 'OBS_VAR2_NAME', "ONAME2") + conf.set('config', 'OBS_VAR2_LEVELS', "OLEVELS21, OLEVELS22") + + # configuration is invalid and parse var list should not give any results + assert(not config_metplus.validate_configuration_variables(conf, force_check=True)[1] and not config_metplus.parse_var_list(conf)) + +# VAR1 defined by OBS, VAR2 by FCST, VAR3 by both FCST AND OBS +@pytest.mark.parametrize( + 'data_type, list_len, name_levels', [ + (None, 0, None), + ('FCST', 4, ('FNAME2:FLEVELS21','FNAME2:FLEVELS22','FNAME3:FLEVELS31','FNAME3:FLEVELS32')), + ('OBS', 4, ('ONAME1:OLEVELS11','ONAME1:OLEVELS12','ONAME3:OLEVELS31','ONAME3:OLEVELS32')), + ] +) +def test_parse_var_list_fcst_and_obs_and_both(metplus_config, data_type, list_len, name_levels): + conf = metplus_config() + conf.set('config', 'OBS_VAR1_NAME', "ONAME1") + conf.set('config', 'OBS_VAR1_LEVELS', "OLEVELS11, OLEVELS12") + conf.set('config', 'FCST_VAR2_NAME', "FNAME2") + conf.set('config', 'FCST_VAR2_LEVELS', "FLEVELS21, FLEVELS22") + conf.set('config', 'FCST_VAR3_NAME', "FNAME3") + conf.set('config', 'FCST_VAR3_LEVELS', "FLEVELS31, FLEVELS32") + conf.set('config', 'OBS_VAR3_NAME', "ONAME3") + conf.set('config', 'OBS_VAR3_LEVELS', "OLEVELS31, OLEVELS32") + + # configuration is invalid and parse var list should not give any results + if config_metplus.validate_configuration_variables(conf, force_check=True)[1]: + assert(False) + + var_list = config_metplus.parse_var_list(conf, time_info=None, data_type=data_type) + + if len(var_list) != list_len: + assert(False) + + if data_type is None: + assert(len(var_list) == 0) + + if name_levels is not None: + dt_lower = data_type.lower() + expected = [] + for name_level in name_levels: + name, level = name_level.split(':') + expected.append({f'{dt_lower}_name': name, + f'{dt_lower}_level': level}) + + for expect, reality in zip(expected,var_list): + if expect[f'{dt_lower}_name'] != reality[f'{dt_lower}_name']: + assert(False) + + if expect[f'{dt_lower}_level'] != reality[f'{dt_lower}_level']: + assert(False) + + assert(True) + +# option defined in obs only +@pytest.mark.parametrize( + 'data_type, list_len', [ + (None, 0), + ('FCST', 2), + ('OBS', 0), + ] +) +def test_parse_var_list_fcst_only_options(metplus_config, data_type, list_len): + conf = metplus_config() + conf.set('config', 'FCST_VAR1_NAME', "NAME1") + conf.set('config', 'FCST_VAR1_LEVELS', "LEVELS11, LEVELS12") + conf.set('config', 'FCST_VAR1_THRESH', ">1, >2") + conf.set('config', 'OBS_VAR1_OPTIONS', "OOPTIONS11") + + # this should not occur because OBS variables are missing + if config_metplus.validate_configuration_variables(conf, force_check=True)[1]: + assert(False) + + var_list = config_metplus.parse_var_list(conf, time_info=None, data_type=data_type) + + assert(len(var_list) == list_len) + +@pytest.mark.parametrize( + 'met_tool, indices', [ + (None, {'1': ['FCST']}), + ('GRID_STAT', {'2': ['FCST']}), + ('ENSEMBLE_STAT', {}), + ] +) +def test_find_var_indices_wrapper_specific(metplus_config, met_tool, indices): + conf = metplus_config() + data_type = 'FCST' + conf.set('config', f'{data_type}_VAR1_NAME', "NAME1") + conf.set('config', f'{data_type}_GRID_STAT_VAR2_NAME', "GSNAME2") + + var_name_indices = config_metplus.find_var_name_indices(conf, data_types=[data_type], + met_tool=met_tool) + + assert(var_name_indices == indices) + +# ensure that the field configuration used for +# met_tool_wrapper/EnsembleStat/EnsembleStat.conf +# works as expected +def test_parse_var_list_ensemble(metplus_config): + config = metplus_config() + config.set('config', 'ENS_VAR1_NAME', 'APCP') + config.set('config', 'ENS_VAR1_LEVELS', 'A24') + config.set('config', 'ENS_VAR1_THRESH', '>0.0, >=10.0') + config.set('config', 'ENS_VAR2_NAME', 'REFC') + config.set('config', 'ENS_VAR2_LEVELS', 'L0') + config.set('config', 'ENS_VAR2_THRESH', '>35.0') + config.set('config', 'ENS_VAR2_OPTIONS', 'GRIB1_ptv = 129;') + config.set('config', 'ENS_VAR3_NAME', 'UGRD') + config.set('config', 'ENS_VAR3_LEVELS', 'Z10') + config.set('config', 'ENS_VAR3_THRESH', '>=5.0') + config.set('config', 'ENS_VAR4_NAME', 'VGRD') + config.set('config', 'ENS_VAR4_LEVELS', 'Z10') + config.set('config', 'ENS_VAR4_THRESH', '>=5.0') + config.set('config', 'ENS_VAR5_NAME', 'WIND') + config.set('config', 'ENS_VAR5_LEVELS', 'Z10') + config.set('config', 'ENS_VAR5_THRESH', '>=5.0') + config.set('config', 'FCST_VAR1_NAME', 'APCP') + config.set('config', 'FCST_VAR1_LEVELS', 'A24') + config.set('config', 'FCST_VAR1_THRESH', '>0.01, >=10.0') + config.set('config', 'FCST_VAR1_OPTIONS', ('ens_ssvar_bin_size = 0.1; ' + 'ens_phist_bin_size = 0.05;')) + config.set('config', 'OBS_VAR1_NAME', 'APCP') + config.set('config', 'OBS_VAR1_LEVELS', 'A24') + config.set('config', 'OBS_VAR1_THRESH', '>0.01, >=10.0') + config.set('config', 'OBS_VAR1_OPTIONS', ('ens_ssvar_bin_size = 0.1; ' + 'ens_phist_bin_size = 0.05;')) + time_info = {} + + expected_ens_list = [{'index': '1', + 'ens_name': 'APCP', + 'ens_level': 'A24', + 'ens_thresh': ['>0.0', '>=10.0']}, + {'index': '2', + 'ens_name': 'REFC', + 'ens_level': 'L0', + 'ens_thresh': ['>35.0']}, + {'index': '3', + 'ens_name': 'UGRD', + 'ens_level': 'Z10', + 'ens_thresh': ['>=5.0']}, + {'index': '4', + 'ens_name': 'VGRD', + 'ens_level': 'Z10', + 'ens_thresh': ['>=5.0']}, + {'index': '5', + 'ens_name': 'WIND', + 'ens_level': 'Z10', + 'ens_thresh': ['>=5.0']}, + ] + expected_var_list = [{'index': '1', + 'fcst_name': 'APCP', + 'fcst_level': 'A24', + 'fcst_thresh': ['>0.01', '>=10.0'], + 'fcst_extra': ('ens_ssvar_bin_size = 0.1; ' + 'ens_phist_bin_size = 0.05;'), + 'obs_name': 'APCP', + 'obs_level': 'A24', + 'obs_thresh': ['>0.01', '>=10.0'], + 'obs_extra': ('ens_ssvar_bin_size = 0.1; ' + 'ens_phist_bin_size = 0.05;') + + }, + ] + + ensemble_var_list = config_metplus.parse_var_list(config, time_info, + data_type='ENS') + + # parse optional var list for FCST and/or OBS fields + var_list = config_metplus.parse_var_list(config, time_info, + met_tool='ensemble_stat') + + pp = pprint.PrettyPrinter() + print(f'ENSEMBLE_VAR_LIST:') + pp.pprint(ensemble_var_list) + print(f'VAR_LIST:') + pp.pprint(var_list) + + assert(len(ensemble_var_list) == len(expected_ens_list)) + for actual_ens, expected_ens in zip(ensemble_var_list, expected_ens_list): + for key, value in expected_ens.items(): + assert(actual_ens.get(key) == value) + + assert(len(var_list) == len(expected_var_list)) + for actual_var, expected_var in zip(var_list, expected_var_list): + for key, value in expected_var.items(): + assert(actual_var.get(key) == value) + +def test_parse_var_list_series_by(metplus_config): + config = metplus_config() + config.set('config', 'BOTH_EXTRACT_TILES_VAR1_NAME', 'RH') + config.set('config', 'BOTH_EXTRACT_TILES_VAR1_LEVELS', 'P850, P700') + config.set('config', 'BOTH_EXTRACT_TILES_VAR1_OUTPUT_NAMES', + 'RH_850mb, RH_700mb') + + config.set('config', 'BOTH_SERIES_ANALYSIS_VAR1_NAME', 'RH_850mb') + config.set('config', 'BOTH_SERIES_ANALYSIS_VAR1_LEVELS', 'P850') + config.set('config', 'BOTH_SERIES_ANALYSIS_VAR2_NAME', 'RH_700mb') + config.set('config', 'BOTH_SERIES_ANALYSIS_VAR2_LEVELS', 'P700') + time_info = {} + + expected_et_list = [{'index': '1', + 'fcst_name': 'RH', + 'fcst_level': 'P850', + 'fcst_output_name': 'RH_850mb', + 'obs_name': 'RH', + 'obs_level': 'P850', + 'obs_output_name': 'RH_850mb', + }, + {'index': '1', + 'fcst_name': 'RH', + 'fcst_level': 'P700', + 'fcst_output_name': 'RH_700mb', + 'obs_name': 'RH', + 'obs_level': 'P700', + 'obs_output_name': 'RH_700mb', + }, + ] + expected_sa_list = [{'index': '1', + 'fcst_name': 'RH_850mb', + 'fcst_level': 'P850', + 'obs_name': 'RH_850mb', + 'obs_level': 'P850', + }, + {'index': '2', + 'fcst_name': 'RH_700mb', + 'fcst_level': 'P700', + 'obs_name': 'RH_700mb', + 'obs_level': 'P700', + }, + ] + + actual_et_list = config_metplus.parse_var_list(config, + time_info=time_info, + met_tool='extract_tiles') + + actual_sa_list = config_metplus.parse_var_list(config, + met_tool='series_analysis') + + pp = pprint.PrettyPrinter() + print(f'ExtractTiles var list:') + pp.pprint(actual_et_list) + print(f'SeriesAnalysis var list:') + pp.pprint(actual_sa_list) + + assert(len(actual_et_list) == len(expected_et_list)) + for actual_et, expected_et in zip(actual_et_list, expected_et_list): + for key, value in expected_et.items(): + assert(actual_et.get(key) == value) + + assert(len(actual_sa_list) == len(expected_sa_list)) + for actual_sa, expected_sa in zip(actual_sa_list, expected_sa_list): + for key, value in expected_sa.items(): + assert(actual_sa.get(key) == value) + +def test_parse_var_list_priority_fcst(metplus_config): + priority_list = ['FCST_GRID_STAT_VAR1_NAME', + 'FCST_GRID_STAT_VAR1_INPUT_FIELD_NAME', + 'FCST_GRID_STAT_VAR1_FIELD_NAME', + 'BOTH_GRID_STAT_VAR1_NAME', + 'BOTH_GRID_STAT_VAR1_INPUT_FIELD_NAME', + 'BOTH_GRID_STAT_VAR1_FIELD_NAME', + 'FCST_VAR1_NAME', + 'FCST_VAR1_INPUT_FIELD_NAME', + 'FCST_VAR1_FIELD_NAME', + 'BOTH_VAR1_NAME', + 'BOTH_VAR1_INPUT_FIELD_NAME', + 'BOTH_VAR1_FIELD_NAME', + ] + time_info = {} + + # loop through priority list, process, then pop first value off and + # process again until all items have been popped. + # This will check that list is in priority order + while(priority_list): + config = metplus_config() + for key in priority_list: + config.set('config', key, key.lower()) + + var_list = config_metplus.parse_var_list(config, time_info=time_info, + data_type='FCST', + met_tool='grid_stat') + + assert(len(var_list) == 1) + assert(var_list[0].get('fcst_name') == priority_list[0].lower()) + priority_list.pop(0) + +# test that if wrapper specific field info is specified, it only gets +# values from that list. All generic values should be read if no +# wrapper specific field info variables are specified +def test_parse_var_list_wrapper_specific(metplus_config): + conf = metplus_config() + conf.set('config', 'FCST_VAR1_NAME', "ENAME1") + conf.set('config', 'FCST_VAR1_LEVELS', "ELEVELS11, ELEVELS12") + conf.set('config', 'FCST_VAR2_NAME', "ENAME2") + conf.set('config', 'FCST_VAR2_LEVELS', "ELEVELS21, ELEVELS22") + conf.set('config', 'FCST_GRID_STAT_VAR1_NAME', "GNAME1") + conf.set('config', 'FCST_GRID_STAT_VAR1_LEVELS', "GLEVELS11, GLEVELS12") + + e_var_list = config_metplus.parse_var_list(conf, + time_info=None, + data_type='FCST', + met_tool='ensemble_stat') + + g_var_list = config_metplus.parse_var_list(conf, + time_info=None, + data_type='FCST', + met_tool='grid_stat') + + assert(len(e_var_list) == 4 and len(g_var_list) == 2 and + e_var_list[0]['fcst_name'] == "ENAME1" and + e_var_list[1]['fcst_name'] == "ENAME1" and + e_var_list[2]['fcst_name'] == "ENAME2" and + e_var_list[3]['fcst_name'] == "ENAME2" and + e_var_list[0]['fcst_level'] == "ELEVELS11" and + e_var_list[1]['fcst_level'] == "ELEVELS12" and + e_var_list[2]['fcst_level'] == "ELEVELS21" and + e_var_list[3]['fcst_level'] == "ELEVELS22" and + g_var_list[0]['fcst_name'] == "GNAME1" and + g_var_list[1]['fcst_name'] == "GNAME1" and + g_var_list[0]['fcst_level'] == "GLEVELS11" and + g_var_list[1]['fcst_level'] == "GLEVELS12") + +@pytest.mark.parametrize( + 'config_overrides, expected_results', [ + # 2 levels + ({'FCST_VAR1_NAME': 'read_data.py TMP {valid?fmt=%Y%m%d} {fcst_level}', + 'FCST_VAR1_LEVELS': 'P500,P250', + 'OBS_VAR1_NAME': 'read_data.py TMP {valid?fmt=%Y%m%d} {obs_level}', + 'OBS_VAR1_LEVELS': 'P500,P250', + }, + ['read_data.py TMP 20200201 P500', + 'read_data.py TMP 20200201 P250', + ]), + ({'BOTH_VAR1_NAME': 'read_data.py TMP {valid?fmt=%Y%m%d} {fcst_level}', + 'BOTH_VAR1_LEVELS': 'P500,P250', + }, + ['read_data.py TMP 20200201 P500', + 'read_data.py TMP 20200201 P250', + ]), + # no level but level specified in name + ({'FCST_VAR1_NAME': 'read_data.py TMP {valid?fmt=%Y%m%d} {fcst_level}', + 'OBS_VAR1_NAME': 'read_data.py TMP {valid?fmt=%Y%m%d} {obs_level}', + }, + ['read_data.py TMP 20200201 ', + ]), + # no level + ({'FCST_VAR1_NAME': 'read_data.py TMP {valid?fmt=%Y%m%d}', + 'OBS_VAR1_NAME': 'read_data.py TMP {valid?fmt=%Y%m%d}', + }, + ['read_data.py TMP 20200201', + ]), + # real example + ({'BOTH_VAR1_NAME': ('myscripts/read_nc2xr.py ' + 'mydata/forecast_file.nc4 TMP ' + '{valid?fmt=%Y%m%d_%H%M} {fcst_level}'), + 'BOTH_VAR1_LEVELS': 'P1000,P850,P700,P500,P250,P100', + }, + [('myscripts/read_nc2xr.py mydata/forecast_file.nc4 TMP 20200201_1225' + ' P1000'), + ('myscripts/read_nc2xr.py mydata/forecast_file.nc4 TMP 20200201_1225' + ' P850'), + ('myscripts/read_nc2xr.py mydata/forecast_file.nc4 TMP 20200201_1225' + ' P700'), + ('myscripts/read_nc2xr.py mydata/forecast_file.nc4 TMP 20200201_1225' + ' P500'), + ('myscripts/read_nc2xr.py mydata/forecast_file.nc4 TMP 20200201_1225' + ' P250'), + ('myscripts/read_nc2xr.py mydata/forecast_file.nc4 TMP 20200201_1225' + ' P100'), + ]), + ] +) +def test_parse_var_list_py_embed_multi_levels(metplus_config, config_overrides, + expected_results): + config = metplus_config() + for key, value in config_overrides.items(): + config.set('config', key, value) + + time_info = {'valid': datetime(2020, 2, 1, 12, 25)} + var_list = config_metplus.parse_var_list(config, + time_info=time_info, + data_type=None) + assert(len(var_list) == len(expected_results)) + + for var_item, expected_result in zip(var_list, expected_results): + assert(var_item['fcst_name'] == expected_result) + + # run again with data type specified + var_list = config_metplus.parse_var_list(config, + time_info=time_info, + data_type='FCST') + assert(len(var_list) == len(expected_results)) + + for var_item, expected_result in zip(var_list, expected_results): + assert(var_item['fcst_name'] == expected_result) + + +@pytest.mark.parametrize( + 'input_list, expected_list', [ + ('Point2Grid', ['Point2Grid']), + # MET documentation syntax (with dashes) + ('Pcp-Combine, Grid-Stat, Ensemble-Stat', ['PCPCombine', + 'GridStat', + 'EnsembleStat']), + ('Point-Stat', ['PointStat']), + ('Mode, MODE Time Domain', ['MODE', + 'MTD']), + # actual tool name (lower case underscore) + ('point_stat, grid_stat, ensemble_stat', ['PointStat', + 'GridStat', + 'EnsembleStat']), + ('mode, mtd', ['MODE', + 'MTD']), + ('ascii2nc, pb2nc, regrid_data_plane', ['ASCII2NC', + 'PB2NC', + 'RegridDataPlane']), + ('pcp_combine, tc_pairs, tc_stat', ['PCPCombine', + 'TCPairs', + 'TCStat']), + ('gen_vx_mask, stat_analysis, series_analysis', ['GenVxMask', + 'StatAnalysis', + 'SeriesAnalysis']), + # old capitalization format + ('PcpCombine, Ascii2Nc, TcStat, TcPairs', ['PCPCombine', + 'ASCII2NC', + 'TCStat', + 'TCPairs']), + # remove MakePlots from list + ('StatAnalysis, MakePlots', ['StatAnalysis']), + ] +) +def test_get_process_list(metplus_config, input_list, expected_list): + conf = metplus_config() + conf.set('config', 'PROCESS_LIST', input_list) + process_list = config_metplus.get_process_list(conf) + output_list = [item[0] for item in process_list] + assert(output_list == expected_list) + +@pytest.mark.parametrize( + 'input_list, expected_list', [ + # no instances + ('Point2Grid', [('Point2Grid', None)]), + # one with instance one without + ('PcpCombine, GridStat(my_instance)', [('PCPCombine', None), + ('GridStat', 'my_instance')]), + # duplicate process, one with instance one without + ('TCStat, ExtractTiles, TCStat(for_series), SeriesAnalysis', ( + [('TCStat',None), + ('ExtractTiles',None), + ('TCStat', 'for_series'), + ('SeriesAnalysis',None),])), + # two processes, both with instances + ('mode(uno), mtd(dos)', [('MODE', 'uno'), + ('MTD', 'dos')]), + # lower-case names, first with instance, second without + ('ascii2nc(some_name), pb2nc', [('ASCII2NC', 'some_name'), + ('PB2NC', None)]), + # duplicate process, both with different instances + ('tc_stat(one), tc_pairs, tc_stat(two)', [('TCStat', 'one'), + ('TCPairs', None), + ('TCStat', 'two')]), + ] +) +def test_get_process_list_instances(metplus_config, input_list, expected_list): + conf = metplus_config() + conf.set('config', 'PROCESS_LIST', input_list) + output_list = config_metplus.get_process_list(conf) + assert(output_list == expected_list) \ No newline at end of file diff --git a/internal_tests/pytests/ensemble_stat/test_ensemble_stat_wrapper.py b/internal_tests/pytests/ensemble_stat/test_ensemble_stat_wrapper.py index be6ee1acb4..1f864709a1 100644 --- a/internal_tests/pytests/ensemble_stat/test_ensemble_stat_wrapper.py +++ b/internal_tests/pytests/ensemble_stat/test_ensemble_stat_wrapper.py @@ -137,7 +137,8 @@ def test_handle_climo_file_variables(metplus_config, config_overrides, ({'ENSEMBLE_STAT_REGRID_TO_GRID': 'FCST', }, - {'METPLUS_REGRID_DICT': 'regrid = {to_grid = FCST;}'}), + {'METPLUS_REGRID_DICT': 'regrid = {to_grid = FCST;}', + 'REGRID_TO_GRID': 'FCST'}), ({'ENSEMBLE_STAT_REGRID_METHOD': 'NEAREST', }, @@ -163,7 +164,8 @@ def test_handle_climo_file_variables(metplus_config, config_overrides, }, {'METPLUS_REGRID_DICT': ('regrid = {to_grid = FCST;method = NEAREST;' 'width = 1;vld_thresh = 0.5;shape = SQUARE;}' - )}), + ), + 'REGRID_TO_GRID': 'FCST'}), ({'ENSEMBLE_STAT_CLIMO_MEAN_INPUT_TEMPLATE': '/some/path/climo/filename.nc', @@ -575,7 +577,6 @@ def test_ensemble_stat_single_field(metplus_config, config_overrides, f"{config_file} -outdir {out_dir}/2005080800"), ] - all_cmds = wrapper.run_all_times() print(f"ALL COMMANDS: {all_cmds}") assert len(all_cmds) == len(expected_cmds) @@ -585,7 +586,11 @@ def test_ensemble_stat_single_field(metplus_config, config_overrides, assert(cmd == expected_cmd) # check that environment variables were set properly - for env_var_key in wrapper.WRAPPER_ENV_VAR_KEYS: + # including deprecated env vars (not in wrapper env var keys) + env_var_keys = (wrapper.WRAPPER_ENV_VAR_KEYS + + [name for name in env_var_values + if name not in wrapper.WRAPPER_ENV_VAR_KEYS]) + for env_var_key in env_var_keys: match = next((item for item in env_vars if item.startswith(env_var_key)), None) assert(match is not None) diff --git a/internal_tests/pytests/gen_ens_prod/test_gen_ens_prod_wrapper.py b/internal_tests/pytests/gen_ens_prod/test_gen_ens_prod_wrapper.py index ed27e0784f..60703aecb2 100644 --- a/internal_tests/pytests/gen_ens_prod/test_gen_ens_prod_wrapper.py +++ b/internal_tests/pytests/gen_ens_prod/test_gen_ens_prod_wrapper.py @@ -53,35 +53,36 @@ def set_minimum_config_settings(config): @pytest.mark.parametrize( 'config_overrides, env_var_values', [ + # 0 ({'MODEL': 'my_model'}, {'METPLUS_MODEL': 'model = "my_model";'}), - + # 1 ({'GEN_ENS_PROD_DESC': 'my_desc'}, {'METPLUS_DESC': 'desc = "my_desc";'}), - + # 2 ({'DESC': 'my_desc'}, {'METPLUS_DESC': 'desc = "my_desc";'}), - + # 3 ({'GEN_ENS_PROD_REGRID_TO_GRID': 'FCST', }, {'METPLUS_REGRID_DICT': 'regrid = {to_grid = FCST;}'}), - + # 4 ({'GEN_ENS_PROD_REGRID_METHOD': 'NEAREST', }, {'METPLUS_REGRID_DICT': 'regrid = {method = NEAREST;}'}), - + # 5 ({'GEN_ENS_PROD_REGRID_WIDTH': '1', }, {'METPLUS_REGRID_DICT': 'regrid = {width = 1;}'}), - + # 6 ({'GEN_ENS_PROD_REGRID_VLD_THRESH': '0.5', }, {'METPLUS_REGRID_DICT': 'regrid = {vld_thresh = 0.5;}'}), - + # 7 ({'GEN_ENS_PROD_REGRID_SHAPE': 'SQUARE', }, {'METPLUS_REGRID_DICT': 'regrid = {shape = SQUARE;}'}), - + # 8 ({'GEN_ENS_PROD_REGRID_TO_GRID': 'FCST', 'GEN_ENS_PROD_REGRID_METHOD': 'NEAREST', 'GEN_ENS_PROD_REGRID_WIDTH': '1', @@ -91,110 +92,65 @@ def set_minimum_config_settings(config): {'METPLUS_REGRID_DICT': ('regrid = {to_grid = FCST;method = NEAREST;' 'width = 1;vld_thresh = 0.5;shape = SQUARE;}' )}), - + # 9 ({'GEN_ENS_PROD_CLIMO_MEAN_INPUT_TEMPLATE': '/some/path/climo/filename.nc', }, {'METPLUS_CLIMO_MEAN_DICT': 'climo_mean = {file_name = ["/some/path/climo/filename.nc"];}', - 'CLIMO_MEAN_FILE': - '"/some/path/climo/filename.nc"', }), + # 10 ({'GEN_ENS_PROD_CLIMO_STDEV_INPUT_TEMPLATE': '/some/path/climo/stdfile.nc', }, {'METPLUS_CLIMO_STDEV_DICT': 'climo_stdev = {file_name = ["/some/path/climo/stdfile.nc"];}', - 'CLIMO_STDEV_FILE': - '"/some/path/climo/stdfile.nc"', }), - # 12 mask grid and poly (old config var) - ({'GEN_ENS_PROD_MASK_GRID': 'FULL', - 'GEN_ENS_PROD_VERIFICATION_MASK_TEMPLATE': 'one, two', - }, - {'METPLUS_MASK_GRID': - 'grid = ["FULL"];', - 'METPLUS_MASK_POLY': - 'poly = ["one","two"];', - }), - # 13 mask grid and poly (new config var) - ({'GEN_ENS_PROD_MASK_GRID': 'FULL', - 'GEN_ENS_PROD_MASK_POLY': 'one, two', - }, - {'METPLUS_MASK_GRID': - 'grid = ["FULL"];', - 'METPLUS_MASK_POLY': - 'poly = ["one","two"];', - }), - # 14 mask grid value - ({'GEN_ENS_PROD_MASK_GRID': 'FULL', - }, - {'METPLUS_MASK_GRID': - 'grid = ["FULL"];', - }), - # 15 mask grid empty string (should create empty list) - ({'GEN_ENS_PROD_MASK_GRID': '', - }, - {'METPLUS_MASK_GRID': - 'grid = [];', - }), - # 16 mask poly (old config var) - ({'GEN_ENS_PROD_VERIFICATION_MASK_TEMPLATE': 'one, two', - }, - {'METPLUS_MASK_POLY': - 'poly = ["one","two"];', - }), - # 27 mask poly (new config var) - ({'GEN_ENS_PROD_MASK_POLY': 'one, two', - }, - {'METPLUS_MASK_POLY': - 'poly = ["one","two"];', - }), - # ensemble_flag + # 11 ensemble_flag ({'GEN_ENS_PROD_ENSEMBLE_FLAG_LATLON': 'FALSE', }, {'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {latlon = FALSE;}'}), - + # 12 ({'GEN_ENS_PROD_ENSEMBLE_FLAG_MEAN': 'FALSE', }, {'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {mean = FALSE;}'}), - + # 13 ({'GEN_ENS_PROD_ENSEMBLE_FLAG_STDEV': 'FALSE', }, {'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {stdev = FALSE;}'}), - + # 14 ({'GEN_ENS_PROD_ENSEMBLE_FLAG_MINUS': 'FALSE', }, {'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {minus = FALSE;}'}), - + # 15 ({'GEN_ENS_PROD_ENSEMBLE_FLAG_PLUS': 'FALSE', }, {'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {plus = FALSE;}'}), - + # 16 ({'GEN_ENS_PROD_ENSEMBLE_FLAG_MIN': 'FALSE', }, {'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {min = FALSE;}'}), - + # 17 ({'GEN_ENS_PROD_ENSEMBLE_FLAG_MAX': 'FALSE', }, {'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {max = FALSE;}'}), - + # 18 ({'GEN_ENS_PROD_ENSEMBLE_FLAG_RANGE': 'FALSE', }, {'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {range = FALSE;}'}), - + # 19 ({'GEN_ENS_PROD_ENSEMBLE_FLAG_VLD_COUNT': 'FALSE', }, { 'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {vld_count = FALSE;}'}), - + # 20 ({'GEN_ENS_PROD_ENSEMBLE_FLAG_FREQUENCY': 'FALSE', }, { 'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {frequency = FALSE;}'}), - + # 21 ({'GEN_ENS_PROD_ENSEMBLE_FLAG_NEP': 'FALSE', }, {'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {nep = FALSE;}'}), - + # 22 ({'GEN_ENS_PROD_ENSEMBLE_FLAG_NMEP': 'FALSE', }, {'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {nmep = FALSE;}'}), - + # 23 ({'GEN_ENS_PROD_ENSEMBLE_FLAG_RANK': 'FALSE', }, {'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {rank = FALSE;}'}), - + # 24 ({'GEN_ENS_PROD_ENSEMBLE_FLAG_WEIGHT': 'FALSE', }, {'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {weight = FALSE;}'}), - + # 25 ({ 'GEN_ENS_PROD_ENSEMBLE_FLAG_LATLON': 'FALSE', 'GEN_ENS_PROD_ENSEMBLE_FLAG_MEAN': 'FALSE', @@ -220,41 +176,40 @@ def set_minimum_config_settings(config): 'frequency = FALSE;nep = FALSE;' 'nmep = FALSE;rank = FALSE;' 'weight = FALSE;}')}), - + # 26 ({'GEN_ENS_PROD_CLIMO_MEAN_FILE_NAME': '/some/climo_mean/file.txt', }, {'METPLUS_CLIMO_MEAN_DICT': ('climo_mean = {file_name = ' - '["/some/climo_mean/file.txt"];}'), - 'CLIMO_MEAN_FILE': '"/some/climo_mean/file.txt"'}), - + '["/some/climo_mean/file.txt"];}'),}), + # 27 ({'GEN_ENS_PROD_CLIMO_MEAN_FIELD': '{name="CLM_NAME"; level="(0,0,*,*)";}', }, {'METPLUS_CLIMO_MEAN_DICT': 'climo_mean = {field = [{name="CLM_NAME"; level="(0,0,*,*)";}];}'}), - + # 28 ({'GEN_ENS_PROD_CLIMO_MEAN_REGRID_METHOD': 'NEAREST', }, {'METPLUS_CLIMO_MEAN_DICT': 'climo_mean = {regrid = {method = NEAREST;}}'}), - + # 29 ({'GEN_ENS_PROD_CLIMO_MEAN_REGRID_WIDTH': '1', }, {'METPLUS_CLIMO_MEAN_DICT': 'climo_mean = {regrid = {width = 1;}}'}), - + # 30 ({'GEN_ENS_PROD_CLIMO_MEAN_REGRID_VLD_THRESH': '0.5', }, { 'METPLUS_CLIMO_MEAN_DICT': 'climo_mean = {regrid = {vld_thresh = 0.5;}}'}), - + # 31 ({'GEN_ENS_PROD_CLIMO_MEAN_REGRID_SHAPE': 'SQUARE', }, {'METPLUS_CLIMO_MEAN_DICT': 'climo_mean = {regrid = {shape = SQUARE;}}'}), - + # 32 ({'GEN_ENS_PROD_CLIMO_MEAN_TIME_INTERP_METHOD': 'NEAREST', }, { 'METPLUS_CLIMO_MEAN_DICT': 'climo_mean = {time_interp_method = NEAREST;}'}), - + # 33 ({'GEN_ENS_PROD_CLIMO_MEAN_MATCH_MONTH': 'True', }, {'METPLUS_CLIMO_MEAN_DICT': 'climo_mean = {match_month = TRUE;}'}), - + # 34 ({'GEN_ENS_PROD_CLIMO_MEAN_DAY_INTERVAL': '30', }, {'METPLUS_CLIMO_MEAN_DICT': 'climo_mean = {day_interval = 30;}'}), - + # 35 ({'GEN_ENS_PROD_CLIMO_MEAN_HOUR_INTERVAL': '12', }, {'METPLUS_CLIMO_MEAN_DICT': 'climo_mean = {hour_interval = 12;}'}), - + # 36 ({ 'GEN_ENS_PROD_CLIMO_MEAN_FILE_NAME': '/some/climo_mean/file.txt', 'GEN_ENS_PROD_CLIMO_MEAN_FIELD': '{name="CLM_NAME"; level="(0,0,*,*)";}', @@ -274,46 +229,43 @@ def set_minimum_config_settings(config): 'vld_thresh = 0.5;shape = SQUARE;}' 'time_interp_method = NEAREST;' 'match_month = TRUE;day_interval = 30;' - 'hour_interval = 12;}'), - 'CLIMO_MEAN_FILE': '"/some/climo_mean/file.txt"'}), - - # climo stdev + 'hour_interval = 12;}')}), + # 37 climo stdev ({'GEN_ENS_PROD_CLIMO_STDEV_FILE_NAME': '/some/climo_stdev/file.txt', }, {'METPLUS_CLIMO_STDEV_DICT': ('climo_stdev = {file_name = ' - '["/some/climo_stdev/file.txt"];}'), - 'CLIMO_STDEV_FILE': '"/some/climo_stdev/file.txt"'}), - + '["/some/climo_stdev/file.txt"];}')}), + # 38 ({'GEN_ENS_PROD_CLIMO_STDEV_FIELD': '{name="CLM_NAME"; level="(0,0,*,*)";}', }, {'METPLUS_CLIMO_STDEV_DICT': 'climo_stdev = {field = [{name="CLM_NAME"; level="(0,0,*,*)";}];}'}), - + # 39 ({'GEN_ENS_PROD_CLIMO_STDEV_REGRID_METHOD': 'NEAREST', }, { 'METPLUS_CLIMO_STDEV_DICT': 'climo_stdev = {regrid = {method = NEAREST;}}'}), - + # 40 ({'GEN_ENS_PROD_CLIMO_STDEV_REGRID_WIDTH': '1', }, {'METPLUS_CLIMO_STDEV_DICT': 'climo_stdev = {regrid = {width = 1;}}'}), - + # 41 ({'GEN_ENS_PROD_CLIMO_STDEV_REGRID_VLD_THRESH': '0.5', }, { 'METPLUS_CLIMO_STDEV_DICT': 'climo_stdev = {regrid = {vld_thresh = 0.5;}}'}), - + # 42 ({'GEN_ENS_PROD_CLIMO_STDEV_REGRID_SHAPE': 'SQUARE', }, { 'METPLUS_CLIMO_STDEV_DICT': 'climo_stdev = {regrid = {shape = SQUARE;}}'}), - + # 43 ({'GEN_ENS_PROD_CLIMO_STDEV_TIME_INTERP_METHOD': 'NEAREST', }, { 'METPLUS_CLIMO_STDEV_DICT': 'climo_stdev = {time_interp_method = NEAREST;}'}), - + # 44 ({'GEN_ENS_PROD_CLIMO_STDEV_MATCH_MONTH': 'True', }, {'METPLUS_CLIMO_STDEV_DICT': 'climo_stdev = {match_month = TRUE;}'}), - + # 45 ({'GEN_ENS_PROD_CLIMO_STDEV_DAY_INTERVAL': '30', }, {'METPLUS_CLIMO_STDEV_DICT': 'climo_stdev = {day_interval = 30;}'}), - + # 46 ({'GEN_ENS_PROD_CLIMO_STDEV_HOUR_INTERVAL': '12', }, {'METPLUS_CLIMO_STDEV_DICT': 'climo_stdev = {hour_interval = 12;}'}), - + # 47 ({ 'GEN_ENS_PROD_CLIMO_STDEV_FILE_NAME': '/some/climo_stdev/file.txt', 'GEN_ENS_PROD_CLIMO_STDEV_FIELD': '{name="CLM_NAME"; level="(0,0,*,*)";}', @@ -333,17 +285,17 @@ def set_minimum_config_settings(config): 'vld_thresh = 0.5;shape = SQUARE;}' 'time_interp_method = NEAREST;' 'match_month = TRUE;day_interval = 30;' - 'hour_interval = 12;}'), - 'CLIMO_STDEV_FILE': '"/some/climo_stdev/file.txt"'}), + 'hour_interval = 12;}')}), + # 48 ({'GEN_ENS_PROD_NBRHD_PROB_WIDTH': '5', }, {'METPLUS_NBRHD_PROB_DICT': 'nbrhd_prob = {width = [5];}'}), - + # 49 ({'GEN_ENS_PROD_NBRHD_PROB_SHAPE': 'circle', }, {'METPLUS_NBRHD_PROB_DICT': 'nbrhd_prob = {shape = CIRCLE;}'}), - + # 50 ({'GEN_ENS_PROD_NBRHD_PROB_VLD_THRESH': '0.0', }, {'METPLUS_NBRHD_PROB_DICT': 'nbrhd_prob = {vld_thresh = 0.0;}'}), - + # 51 ({ 'GEN_ENS_PROD_NBRHD_PROB_WIDTH': '5', 'GEN_ENS_PROD_NBRHD_PROB_SHAPE': 'CIRCLE', @@ -355,25 +307,26 @@ def set_minimum_config_settings(config): 'vld_thresh = 0.0;}' ) }), + # 52 ({'GEN_ENS_PROD_NMEP_SMOOTH_VLD_THRESH': '0.0', }, {'METPLUS_NMEP_SMOOTH_DICT': 'nmep_smooth = {vld_thresh = 0.0;}'}), - + # 53 ({'GEN_ENS_PROD_NMEP_SMOOTH_SHAPE': 'circle', }, {'METPLUS_NMEP_SMOOTH_DICT': 'nmep_smooth = {shape = CIRCLE;}'}), - + # 54 ({'GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_DX': '81.27', }, {'METPLUS_NMEP_SMOOTH_DICT': 'nmep_smooth = {gaussian_dx = 81.27;}'}), - + # 55 ({'GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_RADIUS': '120', }, { 'METPLUS_NMEP_SMOOTH_DICT': 'nmep_smooth = {gaussian_radius = 120;}'}), - + # 56 ({'GEN_ENS_PROD_NMEP_SMOOTH_TYPE_METHOD': 'GAUSSIAN', }, {'METPLUS_NMEP_SMOOTH_DICT': 'nmep_smooth = {type = [{method = GAUSSIAN;}];}'}), - + # 57 ({'GEN_ENS_PROD_NMEP_SMOOTH_TYPE_WIDTH': '1', }, {'METPLUS_NMEP_SMOOTH_DICT': 'nmep_smooth = {type = [{width = 1;}];}'}), - + # 58 ({ 'GEN_ENS_PROD_NMEP_SMOOTH_VLD_THRESH': '0.0', 'GEN_ENS_PROD_NMEP_SMOOTH_SHAPE': 'circle', @@ -442,7 +395,11 @@ def test_gen_ens_prod_single_field(metplus_config, config_overrides, assert(cmd == expected_cmd) # check that environment variables were set properly - for env_var_key in wrapper.WRAPPER_ENV_VAR_KEYS: + # including deprecated env vars (not in wrapper env var keys) + env_var_keys = (wrapper.WRAPPER_ENV_VAR_KEYS + + [name for name in env_var_values + if name not in wrapper.WRAPPER_ENV_VAR_KEYS]) + for env_var_key in env_var_keys: match = next((item for item in env_vars if item.startswith(env_var_key)), None) assert(match is not None) diff --git a/internal_tests/pytests/grid_stat/test_grid_stat_wrapper.py b/internal_tests/pytests/grid_stat/test_grid_stat_wrapper.py index 3fd1e83f5e..b38bee453a 100644 --- a/internal_tests/pytests/grid_stat/test_grid_stat_wrapper.py +++ b/internal_tests/pytests/grid_stat/test_grid_stat_wrapper.py @@ -129,7 +129,8 @@ def test_handle_climo_file_variables(metplus_config, config_overrides, ({'GRID_STAT_REGRID_TO_GRID': 'FCST', }, - {'METPLUS_REGRID_DICT': 'regrid = {to_grid = FCST;}'}), + {'METPLUS_REGRID_DICT': 'regrid = {to_grid = FCST;}', + 'REGRID_TO_GRID': 'FCST'}), ({'GRID_STAT_REGRID_METHOD': 'NEAREST', }, @@ -155,7 +156,8 @@ def test_handle_climo_file_variables(metplus_config, config_overrides, }, {'METPLUS_REGRID_DICT': ('regrid = {to_grid = FCST;method = NEAREST;' 'width = 1;vld_thresh = 0.5;shape = SQUARE;}' - )}), + ), + 'REGRID_TO_GRID': 'FCST'}), ({'GRID_STAT_CLIMO_MEAN_INPUT_TEMPLATE': '/some/path/climo/filename.nc', @@ -605,7 +607,11 @@ def test_grid_stat_single_field(metplus_config, config_overrides, assert(cmd == expected_cmd) # check that environment variables were set properly - for env_var_key in wrapper.WRAPPER_ENV_VAR_KEYS: + # including deprecated env vars (not in wrapper env var keys) + env_var_keys = (wrapper.WRAPPER_ENV_VAR_KEYS + + [name for name in env_var_values + if name not in wrapper.WRAPPER_ENV_VAR_KEYS]) + for env_var_key in env_var_keys: match = next((item for item in env_vars if item.startswith(env_var_key)), None) assert(match is not None) diff --git a/internal_tests/pytests/ioda2nc/test_ioda2nc_wrapper.py b/internal_tests/pytests/ioda2nc/test_ioda2nc_wrapper.py index 47030515d3..d07460055e 100644 --- a/internal_tests/pytests/ioda2nc/test_ioda2nc_wrapper.py +++ b/internal_tests/pytests/ioda2nc/test_ioda2nc_wrapper.py @@ -221,7 +221,11 @@ def test_ioda2nc_wrapper(metplus_config, config_overrides, assert cmd == expected_cmd # check that environment variables were set properly - for env_var_key in wrapper.WRAPPER_ENV_VAR_KEYS: + # including deprecated env vars (not in wrapper env var keys) + env_var_keys = (wrapper.WRAPPER_ENV_VAR_KEYS + + [name for name in env_var_values + if name not in wrapper.WRAPPER_ENV_VAR_KEYS]) + for env_var_key in env_var_keys: match = next((item for item in env_vars if item.startswith(env_var_key)), None) assert match is not None diff --git a/internal_tests/pytests/met_config/test_met_config.py b/internal_tests/pytests/met_config/test_met_config.py new file mode 100644 index 0000000000..1e2e5e5f31 --- /dev/null +++ b/internal_tests/pytests/met_config/test_met_config.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 + +import pytest + +from metplus.util.met_config import * + +@pytest.mark.parametrize( + 'name, data_type, mp_configs, extra_args', [ + ('beg', 'int', 'BEG', None), + ('end', 'int', ['END'], None), + ] +) +def test_met_config_info(name, data_type, mp_configs, extra_args): + item = METConfig(name=name, data_type=data_type) + + item.metplus_configs = mp_configs + item.extra_args = extra_args + + assert(item.name == name) + assert(item.data_type == data_type) + if isinstance(mp_configs, list): + assert(item.metplus_configs == mp_configs) + else: + assert(item.metplus_configs == [mp_configs]) + + if not extra_args: + assert(item.extra_args == {}) + +@pytest.mark.parametrize( + 'data_type, expected_function', [ + ('int', 'set_met_config_int'), + ('float', 'set_met_config_float'), + ('list', 'set_met_config_list'), + ('string', 'set_met_config_string'), + ('thresh', 'set_met_config_thresh'), + ('bool', 'set_met_config_bool'), + ('bad_name', None), + ] +) +def test_set_met_config_function(data_type, expected_function): + try: + function_found = set_met_config_function(data_type) + function_name = function_found.__name__ if function_found else None + assert(function_name == expected_function) + except ValueError: + assert expected_function is None + + +@pytest.mark.parametrize( + 'input, output', [ + ('', 'NONE'), + ('NONE', 'NONE'), + ('FCST', 'FCST'), + ('OBS', 'OBS'), + ('G002', '"G002"'), + ] +) +def test_format_regrid_to_grid(input, output): + assert format_regrid_to_grid(input) == output diff --git a/internal_tests/pytests/met_dictionary_info/test_met_dictionary_info.py b/internal_tests/pytests/met_dictionary_info/test_met_dictionary_info.py deleted file mode 100644 index 4bf2b7b19c..0000000000 --- a/internal_tests/pytests/met_dictionary_info/test_met_dictionary_info.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python3 - -import pytest - -from metplus.util import METConfigInfo - -@pytest.mark.parametrize( - 'name, data_type, mp_configs, extra_args', [ - ('beg', 'int', 'BEG', None), - ('end', 'int', ['END'], None), - ] -) -def test_met_config_info(name, data_type, mp_configs, extra_args): - item = METConfigInfo(name=name, - data_type=data_type, - ) - item.metplus_configs = mp_configs - item.extra_args = extra_args - - assert(item.name == name) - assert(item.data_type == data_type) - if isinstance(mp_configs, list): - assert(item.metplus_configs == mp_configs) - else: - assert(item.metplus_configs == [mp_configs]) - - if not extra_args: - assert(item.extra_args == {}) - diff --git a/internal_tests/pytests/met_util/test_met_util.py b/internal_tests/pytests/met_util/test_met_util.py index 0154ff68e6..978c2a4592 100644 --- a/internal_tests/pytests/met_util/test_met_util.py +++ b/internal_tests/pytests/met_util/test_met_util.py @@ -10,491 +10,7 @@ from metplus.util import met_util as util from metplus.util import time_util - -@pytest.mark.parametrize( - 'regex,index,id,expected_result', [ - # 0: No ID - (r'^FCST_VAR(\d+)_NAME$', 1, None, - {'1': [None], - '2': [None], - '4': [None]}), - # 1: ID and index 2 - (r'(\w+)_VAR(\d+)_NAME', 2, 1, - {'1': ['FCST'], - '2': ['FCST'], - '4': ['FCST']}), - # 2: index 1, ID 2, multiple identifiers - (r'^FCST_VAR(\d+)_(\w+)$', 1, 2, - {'1': ['NAME', 'LEVELS'], - '2': ['NAME'], - '4': ['NAME']}), - # 3: command that StatAnalysis wrapper uses - (r'MODEL(\d+)$', 1, None, - {'1': [None], - '2': [None],}), - # 4: TCPairs conensus logic - (r'^TC_PAIRS_CONSENSUS(\d+)_(\w+)$', 1, 2, - {'1': ['NAME', 'MEMBERS', 'REQUIRED', 'MIN_REQ'], - '2': ['NAME', 'MEMBERS', 'REQUIRED', 'MIN_REQ']}), - ] -) -def test_find_indices_in_config_section(metplus_config, regex, index, - id, expected_result): - config = metplus_config() - config.set('config', 'FCST_VAR1_NAME', 'name1') - config.set('config', 'FCST_VAR1_LEVELS', 'level1') - config.set('config', 'FCST_VAR2_NAME', 'name2') - config.set('config', 'FCST_VAR4_NAME', 'name4') - config.set('config', 'MODEL1', 'model1') - config.set('config', 'MODEL2', 'model2') - - config.set('config', 'TC_PAIRS_CONSENSUS1_NAME', 'name1') - config.set('config', 'TC_PAIRS_CONSENSUS1_MEMBERS', 'member1') - config.set('config', 'TC_PAIRS_CONSENSUS1_REQUIRED', 'True') - config.set('config', 'TC_PAIRS_CONSENSUS1_MIN_REQ', '1') - config.set('config', 'TC_PAIRS_CONSENSUS2_NAME', 'name2') - config.set('config', 'TC_PAIRS_CONSENSUS2_MEMBERS', 'member2') - config.set('config', 'TC_PAIRS_CONSENSUS2_REQUIRED', 'True') - config.set('config', 'TC_PAIRS_CONSENSUS2_MIN_REQ', '2') - - - indices = util.find_indices_in_config_section(regex, config, - index_index=index, - id_index=id) - - pp = pprint.PrettyPrinter() - print(f'Indices:') - pp.pprint(indices) - - assert indices == expected_result - -@pytest.mark.parametrize( - 'data_type, met_tool, expected_out', [ - ('FCST', None, ['FCST_', - 'BOTH_',]), - ('OBS', None, ['OBS_', - 'BOTH_',]), - ('FCST', 'grid_stat', ['FCST_GRID_STAT_', - 'BOTH_GRID_STAT_', - 'FCST_', - 'BOTH_', - ]), - ('OBS', 'extract_tiles', ['OBS_EXTRACT_TILES_', - 'BOTH_EXTRACT_TILES_', - 'OBS_', - 'BOTH_', - ]), - ('ENS', None, ['ENS_']), - ('DATA', None, ['DATA_']), - ('DATA', 'tc_gen', ['DATA_TC_GEN_', - 'DATA_']), - - ] -) -def test_get_field_search_prefixes(data_type, met_tool, expected_out): - assert(util.get_field_search_prefixes(data_type, - met_tool) == expected_out) - -# search prefixes are valid prefixes to append to field info variables -# config_overrides are a dict of config vars and their values -# search_key is the key of the field config item to check -# expected_value is the variable that search_key is set to -@pytest.mark.parametrize( - 'search_prefixes, config_overrides, expected_value', [ - (['BOTH_', 'FCST_'], - {'FCST_VAR1_': 'fcst_var1'}, - 'fcst_var1' - ), - (['BOTH_', 'FCST_'], {}, None), - - (['BOTH_', 'FCST_'], - {'FCST_VAR1_': 'fcst_var1', - 'BOTH_VAR1_': 'both_var1'}, - 'both_var1' - ), - - (['BOTH_GRID_STAT_', 'FCST_GRID_STAT_'], - {'FCST_GRID_STAT_VAR1_': 'fcst_grid_stat_var1'}, - 'fcst_grid_stat_var1' - ), - (['BOTH_GRID_STAT_', 'FCST_GRID_STAT_'], {}, None), - (['BOTH_GRID_STAT_', 'FCST_GRID_STAT_'], - {'FCST_GRID_STAT_VAR1_': 'fcst_grid_stat_var1', - 'BOTH_GRID_STAT_VAR1_': 'both_grid_stat_var1'}, - 'both_grid_stat_var1' - ), - - (['ENS_'], - {'ENS_VAR1_': 'env_var1'}, - 'env_var1' - ), - (['ENS_'], {}, None), - - ] -) -def test_get_field_config_variables(metplus_config, - search_prefixes, - config_overrides, - expected_value): - config = metplus_config() - index = '1' - field_info_types = ['name', 'levels', 'thresh', 'options', 'output_names'] - for field_info_type in field_info_types: - for key, value in config_overrides.items(): - config.set('config', - f'{key}{field_info_type.upper()}', - value) - - field_configs = util.get_field_config_variables(config, - index, - search_prefixes) - - assert(field_configs.get(field_info_type) == expected_value) - -@pytest.mark.parametrize( - 'config_keys, field_key, expected_value', [ - (['NAME', - ], - 'name', 'NAME' - ), - (['NAME', - 'INPUT_FIELD_NAME', - ], - 'name', 'NAME' - ), - (['INPUT_FIELD_NAME', - ], - 'name', 'INPUT_FIELD_NAME' - ), - ([], 'name', None), - (['LEVELS', - ], - 'levels', 'LEVELS' - ), - (['LEVELS', - 'FIELD_LEVEL', - ], - 'levels', 'LEVELS' - ), - (['FIELD_LEVEL', - ], - 'levels', 'FIELD_LEVEL' - ), - ([], 'levels', None), - (['OUTPUT_NAMES', - ], - 'output_names', 'OUTPUT_NAMES' - ), - (['OUTPUT_NAMES', - 'OUTPUT_FIELD_NAME', - ], - 'output_names', 'OUTPUT_NAMES' - ), - (['OUTPUT_FIELD_NAME', - ], - 'output_names', 'OUTPUT_FIELD_NAME' - ), - ([], 'output_names', None), - ] -) -def test_get_field_config_variables_synonyms(metplus_config, - config_keys, - field_key, - expected_value): - config = metplus_config() - index = '1' - prefix = 'BOTH_REGRID_DATA_PLANE_' - for key in config_keys: - config.set('config', f'{prefix}VAR{index}_{key}', key) - - field_configs = util.get_field_config_variables(config, - index, - [prefix]) - - assert(field_configs.get(field_key) == expected_value) - -# ensure that the field configuration used for -# met_tool_wrapper/EnsembleStat/EnsembleStat.conf -# works as expected -def test_parse_var_list_ensemble(metplus_config): - config = metplus_config() - config.set('config', 'ENS_VAR1_NAME', 'APCP') - config.set('config', 'ENS_VAR1_LEVELS', 'A24') - config.set('config', 'ENS_VAR1_THRESH', '>0.0, >=10.0') - config.set('config', 'ENS_VAR2_NAME', 'REFC') - config.set('config', 'ENS_VAR2_LEVELS', 'L0') - config.set('config', 'ENS_VAR2_THRESH', '>35.0') - config.set('config', 'ENS_VAR2_OPTIONS', 'GRIB1_ptv = 129;') - config.set('config', 'ENS_VAR3_NAME', 'UGRD') - config.set('config', 'ENS_VAR3_LEVELS', 'Z10') - config.set('config', 'ENS_VAR3_THRESH', '>=5.0') - config.set('config', 'ENS_VAR4_NAME', 'VGRD') - config.set('config', 'ENS_VAR4_LEVELS', 'Z10') - config.set('config', 'ENS_VAR4_THRESH', '>=5.0') - config.set('config', 'ENS_VAR5_NAME', 'WIND') - config.set('config', 'ENS_VAR5_LEVELS', 'Z10') - config.set('config', 'ENS_VAR5_THRESH', '>=5.0') - config.set('config', 'FCST_VAR1_NAME', 'APCP') - config.set('config', 'FCST_VAR1_LEVELS', 'A24') - config.set('config', 'FCST_VAR1_THRESH', '>0.01, >=10.0') - config.set('config', 'FCST_VAR1_OPTIONS', ('ens_ssvar_bin_size = 0.1; ' - 'ens_phist_bin_size = 0.05;')) - config.set('config', 'OBS_VAR1_NAME', 'APCP') - config.set('config', 'OBS_VAR1_LEVELS', 'A24') - config.set('config', 'OBS_VAR1_THRESH', '>0.01, >=10.0') - config.set('config', 'OBS_VAR1_OPTIONS', ('ens_ssvar_bin_size = 0.1; ' - 'ens_phist_bin_size = 0.05;')) - time_info = {} - - expected_ens_list = [{'index': '1', - 'ens_name': 'APCP', - 'ens_level': 'A24', - 'ens_thresh': ['>0.0', '>=10.0']}, - {'index': '2', - 'ens_name': 'REFC', - 'ens_level': 'L0', - 'ens_thresh': ['>35.0']}, - {'index': '3', - 'ens_name': 'UGRD', - 'ens_level': 'Z10', - 'ens_thresh': ['>=5.0']}, - {'index': '4', - 'ens_name': 'VGRD', - 'ens_level': 'Z10', - 'ens_thresh': ['>=5.0']}, - {'index': '5', - 'ens_name': 'WIND', - 'ens_level': 'Z10', - 'ens_thresh': ['>=5.0']}, - ] - expected_var_list = [{'index': '1', - 'fcst_name': 'APCP', - 'fcst_level': 'A24', - 'fcst_thresh': ['>0.01', '>=10.0'], - 'fcst_extra': ('ens_ssvar_bin_size = 0.1; ' - 'ens_phist_bin_size = 0.05;'), - 'obs_name': 'APCP', - 'obs_level': 'A24', - 'obs_thresh': ['>0.01', '>=10.0'], - 'obs_extra': ('ens_ssvar_bin_size = 0.1; ' - 'ens_phist_bin_size = 0.05;') - - }, - ] - - ensemble_var_list = util.parse_var_list(config, time_info, - data_type='ENS') - - # parse optional var list for FCST and/or OBS fields - var_list = util.parse_var_list(config, time_info, - met_tool='ensemble_stat') - - pp = pprint.PrettyPrinter() - print(f'ENSEMBLE_VAR_LIST:') - pp.pprint(ensemble_var_list) - print(f'VAR_LIST:') - pp.pprint(var_list) - - assert(len(ensemble_var_list) == len(expected_ens_list)) - for actual_ens, expected_ens in zip(ensemble_var_list, expected_ens_list): - for key, value in expected_ens.items(): - assert(actual_ens.get(key) == value) - - assert(len(var_list) == len(expected_var_list)) - for actual_var, expected_var in zip(var_list, expected_var_list): - for key, value in expected_var.items(): - assert(actual_var.get(key) == value) - -def test_parse_var_list_series_by(metplus_config): - config = metplus_config() - config.set('config', 'BOTH_EXTRACT_TILES_VAR1_NAME', 'RH') - config.set('config', 'BOTH_EXTRACT_TILES_VAR1_LEVELS', 'P850, P700') - config.set('config', 'BOTH_EXTRACT_TILES_VAR1_OUTPUT_NAMES', - 'RH_850mb, RH_700mb') - - config.set('config', 'BOTH_SERIES_ANALYSIS_VAR1_NAME', 'RH_850mb') - config.set('config', 'BOTH_SERIES_ANALYSIS_VAR1_LEVELS', 'P850') - config.set('config', 'BOTH_SERIES_ANALYSIS_VAR2_NAME', 'RH_700mb') - config.set('config', 'BOTH_SERIES_ANALYSIS_VAR2_LEVELS', 'P700') - time_info = {} - - expected_et_list = [{'index': '1', - 'fcst_name': 'RH', - 'fcst_level': 'P850', - 'fcst_output_name': 'RH_850mb', - 'obs_name': 'RH', - 'obs_level': 'P850', - 'obs_output_name': 'RH_850mb', - }, - {'index': '1', - 'fcst_name': 'RH', - 'fcst_level': 'P700', - 'fcst_output_name': 'RH_700mb', - 'obs_name': 'RH', - 'obs_level': 'P700', - 'obs_output_name': 'RH_700mb', - }, - ] - expected_sa_list = [{'index': '1', - 'fcst_name': 'RH_850mb', - 'fcst_level': 'P850', - 'obs_name': 'RH_850mb', - 'obs_level': 'P850', - }, - {'index': '2', - 'fcst_name': 'RH_700mb', - 'fcst_level': 'P700', - 'obs_name': 'RH_700mb', - 'obs_level': 'P700', - }, - ] - - actual_et_list = util.parse_var_list(config, - time_info=time_info, - met_tool='extract_tiles') - - actual_sa_list = util.parse_var_list(config, - met_tool='series_analysis') - - pp = pprint.PrettyPrinter() - print(f'ExtractTiles var list:') - pp.pprint(actual_et_list) - print(f'SeriesAnalysis var list:') - pp.pprint(actual_sa_list) - - assert(len(actual_et_list) == len(expected_et_list)) - for actual_et, expected_et in zip(actual_et_list, expected_et_list): - for key, value in expected_et.items(): - assert(actual_et.get(key) == value) - - assert(len(actual_sa_list) == len(expected_sa_list)) - for actual_sa, expected_sa in zip(actual_sa_list, expected_sa_list): - for key, value in expected_sa.items(): - assert(actual_sa.get(key) == value) - -@pytest.mark.parametrize( - 'input_dict, expected_list', [ - ({'init': datetime.datetime(2019, 2, 1, 6), - 'lead': 7200, }, - [ - {'index': '1', - 'fcst_name': 'FNAME_2019', - 'fcst_level': 'Z06', - 'obs_name': 'ONAME_2019', - 'obs_level': 'L06', - }, - {'index': '1', - 'fcst_name': 'FNAME_2019', - 'fcst_level': 'Z08', - 'obs_name': 'ONAME_2019', - 'obs_level': 'L08', - }, - ]), - ({'init': datetime.datetime(2021, 4, 13, 9), - 'lead': 10800, }, - [ - {'index': '1', - 'fcst_name': 'FNAME_2021', - 'fcst_level': 'Z09', - 'obs_name': 'ONAME_2021', - 'obs_level': 'L09', - }, - {'index': '1', - 'fcst_name': 'FNAME_2021', - 'fcst_level': 'Z12', - 'obs_name': 'ONAME_2021', - 'obs_level': 'L12', - }, - ]), - ] -) -def test_sub_var_list(metplus_config, input_dict, expected_list): - config = metplus_config() - config.set('config', 'FCST_VAR1_NAME', 'FNAME_{init?fmt=%Y}') - config.set('config', 'FCST_VAR1_LEVELS', 'Z{init?fmt=%H}, Z{valid?fmt=%H}') - config.set('config', 'OBS_VAR1_NAME', 'ONAME_{init?fmt=%Y}') - config.set('config', 'OBS_VAR1_LEVELS', 'L{init?fmt=%H}, L{valid?fmt=%H}') - - time_info = time_util.ti_calculate(input_dict) - - actual_temp = util.parse_var_list(config) - - pp = pprint.PrettyPrinter() - print(f'Actual var list (before sub):') - pp.pprint(actual_temp) - - actual_list = util.sub_var_list(actual_temp, time_info) - print(f'Actual var list (after sub):') - pp.pprint(actual_list) - - assert(len(actual_list) == len(expected_list)) - for actual, expected in zip(actual_list, expected_list): - for key, value in expected.items(): - assert(actual.get(key) == value) - -@pytest.mark.parametrize( - 'config_var_name, expected_indices, set_met_tool', [ - ('FCST_GRID_STAT_VAR1_NAME', ['1'], True), - ('FCST_GRID_STAT_VAR2_INPUT_FIELD_NAME', ['2'], True), - ('FCST_GRID_STAT_VAR3_FIELD_NAME', ['3'], True), - ('BOTH_GRID_STAT_VAR4_NAME', ['4'], True), - ('BOTH_GRID_STAT_VAR5_INPUT_FIELD_NAME', ['5'], True), - ('BOTH_GRID_STAT_VAR6_FIELD_NAME', ['6'], True), - ('FCST_VAR7_NAME', ['7'], False), - ('FCST_VAR8_INPUT_FIELD_NAME', ['8'], False), - ('FCST_VAR9_FIELD_NAME', ['9'], False), - ('BOTH_VAR10_NAME', ['10'], False), - ('BOTH_VAR11_INPUT_FIELD_NAME', ['11'], False), - ('BOTH_VAR12_FIELD_NAME', ['12'], False), - ] -) -def test_find_var_indices_fcst(metplus_config, - config_var_name, - expected_indices, - set_met_tool): - config = metplus_config() - data_types = ['FCST'] - config.set('config', config_var_name, "NAME1") - met_tool = 'grid_stat' if set_met_tool else None - var_name_indices = util.find_var_name_indices(config, - data_types=data_types, - met_tool=met_tool) - - assert(len(var_name_indices) == len(expected_indices)) - for actual_index in var_name_indices: - assert(actual_index in expected_indices) - -def test_parse_var_list_priority_fcst(metplus_config): - priority_list = ['FCST_GRID_STAT_VAR1_NAME', - 'FCST_GRID_STAT_VAR1_INPUT_FIELD_NAME', - 'FCST_GRID_STAT_VAR1_FIELD_NAME', - 'BOTH_GRID_STAT_VAR1_NAME', - 'BOTH_GRID_STAT_VAR1_INPUT_FIELD_NAME', - 'BOTH_GRID_STAT_VAR1_FIELD_NAME', - 'FCST_VAR1_NAME', - 'FCST_VAR1_INPUT_FIELD_NAME', - 'FCST_VAR1_FIELD_NAME', - 'BOTH_VAR1_NAME', - 'BOTH_VAR1_INPUT_FIELD_NAME', - 'BOTH_VAR1_FIELD_NAME', - ] - time_info = {} - - # loop through priority list, process, then pop first value off and - # process again until all items have been popped. - # This will check that list is in priority order - while(priority_list): - config = metplus_config() - for key in priority_list: - config.set('config', key, key.lower()) - - var_list = util.parse_var_list(config, time_info=time_info, - data_type='FCST', - met_tool='grid_stat') - - assert(len(var_list) == 1) - assert(var_list[0].get('fcst_name') == priority_list[0].lower()) - priority_list.pop(0) +from metplus.util.config_metplus import parse_var_list @pytest.mark.parametrize( 'before, after', [ @@ -662,244 +178,6 @@ def test_getlist_empty(): test_list = util.getlist(l) assert(test_list == []) -# field info only defined in the FCST_* variables -@pytest.mark.parametrize( - 'data_type, list_created', [ - (None, False), - ('FCST', True), - ('OBS', False), - ] -) -def test_parse_var_list_fcst_only(metplus_config, data_type, list_created): - conf = metplus_config() - conf.set('config', 'FCST_VAR1_NAME', "NAME1") - conf.set('config', 'FCST_VAR1_LEVELS', "LEVELS11, LEVELS12") - conf.set('config', 'FCST_VAR2_NAME', "NAME2") - conf.set('config', 'FCST_VAR2_LEVELS', "LEVELS21, LEVELS22") - - # this should not occur because OBS variables are missing - if util.validate_configuration_variables(conf, force_check=True)[1]: - assert(False) - - var_list = util.parse_var_list(conf, time_info=None, data_type=data_type) - - # list will be created if requesting just OBS, but it should not be created if - # nothing was requested because FCST values are missing - if list_created: - assert(var_list[0]['fcst_name'] == "NAME1" and \ - var_list[1]['fcst_name'] == "NAME1" and \ - var_list[2]['fcst_name'] == "NAME2" and \ - var_list[3]['fcst_name'] == "NAME2" and \ - var_list[0]['fcst_level'] == "LEVELS11" and \ - var_list[1]['fcst_level'] == "LEVELS12" and \ - var_list[2]['fcst_level'] == "LEVELS21" and \ - var_list[3]['fcst_level'] == "LEVELS22") - else: - assert(not var_list) - -# field info only defined in the OBS_* variables -@pytest.mark.parametrize( - 'data_type, list_created', [ - (None, False), - ('OBS', True), - ('FCST', False), - ] -) -def test_parse_var_list_obs(metplus_config, data_type, list_created): - conf = metplus_config() - conf.set('config', 'OBS_VAR1_NAME', "NAME1") - conf.set('config', 'OBS_VAR1_LEVELS', "LEVELS11, LEVELS12") - conf.set('config', 'OBS_VAR2_NAME', "NAME2") - conf.set('config', 'OBS_VAR2_LEVELS', "LEVELS21, LEVELS22") - - # this should not occur because FCST variables are missing - if util.validate_configuration_variables(conf, force_check=True)[1]: - assert(False) - - var_list = util.parse_var_list(conf, time_info=None, data_type=data_type) - - # list will be created if requesting just OBS, but it should not be created if - # nothing was requested because FCST values are missing - if list_created: - assert(var_list[0]['obs_name'] == "NAME1" and \ - var_list[1]['obs_name'] == "NAME1" and \ - var_list[2]['obs_name'] == "NAME2" and \ - var_list[3]['obs_name'] == "NAME2" and \ - var_list[0]['obs_level'] == "LEVELS11" and \ - var_list[1]['obs_level'] == "LEVELS12" and \ - var_list[2]['obs_level'] == "LEVELS21" and \ - var_list[3]['obs_level'] == "LEVELS22") - else: - assert(not var_list) - - -# field info only defined in the BOTH_* variables -@pytest.mark.parametrize( - 'data_type, list_created', [ - (None, 'fcst:obs'), - ('FCST', 'fcst'), - ('OBS', 'obs'), - ] -) -def test_parse_var_list_both(metplus_config, data_type, list_created): - conf = metplus_config() - conf.set('config', 'BOTH_VAR1_NAME', "NAME1") - conf.set('config', 'BOTH_VAR1_LEVELS', "LEVELS11, LEVELS12") - conf.set('config', 'BOTH_VAR2_NAME', "NAME2") - conf.set('config', 'BOTH_VAR2_LEVELS', "LEVELS21, LEVELS22") - - # this should not occur because BOTH variables are used - if not util.validate_configuration_variables(conf, force_check=True)[1]: - assert(False) - - var_list = util.parse_var_list(conf, time_info=None, data_type=data_type) - print(f'var_list:{var_list}') - for list_to_check in list_created.split(':'): - if not var_list[0][f'{list_to_check}_name'] == "NAME1" or \ - not var_list[1][f'{list_to_check}_name'] == "NAME1" or \ - not var_list[2][f'{list_to_check}_name'] == "NAME2" or \ - not var_list[3][f'{list_to_check}_name'] == "NAME2" or \ - not var_list[0][f'{list_to_check}_level'] == "LEVELS11" or \ - not var_list[1][f'{list_to_check}_level'] == "LEVELS12" or \ - not var_list[2][f'{list_to_check}_level'] == "LEVELS21" or \ - not var_list[3][f'{list_to_check}_level'] == "LEVELS22": - assert(False) - -# field info defined in both FCST_* and OBS_* variables -def test_parse_var_list_fcst_and_obs(metplus_config): - conf = metplus_config() - conf.set('config', 'FCST_VAR1_NAME', "FNAME1") - conf.set('config', 'FCST_VAR1_LEVELS', "FLEVELS11, FLEVELS12") - conf.set('config', 'FCST_VAR2_NAME', "FNAME2") - conf.set('config', 'FCST_VAR2_LEVELS', "FLEVELS21, FLEVELS22") - conf.set('config', 'OBS_VAR1_NAME', "ONAME1") - conf.set('config', 'OBS_VAR1_LEVELS', "OLEVELS11, OLEVELS12") - conf.set('config', 'OBS_VAR2_NAME', "ONAME2") - conf.set('config', 'OBS_VAR2_LEVELS', "OLEVELS21, OLEVELS22") - - # this should not occur because FCST and OBS variables are found - if not util.validate_configuration_variables(conf, force_check=True)[1]: - assert(False) - - var_list = util.parse_var_list(conf) - - assert(var_list[0]['fcst_name'] == "FNAME1" and \ - var_list[0]['obs_name'] == "ONAME1" and \ - var_list[1]['fcst_name'] == "FNAME1" and \ - var_list[1]['obs_name'] == "ONAME1" and \ - var_list[2]['fcst_name'] == "FNAME2" and \ - var_list[2]['obs_name'] == "ONAME2" and \ - var_list[3]['fcst_name'] == "FNAME2" and \ - var_list[3]['obs_name'] == "ONAME2" and \ - var_list[0]['fcst_level'] == "FLEVELS11" and \ - var_list[0]['obs_level'] == "OLEVELS11" and \ - var_list[1]['fcst_level'] == "FLEVELS12" and \ - var_list[1]['obs_level'] == "OLEVELS12" and \ - var_list[2]['fcst_level'] == "FLEVELS21" and \ - var_list[2]['obs_level'] == "OLEVELS21" and \ - var_list[3]['fcst_level'] == "FLEVELS22" and \ - var_list[3]['obs_level'] == "OLEVELS22") - -# VAR1 defined by FCST, VAR2 defined by OBS -def test_parse_var_list_fcst_and_obs_alternate(metplus_config): - conf = metplus_config() - conf.set('config', 'FCST_VAR1_NAME', "FNAME1") - conf.set('config', 'FCST_VAR1_LEVELS', "FLEVELS11, FLEVELS12") - conf.set('config', 'OBS_VAR2_NAME', "ONAME2") - conf.set('config', 'OBS_VAR2_LEVELS', "OLEVELS21, OLEVELS22") - - # configuration is invalid and parse var list should not give any results - assert(not util.validate_configuration_variables(conf, force_check=True)[1] and not util.parse_var_list(conf)) - -# VAR1 defined by OBS, VAR2 by FCST, VAR3 by both FCST AND OBS -@pytest.mark.parametrize( - 'data_type, list_len, name_levels', [ - (None, 0, None), - ('FCST', 4, ('FNAME2:FLEVELS21','FNAME2:FLEVELS22','FNAME3:FLEVELS31','FNAME3:FLEVELS32')), - ('OBS', 4, ('ONAME1:OLEVELS11','ONAME1:OLEVELS12','ONAME3:OLEVELS31','ONAME3:OLEVELS32')), - ] -) -def test_parse_var_list_fcst_and_obs_and_both(metplus_config, data_type, list_len, name_levels): - conf = metplus_config() - conf.set('config', 'OBS_VAR1_NAME', "ONAME1") - conf.set('config', 'OBS_VAR1_LEVELS', "OLEVELS11, OLEVELS12") - conf.set('config', 'FCST_VAR2_NAME', "FNAME2") - conf.set('config', 'FCST_VAR2_LEVELS', "FLEVELS21, FLEVELS22") - conf.set('config', 'FCST_VAR3_NAME', "FNAME3") - conf.set('config', 'FCST_VAR3_LEVELS', "FLEVELS31, FLEVELS32") - conf.set('config', 'OBS_VAR3_NAME', "ONAME3") - conf.set('config', 'OBS_VAR3_LEVELS', "OLEVELS31, OLEVELS32") - - # configuration is invalid and parse var list should not give any results - if util.validate_configuration_variables(conf, force_check=True)[1]: - assert(False) - - var_list = util.parse_var_list(conf, time_info=None, data_type=data_type) - - if len(var_list) != list_len: - assert(False) - - if data_type is None: - assert(len(var_list) == 0) - - if name_levels is not None: - dt_lower = data_type.lower() - expected = [] - for name_level in name_levels: - name, level = name_level.split(':') - expected.append({f'{dt_lower}_name': name, - f'{dt_lower}_level': level}) - - for expect, reality in zip(expected,var_list): - if expect[f'{dt_lower}_name'] != reality[f'{dt_lower}_name']: - assert(False) - - if expect[f'{dt_lower}_level'] != reality[f'{dt_lower}_level']: - assert(False) - - assert(True) - -# option defined in obs only -@pytest.mark.parametrize( - 'data_type, list_len', [ - (None, 0), - ('FCST', 2), - ('OBS', 0), - ] -) -def test_parse_var_list_fcst_only_options(metplus_config, data_type, list_len): - conf = metplus_config() - conf.set('config', 'FCST_VAR1_NAME', "NAME1") - conf.set('config', 'FCST_VAR1_LEVELS', "LEVELS11, LEVELS12") - conf.set('config', 'FCST_VAR1_THRESH', ">1, >2") - conf.set('config', 'OBS_VAR1_OPTIONS', "OOPTIONS11") - - # this should not occur because OBS variables are missing - if util.validate_configuration_variables(conf, force_check=True)[1]: - assert(False) - - var_list = util.parse_var_list(conf, time_info=None, data_type=data_type) - - assert(len(var_list) == list_len) - -@pytest.mark.parametrize( - 'met_tool, indices', [ - (None, {'1': ['FCST']}), - ('GRID_STAT', {'2': ['FCST']}), - ('ENSEMBLE_STAT', {}), - ] -) -def test_find_var_indices_wrapper_specific(metplus_config, met_tool, indices): - conf = metplus_config() - data_type = 'FCST' - conf.set('config', f'{data_type}_VAR1_NAME', "NAME1") - conf.set('config', f'{data_type}_GRID_STAT_VAR2_NAME', "GSNAME2") - - var_name_indices = util.find_var_name_indices(conf, data_types=[data_type], - met_tool=met_tool) - - assert(var_name_indices == indices) - def test_get_lead_sequence_lead(metplus_config): input_dict = {'valid': datetime.datetime(2019, 2, 1, 13)} conf = metplus_config() @@ -1089,201 +367,6 @@ def test_get_lead_sequence_init_min_10(metplus_config): lead_seq = [12, 24] assert(test_seq == [relativedelta(hours=lead) for lead in lead_seq]) -@pytest.mark.parametrize( - 'item_list, extension, is_valid', [ - (['FCST'], 'NAME', False), - (['OBS'], 'NAME', False), - (['FCST', 'OBS'], 'NAME', True), - (['BOTH'], 'NAME', True), - (['FCST', 'OBS', 'BOTH'], 'NAME', False), - (['FCST', 'ENS'], 'NAME', False), - (['OBS', 'ENS'], 'NAME', False), - (['FCST', 'OBS', 'ENS'], 'NAME', True), - (['BOTH', 'ENS'], 'NAME', True), - (['FCST', 'OBS', 'BOTH', 'ENS'], 'NAME', False), - - (['FCST', 'OBS'], 'THRESH', True), - (['BOTH'], 'THRESH', True), - (['FCST', 'OBS', 'BOTH'], 'THRESH', False), - (['FCST', 'OBS', 'ENS'], 'THRESH', True), - (['BOTH', 'ENS'], 'THRESH', True), - (['FCST', 'OBS', 'BOTH', 'ENS'], 'THRESH', False), - - (['FCST'], 'OPTIONS', True), - (['OBS'], 'OPTIONS', True), - (['FCST', 'OBS'], 'OPTIONS', True), - (['BOTH'], 'OPTIONS', True), - (['FCST', 'OBS', 'BOTH'], 'OPTIONS', False), - (['FCST', 'ENS'], 'OPTIONS', True), - (['OBS', 'ENS'], 'OPTIONS', True), - (['FCST', 'OBS', 'ENS'], 'OPTIONS', True), - (['BOTH', 'ENS'], 'OPTIONS', True), - (['FCST', 'OBS', 'BOTH', 'ENS'], 'OPTIONS', False), - - (['FCST', 'OBS', 'BOTH'], 'LEVELS', False), - (['FCST', 'OBS'], 'LEVELS', True), - (['BOTH'], 'LEVELS', True), - (['FCST', 'OBS', 'ENS'], 'LEVELS', True), - (['BOTH', 'ENS'], 'LEVELS', True), - - ] -) -def test_is_var_item_valid(metplus_config, item_list, extension, is_valid): - conf = metplus_config() - assert(util.is_var_item_valid(item_list, '1', extension, conf)[0] == is_valid) - -@pytest.mark.parametrize( - 'item_list, configs_to_set, is_valid', [ - - (['FCST'], {'FCST_VAR1_LEVELS': 'A06', - 'OBS_VAR1_NAME': 'script_name.py something else'}, True), - (['FCST'], {'FCST_VAR1_LEVELS': 'A06', - 'OBS_VAR1_NAME': 'APCP'}, False), - (['OBS'], {'OBS_VAR1_LEVELS': '"(*,*)"', - 'FCST_VAR1_NAME': 'script_name.py something else'}, True), - (['OBS'], {'OBS_VAR1_LEVELS': '"(*,*)"', - 'FCST_VAR1_NAME': 'APCP'}, False), - - (['FCST', 'ENS'], {'FCST_VAR1_LEVELS': 'A06', - 'OBS_VAR1_NAME': 'script_name.py something else'}, True), - (['FCST', 'ENS'], {'FCST_VAR1_LEVELS': 'A06', - 'OBS_VAR1_NAME': 'APCP'}, False), - (['OBS', 'ENS'], {'OBS_VAR1_LEVELS': '"(*,*)"', - 'FCST_VAR1_NAME': 'script_name.py something else'}, True), - (['OBS', 'ENS'], {'OBS_VAR1_LEVELS': '"(*,*)"', - 'FCST_VAR1_NAME': 'APCP'}, False), - - (['FCST'], {'FCST_VAR1_LEVELS': 'A06, A12', - 'OBS_VAR1_NAME': 'script_name.py something else'}, False), - (['FCST'], {'FCST_VAR1_LEVELS': 'A06, A12', - 'OBS_VAR1_NAME': 'APCP'}, False), - (['OBS'], {'OBS_VAR1_LEVELS': '"(0,*,*)", "(1,*,*)"', - 'FCST_VAR1_NAME': 'script_name.py something else'}, False), - - (['FCST', 'ENS'], {'FCST_VAR1_LEVELS': 'A06, A12', - 'OBS_VAR1_NAME': 'script_name.py something else'}, False), - (['FCST', 'ENS'], {'FCST_VAR1_LEVELS': 'A06, A12', - 'OBS_VAR1_NAME': 'APCP'}, False), - (['OBS', 'ENS'], {'OBS_VAR1_LEVELS': '"(0,*,*)", "(1,*,*)"', - 'FCST_VAR1_NAME': 'script_name.py something else'}, False), - - ] -) -def test_is_var_item_valid_levels(metplus_config, item_list, configs_to_set, is_valid): - conf = metplus_config() - for key, value in configs_to_set.items(): - conf.set('config', key, value) - - assert(util.is_var_item_valid(item_list, '1', 'LEVELS', conf)[0] == is_valid) - -# test that if wrapper specific field info is specified, it only gets -# values from that list. All generic values should be read if no -# wrapper specific field info variables are specified -def test_parse_var_list_wrapper_specific(metplus_config): - conf = metplus_config() - conf.set('config', 'FCST_VAR1_NAME', "ENAME1") - conf.set('config', 'FCST_VAR1_LEVELS', "ELEVELS11, ELEVELS12") - conf.set('config', 'FCST_VAR2_NAME', "ENAME2") - conf.set('config', 'FCST_VAR2_LEVELS', "ELEVELS21, ELEVELS22") - conf.set('config', 'FCST_GRID_STAT_VAR1_NAME', "GNAME1") - conf.set('config', 'FCST_GRID_STAT_VAR1_LEVELS', "GLEVELS11, GLEVELS12") - - e_var_list = util.parse_var_list(conf, - time_info=None, - data_type='FCST', - met_tool='ensemble_stat') - - g_var_list = util.parse_var_list(conf, - time_info=None, - data_type='FCST', - met_tool='grid_stat') - - assert(len(e_var_list) == 4 and len(g_var_list) == 2 and - e_var_list[0]['fcst_name'] == "ENAME1" and - e_var_list[1]['fcst_name'] == "ENAME1" and - e_var_list[2]['fcst_name'] == "ENAME2" and - e_var_list[3]['fcst_name'] == "ENAME2" and - e_var_list[0]['fcst_level'] == "ELEVELS11" and - e_var_list[1]['fcst_level'] == "ELEVELS12" and - e_var_list[2]['fcst_level'] == "ELEVELS21" and - e_var_list[3]['fcst_level'] == "ELEVELS22" and - g_var_list[0]['fcst_name'] == "GNAME1" and - g_var_list[1]['fcst_name'] == "GNAME1" and - g_var_list[0]['fcst_level'] == "GLEVELS11" and - g_var_list[1]['fcst_level'] == "GLEVELS12") - -@pytest.mark.parametrize( - 'input_list, expected_list', [ - ('Point2Grid', ['Point2Grid']), - # MET documentation syntax (with dashes) - ('Pcp-Combine, Grid-Stat, Ensemble-Stat', ['PCPCombine', - 'GridStat', - 'EnsembleStat']), - ('Point-Stat', ['PointStat']), - ('Mode, MODE Time Domain', ['MODE', - 'MTD']), - # actual tool name (lower case underscore) - ('point_stat, grid_stat, ensemble_stat', ['PointStat', - 'GridStat', - 'EnsembleStat']), - ('mode, mtd', ['MODE', - 'MTD']), - ('ascii2nc, pb2nc, regrid_data_plane', ['ASCII2NC', - 'PB2NC', - 'RegridDataPlane']), - ('pcp_combine, tc_pairs, tc_stat', ['PCPCombine', - 'TCPairs', - 'TCStat']), - ('gen_vx_mask, stat_analysis, series_analysis', ['GenVxMask', - 'StatAnalysis', - 'SeriesAnalysis']), - # old capitalization format - ('PcpCombine, Ascii2Nc, TcStat, TcPairs', ['PCPCombine', - 'ASCII2NC', - 'TCStat', - 'TCPairs']), - # remove MakePlots from list - ('StatAnalysis, MakePlots', ['StatAnalysis']), - ] -) -def test_get_process_list(metplus_config, input_list, expected_list): - conf = metplus_config() - conf.set('config', 'PROCESS_LIST', input_list) - process_list = util.get_process_list(conf) - output_list = [item[0] for item in process_list] - assert(output_list == expected_list) - -@pytest.mark.parametrize( - 'input_list, expected_list', [ - # no instances - ('Point2Grid', [('Point2Grid', None)]), - # one with instance one without - ('PcpCombine, GridStat(my_instance)', [('PCPCombine', None), - ('GridStat', 'my_instance')]), - # duplicate process, one with instance one without - ('TCStat, ExtractTiles, TCStat(for_series), SeriesAnalysis', ( - [('TCStat',None), - ('ExtractTiles',None), - ('TCStat', 'for_series'), - ('SeriesAnalysis',None),])), - # two processes, both with instances - ('mode(uno), mtd(dos)', [('MODE', 'uno'), - ('MTD', 'dos')]), - # lower-case names, first with instance, second without - ('ascii2nc(some_name), pb2nc', [('ASCII2NC', 'some_name'), - ('PB2NC', None)]), - # duplicate process, both with different instances - ('tc_stat(one), tc_pairs, tc_stat(two)', [('TCStat', 'one'), - ('TCPairs', None), - ('TCStat', 'two')]), - ] -) -def test_get_process_list_instances(metplus_config, input_list, expected_list): - conf = metplus_config() - conf.set('config', 'PROCESS_LIST', input_list) - output_list = util.get_process_list(conf) - assert(output_list == expected_list) - @pytest.mark.parametrize( 'time_from_conf, fmt, is_datetime', [ ('', '%Y', False), @@ -1404,29 +487,6 @@ def test_round_0p5(value, expected_result): def test_comparison_to_letter_format(expression, expected_result): assert(util.comparison_to_letter_format(expression) == expected_result) -@pytest.mark.parametrize( - 'conf_items, met_tool, expected_result', [ - ({'CUSTOM_LOOP_LIST': "one, two, three"}, '', ['one', 'two', 'three']), - ({'CUSTOM_LOOP_LIST': "one, two, three", - 'GRID_STAT_CUSTOM_LOOP_LIST': "four, five",}, 'grid_stat', ['four', 'five']), - ({'CUSTOM_LOOP_LIST': "one, two, three", - 'GRID_STAT_CUSTOM_LOOP_LIST': "four, five",}, 'point_stat', ['one', 'two', 'three']), - ({'CUSTOM_LOOP_LIST': "one, two, three", - 'ASCII2NC_CUSTOM_LOOP_LIST': "four, five",}, 'ascii2nc', ['four', 'five']), - # fails to read custom loop list for point2grid because there are underscores in name - ({'CUSTOM_LOOP_LIST': "one, two, three", - 'POINT_2_GRID_CUSTOM_LOOP_LIST': "four, five",}, 'point2grid', ['one', 'two', 'three']), - ({'CUSTOM_LOOP_LIST': "one, two, three", - 'POINT2GRID_CUSTOM_LOOP_LIST': "four, five",}, 'point2grid', ['four', 'five']), - ] -) -def test_get_custom_string_list(metplus_config, conf_items, met_tool, expected_result): - config = metplus_config() - for conf_key, conf_value in conf_items.items(): - config.set('config', conf_key, conf_value) - - assert(util.get_custom_string_list(config, met_tool) == expected_result) - @pytest.mark.parametrize( 'skip_times_conf, expected_dict', [ ('"%d:30,31"', {'%d': ['30','31']}), @@ -1574,7 +634,7 @@ def test_get_storm_ids(metplus_config, filename, expected_result): 'stat_data', filename) - assert(util.get_storm_ids(filepath) == expected_result) + assert(util.get_storms(filepath, id_only=True) == expected_result) @pytest.mark.parametrize( 'filename, expected_result', [ @@ -1669,80 +729,6 @@ def test_format_var_items_options_semicolon(config_value, result = var_items.get('extra') assert(result == expected_result) -@pytest.mark.parametrize( - 'config_overrides, expected_results', [ - # 2 levels - ({'FCST_VAR1_NAME': 'read_data.py TMP {valid?fmt=%Y%m%d} {fcst_level}', - 'FCST_VAR1_LEVELS': 'P500,P250', - 'OBS_VAR1_NAME': 'read_data.py TMP {valid?fmt=%Y%m%d} {obs_level}', - 'OBS_VAR1_LEVELS': 'P500,P250', - }, - ['read_data.py TMP 20200201 P500', - 'read_data.py TMP 20200201 P250', - ]), - ({'BOTH_VAR1_NAME': 'read_data.py TMP {valid?fmt=%Y%m%d} {fcst_level}', - 'BOTH_VAR1_LEVELS': 'P500,P250', - }, - ['read_data.py TMP 20200201 P500', - 'read_data.py TMP 20200201 P250', - ]), - # no level but level specified in name - ({'FCST_VAR1_NAME': 'read_data.py TMP {valid?fmt=%Y%m%d} {fcst_level}', - 'OBS_VAR1_NAME': 'read_data.py TMP {valid?fmt=%Y%m%d} {obs_level}', - }, - ['read_data.py TMP 20200201 ', - ]), - # no level - ({'FCST_VAR1_NAME': 'read_data.py TMP {valid?fmt=%Y%m%d}', - 'OBS_VAR1_NAME': 'read_data.py TMP {valid?fmt=%Y%m%d}', - }, - ['read_data.py TMP 20200201', - ]), - # real example - ({'BOTH_VAR1_NAME': ('myscripts/read_nc2xr.py ' - 'mydata/forecast_file.nc4 TMP ' - '{valid?fmt=%Y%m%d_%H%M} {fcst_level}'), - 'BOTH_VAR1_LEVELS': 'P1000,P850,P700,P500,P250,P100', - }, - [('myscripts/read_nc2xr.py mydata/forecast_file.nc4 TMP 20200201_1225' - ' P1000'), - ('myscripts/read_nc2xr.py mydata/forecast_file.nc4 TMP 20200201_1225' - ' P850'), - ('myscripts/read_nc2xr.py mydata/forecast_file.nc4 TMP 20200201_1225' - ' P700'), - ('myscripts/read_nc2xr.py mydata/forecast_file.nc4 TMP 20200201_1225' - ' P500'), - ('myscripts/read_nc2xr.py mydata/forecast_file.nc4 TMP 20200201_1225' - ' P250'), - ('myscripts/read_nc2xr.py mydata/forecast_file.nc4 TMP 20200201_1225' - ' P100'), - ]), - ] -) -def test_parse_var_list_py_embed_multi_levels(metplus_config, config_overrides, - expected_results): - config = metplus_config() - for key, value in config_overrides.items(): - config.set('config', key, value) - - time_info = {'valid': datetime.datetime(2020, 2, 1, 12, 25)} - var_list = util.parse_var_list(config, - time_info=time_info, - data_type=None) - assert(len(var_list) == len(expected_results)) - - for var_item, expected_result in zip(var_list, expected_results): - assert(var_item['fcst_name'] == expected_result) - - # run again with data type specified - var_list = util.parse_var_list(config, - time_info=time_info, - data_type='FCST') - assert(len(var_list) == len(expected_results)) - - for var_item, expected_result in zip(var_list, expected_results): - assert(var_item['fcst_name'] == expected_result) - @pytest.mark.parametrize( 'level, expected_result', [ ('level', 'level'), @@ -1753,3 +739,63 @@ def test_parse_var_list_py_embed_multi_levels(metplus_config, config_overrides, ) def test_format_level(level, expected_result): assert(util.format_level(level) == expected_result) + +@pytest.mark.parametrize( + 'input_dict, expected_list', [ + ({'init': datetime.datetime(2019, 2, 1, 6), + 'lead': 7200, }, + [ + {'index': '1', + 'fcst_name': 'FNAME_2019', + 'fcst_level': 'Z06', + 'obs_name': 'ONAME_2019', + 'obs_level': 'L06', + }, + {'index': '1', + 'fcst_name': 'FNAME_2019', + 'fcst_level': 'Z08', + 'obs_name': 'ONAME_2019', + 'obs_level': 'L08', + }, + ]), + ({'init': datetime.datetime(2021, 4, 13, 9), + 'lead': 10800, }, + [ + {'index': '1', + 'fcst_name': 'FNAME_2021', + 'fcst_level': 'Z09', + 'obs_name': 'ONAME_2021', + 'obs_level': 'L09', + }, + {'index': '1', + 'fcst_name': 'FNAME_2021', + 'fcst_level': 'Z12', + 'obs_name': 'ONAME_2021', + 'obs_level': 'L12', + }, + ]), + ] +) +def test_sub_var_list(metplus_config, input_dict, expected_list): + config = metplus_config() + config.set('config', 'FCST_VAR1_NAME', 'FNAME_{init?fmt=%Y}') + config.set('config', 'FCST_VAR1_LEVELS', 'Z{init?fmt=%H}, Z{valid?fmt=%H}') + config.set('config', 'OBS_VAR1_NAME', 'ONAME_{init?fmt=%Y}') + config.set('config', 'OBS_VAR1_LEVELS', 'L{init?fmt=%H}, L{valid?fmt=%H}') + + time_info = time_util.ti_calculate(input_dict) + + actual_temp = parse_var_list(config) + + pp = pprint.PrettyPrinter() + print(f'Actual var list (before sub):') + pp.pprint(actual_temp) + + actual_list = util.sub_var_list(actual_temp, time_info) + print(f'Actual var list (after sub):') + pp.pprint(actual_list) + + assert(len(actual_list) == len(expected_list)) + for actual, expected in zip(actual_list, expected_list): + for key, value in expected.items(): + assert(actual.get(key) == value) diff --git a/internal_tests/pytests/mode/test_mode_wrapper.py b/internal_tests/pytests/mode/test_mode_wrapper.py index 69a17f9146..8c4144fd68 100644 --- a/internal_tests/pytests/mode/test_mode_wrapper.py +++ b/internal_tests/pytests/mode/test_mode_wrapper.py @@ -74,7 +74,8 @@ def set_minimum_config_settings(config): ({'MODE_REGRID_TO_GRID': 'FCST', }, - {'METPLUS_REGRID_DICT': 'regrid = {to_grid = FCST;}'}), + {'METPLUS_REGRID_DICT': 'regrid = {to_grid = FCST;}', + 'REGRID_TO_GRID': 'FCST'}), ({'MODE_REGRID_METHOD': 'NEAREST', }, @@ -100,7 +101,8 @@ def set_minimum_config_settings(config): }, {'METPLUS_REGRID_DICT': ('regrid = {to_grid = FCST;method = NEAREST;' 'width = 1;vld_thresh = 0.5;shape = SQUARE;}' - )}), + ), + 'REGRID_TO_GRID': 'FCST'}), ({'MODE_QUILT': 'True'}, {'METPLUS_QUILT': 'quilt = TRUE;'}), @@ -362,7 +364,11 @@ def test_mode_single_field(metplus_config, config_overrides, assert(cmd == expected_cmd) # check that environment variables were set properly - for env_var_key in wrapper.WRAPPER_ENV_VAR_KEYS: + # including deprecated env vars (not in wrapper env var keys) + env_var_keys = (wrapper.WRAPPER_ENV_VAR_KEYS + + [name for name in expected_output + if name not in wrapper.WRAPPER_ENV_VAR_KEYS]) + for env_var_key in env_var_keys: match = next((item for item in env_vars if item.startswith(env_var_key)), None) assert(match is not None) diff --git a/internal_tests/pytests/mtd/test_mtd_wrapper.py b/internal_tests/pytests/mtd/test_mtd_wrapper.py index 27cf14b611..aff12a1bb7 100644 --- a/internal_tests/pytests/mtd/test_mtd_wrapper.py +++ b/internal_tests/pytests/mtd/test_mtd_wrapper.py @@ -1,39 +1,12 @@ #!/usr/bin/env python3 import os -import sys -import re -import logging import datetime -from collections import namedtuple import pytest -import produtil - from metplus.wrappers.mtd_wrapper import MTDWrapper -from metplus.util import met_util as util - -# --------------------TEST CONFIGURATION and FIXTURE SUPPORT ------------- -# -# The test configuration and fixture support the additional configuration -# files used in METplus -# !!!!!!!!!!!!!!! -# !!!IMPORTANT!!! -# !!!!!!!!!!!!!!! -# The following two methods should be included in ALL pytest tests for METplus. -# -# -#def pytest_addoption(parser): -# parser.addoption("-c", action="store", help=" -c ") - - -# @pytest.fixture -#def cmdopt(request): -# return request.config.getoption("-c") -# -----------------FIXTURES THAT CAN BE USED BY ALL TESTS---------------- -#@pytest.fixture def mtd_wrapper(metplus_config, lead_seq=None): """! Returns a default MTDWrapper with /path/to entries in the metplus_system.conf and metplus_runtime.conf configuration diff --git a/internal_tests/pytests/pb2nc/test_pb2nc_wrapper.py b/internal_tests/pytests/pb2nc/test_pb2nc_wrapper.py index f9c3ab8cc0..a1c202c335 100644 --- a/internal_tests/pytests/pb2nc/test_pb2nc_wrapper.py +++ b/internal_tests/pytests/pb2nc/test_pb2nc_wrapper.py @@ -283,7 +283,7 @@ def test_find_input_files(metplus_config, offsets, offset_to_find): ] ) def test_pb2nc_all_fields(metplus_config, config_overrides, - env_var_values): + env_var_values): input_dir = '/some/input/dir' config = metplus_config() @@ -341,7 +341,11 @@ def test_pb2nc_all_fields(metplus_config, config_overrides, assert(cmd == expected_cmd) # check that environment variables were set properly - for env_var_key in wrapper.WRAPPER_ENV_VAR_KEYS: + # including deprecated env vars (not in wrapper env var keys) + env_var_keys = (wrapper.WRAPPER_ENV_VAR_KEYS + + [name for name in env_var_values + if name not in wrapper.WRAPPER_ENV_VAR_KEYS]) + for env_var_key in env_var_keys: match = next((item for item in env_vars if item.startswith(env_var_key)), None) assert(match is not None) diff --git a/internal_tests/pytests/point_stat/test_point_stat_wrapper.py b/internal_tests/pytests/point_stat/test_point_stat_wrapper.py index e8c2302f35..90d6840491 100755 --- a/internal_tests/pytests/point_stat/test_point_stat_wrapper.py +++ b/internal_tests/pytests/point_stat/test_point_stat_wrapper.py @@ -61,12 +61,10 @@ def test_met_dictionary_in_var_options(metplus_config): ({'DESC': 'my_desc'}, {'METPLUS_DESC': 'desc = "my_desc";'}), - ({'OBTYPE': 'my_obtype'}, - {'METPLUS_OBTYPE': 'obtype = "my_obtype";'}), - ({'POINT_STAT_REGRID_TO_GRID': 'FCST', }, - {'METPLUS_REGRID_DICT': 'regrid = {to_grid = FCST;}'}), + {'METPLUS_REGRID_DICT': 'regrid = {to_grid = FCST;}', + 'REGRID_TO_GRID': 'FCST'}), ({'POINT_STAT_REGRID_METHOD': 'NEAREST', }, @@ -92,7 +90,8 @@ def test_met_dictionary_in_var_options(metplus_config): }, {'METPLUS_REGRID_DICT': ('regrid = {to_grid = FCST;method = NEAREST;' 'width = 1;vld_thresh = 0.5;shape = SQUARE;}' - )}), + ), + 'REGRID_TO_GRID': 'FCST'}), # mask grid and poly (old config var) ({'POINT_STAT_MASK_GRID': 'FULL', @@ -139,15 +138,6 @@ def test_met_dictionary_in_var_options(metplus_config): 'sid = ["one", "two"];', }), - ({'POINT_STAT_NEIGHBORHOOD_COV_THRESH': '>=0.5'}, - {'METPLUS_NBRHD_COV_THRESH': 'cov_thresh = [>=0.5];'}), - - ({'POINT_STAT_NEIGHBORHOOD_WIDTH': '1,2'}, - {'METPLUS_NBRHD_WIDTH': 'width = [1, 2];'}), - - ({'POINT_STAT_NEIGHBORHOOD_SHAPE': 'CIRCLE'}, - {'METPLUS_NBRHD_SHAPE': 'shape = CIRCLE;'}), - ({'POINT_STAT_OUTPUT_PREFIX': 'my_output_prefix'}, {'METPLUS_OUTPUT_PREFIX': 'output_prefix = "my_output_prefix";'}), @@ -507,7 +497,11 @@ def test_point_stat_all_fields(metplus_config, config_overrides, assert(cmd == expected_cmd) # check that environment variables were set properly - for env_var_key in wrapper.WRAPPER_ENV_VAR_KEYS: + # including deprecated env vars (not in wrapper env var keys) + env_var_keys = (wrapper.WRAPPER_ENV_VAR_KEYS + + [name for name in env_var_values + if name not in wrapper.WRAPPER_ENV_VAR_KEYS]) + for env_var_key in env_var_keys: match = next((item for item in env_vars if item.startswith(env_var_key)), None) assert(match is not None) diff --git a/internal_tests/pytests/tc_gen/test_tc_gen_wrapper.py b/internal_tests/pytests/tc_gen/test_tc_gen_wrapper.py index 9486c629b6..825a48d67a 100644 --- a/internal_tests/pytests/tc_gen/test_tc_gen_wrapper.py +++ b/internal_tests/pytests/tc_gen/test_tc_gen_wrapper.py @@ -346,7 +346,11 @@ def test_tc_gen(metplus_config, config_overrides, env_var_values): assert(cmd == expected_cmd) # check that environment variables were set properly - for env_var_key in wrapper.WRAPPER_ENV_VAR_KEYS: + # including deprecated env vars (not in wrapper env var keys) + env_var_keys = (wrapper.WRAPPER_ENV_VAR_KEYS + + [name for name in env_var_values + if name not in wrapper.WRAPPER_ENV_VAR_KEYS]) + for env_var_key in env_var_keys: match = next((item for item in env_vars if item.startswith(env_var_key)), None) assert(match is not None) diff --git a/metplus/util/__init__.py b/metplus/util/__init__.py index 570ee45208..c333616cbf 100644 --- a/metplus/util/__init__.py +++ b/metplus/util/__init__.py @@ -4,4 +4,4 @@ from .time_util import * from .met_util import * from .string_template_substitution import * -from .met_dictionary_info import * +from .met_config import * diff --git a/metplus/util/config_metplus.py b/metplus/util/config_metplus.py index 3a6df221d3..892990994e 100644 --- a/metplus/util/config_metplus.py +++ b/metplus/util/config_metplus.py @@ -13,26 +13,23 @@ import re import sys import logging -import collections import datetime import shutil -from os.path import dirname, realpath -import inspect from configparser import ConfigParser, NoOptionError from pathlib import Path -import copy from produtil.config import ProdConfig -import produtil.fileop from . import met_util as util +from .string_template_substitution import get_tags, do_string_sub +from .met_util import getlist, is_python_script, format_var_items +from .doc_util import get_wrapper_name """!Creates the initial METplus directory structure, loads information into each job. This module is used to create the initial METplus conf file in the first METplus job via the metplus.config_metplus.launch(). -The metplus.config_metplus.load() then reloads that configuration. The launch() function does more than just create the conf file though. It creates several initial files and directories @@ -43,15 +40,16 @@ """ '''!@var __all__ -All symbols exported by "from metplus.util.config.config_metplus import *" +All symbols exported by "from metplus.util.config_metplus import *" ''' -__all__ = ['load', - 'launch', - 'parse_launch_args', - 'setup', - 'METplusConfig', - 'METplusLogFormatter', - ] +__all__ = [ + 'setup', + 'get_custom_string_list', + 'find_indices_in_config_section', + 'parse_var_list', + 'get_process_list', + 'validate_configuration_variables', +] '''!@var METPLUS_BASE The METplus installation directory @@ -81,7 +79,33 @@ 'metplus_logging.conf' ] -def get_default_config_list(parm_base=None): +def setup(args, logger=None, base_confs=None): + """!The METplus setup function. + @param args list of configuration files or configuration + variable overrides. Reads all configuration inputs and returns + a configuration object. + """ + if base_confs is None: + base_confs = _get_default_config_list() + + # Setup Task logger, Until a Conf object is created, Task logger is + # only logging to tty, not a file. + if logger is None: + logger = logging.getLogger('metplus') + + logger.info('Starting METplus configuration setup.') + + override_list = _parse_launch_args(args, logger) + + # add default config files to override list + override_list = base_confs + override_list + config = launch(override_list) + + logger.debug('Completed METplus configuration setup.') + + return config + +def _get_default_config_list(parm_base=None): """! Get list of default METplus config files. Look through BASE_CONFS list and check if each file exists under the parm base. Add each to a list if they do exist. @@ -105,38 +129,12 @@ def get_default_config_list(parm_base=None): default_config_list.append(conf_path) if not default_config_list: - print(f"ERROR: No default config files found in {conf_dir}") + print(f"FATAL: No default config files found in {conf_dir}") sys.exit(1) return default_config_list -def setup(args, logger=None, base_confs=None): - """!The METplus setup function. - @param args list of configuration files or configuration - variable overrides. Reads all configuration inputs and returns - a configuration object. - """ - if base_confs is None: - base_confs = get_default_config_list() - - # Setup Task logger, Until a Conf object is created, Task logger is - # only logging to tty, not a file. - if logger is None: - logger = logging.getLogger('metplus') - - logger.info('Starting METplus configuration setup.') - - override_list = parse_launch_args(args, logger) - - # add default config files to override list - override_list = base_confs + override_list - config = launch(override_list) - - logger.debug('Completed METplus configuration setup.') - - return config - -def parse_launch_args(args, logger): +def _parse_launch_args(args, logger): """! Parsed arguments to scripts that launch the METplus wrappers. Options: @@ -207,8 +205,7 @@ def launch(config_list): read files. Explicit configuration variables are read after all config files are processed. - @param file_list list of configuration files to read - @param moreopt explicit configuration variable overrides + @param config_list list of configuration files to process """ config = METplusConfig() logger = config.log() @@ -235,7 +232,7 @@ def launch(config_list): config_format_list.append(f'{section}.{key}={value}') # move all config variables from old sections into the [config] section - config.move_all_to_config_section() + config._move_all_to_config_section() # save list of user configuration files in a variable config.set('config', 'CONFIG_INPUT', ','.join(config_format_list)) @@ -265,19 +262,6 @@ def launch(config_list): return config -def load(filename): - """!Loads the METplusConfig created by the launch() function. - - Creates an METplusConfig object for a METplus workflow that was - previously initialized by metplus.config_metplus.launch. - The only argument is the name of the config file produced by - the launch command. - - @param filename The metplus*.conf file created by launch()""" - config = METplusConfig() - config.read(filename) - return config - def _set_logvars(config, logger=None): """!Sets and adds the LOG_METPLUS and LOG_TIMESTAMP to the config object. If LOG_METPLUS was already defined by the @@ -292,44 +276,22 @@ def _set_logvars(config, logger=None): if logger is None: logger = config.log() - # LOG_TIMESTAMP_TEMPLATE is not required in the conf file, - # so lets first test for that. - log_timestamp_template = config.getstr('config', 'LOG_TIMESTAMP_TEMPLATE', '') - if log_timestamp_template: - # Note: strftime appears to handle if log_timestamp_template - # is a string ie. 'blah' and not a valid set of % directives %Y%m%d, - # it does return the string 'blah', instead of crashing. - # However, I'm still going to test for a valid % directive and - # set a default. It probably is ok to remove the if not block pattern - # test, and not set a default, especially if causing some unintended - # consequences or the pattern is not capturing a valid directive. - # The reality is, the user is expected to have entered a correct - # directive in the conf file. - # This pattern is meant to test for a repeating set of - # case insensitive %(AnyAlphabeticCharacter), ie. %Y%m ... - # The basic pattern is (%+[a-z])+ , %+ allows for 1 or more - # % characters, ie. %%Y, %% is a valid directive. - # (?i) case insensitive, \A begin string \Z end of string - if not re.match(r'(?i)\A(?:(%+[a-z])+)\Z', log_timestamp_template): - logger.warning('Your LOG_TIMESTAMP_TEMPLATE is not ' - 'a valid strftime directive: %s' % repr(log_timestamp_template)) - logger.info('Using the following default: %Y%m%d%H') - log_timestamp_template = '%Y%m%d%H' - date_t = datetime.datetime.now() - if config.getbool('config', 'LOG_TIMESTAMP_USE_DATATIME', False): - if util.is_loop_by_init(config): - date_t = datetime.datetime.strptime(config.getstr('config', - 'INIT_BEG'), - config.getstr('config', - 'INIT_TIME_FMT')) - else: - date_t = datetime.datetime.strptime(config.getstr('config', - 'VALID_BEG'), - config.getstr('config', - 'VALID_TIME_FMT')) - log_filenametimestamp = date_t.strftime(log_timestamp_template) + log_timestamp_template = config.getstr('config', 'LOG_TIMESTAMP_TEMPLATE', + '') + if config.getbool('config', 'LOG_TIMESTAMP_USE_DATATIME', False): + if util.is_loop_by_init(config): + loop_by = 'INIT' + else: + loop_by = 'VALID' + + date_t = datetime.datetime.strptime( + config.getstr('config', f'{loop_by}_BEG'), + config.getstr('config', f'{loop_by}_TIME_FMT') + ) else: - log_filenametimestamp = '' + date_t = datetime.datetime.now() + + log_filenametimestamp = date_t.strftime(log_timestamp_template) log_dir = config.getdir('LOG_DIR') @@ -343,11 +305,14 @@ def _set_logvars(config, logger=None): if config.has_option('config', 'LOG_METPLUS'): user_defined_log_file = True # strinterp will set metpluslog to '' if LOG_METPLUS = is unset. - metpluslog = config.strinterp('config', '{LOG_METPLUS}', - LOG_TIMESTAMP_TEMPLATE=log_filenametimestamp) - - # test if there is any path information, if there is, assUme it is as intended, - # if there is not, than add log_dir. + metpluslog = config.strinterp( + 'config', + '{LOG_METPLUS}', + LOG_TIMESTAMP_TEMPLATE=log_filenametimestamp + ) + + # test if there is any path information, if there is, + # assume it is as intended, if there is not, than add log_dir. if metpluslog: if os.path.basename(metpluslog) == metpluslog: metpluslog = os.path.join(log_dir, metpluslog) @@ -360,12 +325,15 @@ def _set_logvars(config, logger=None): # it out, in case the group wanted a stand alone metplus log filename # template variable. - # If metpluslog_filename includes a path, python joins it intelligently. + # If metpluslog_filename includes a path, python joins it intelligently # Set the metplus log filename. - # strinterp will set metpluslog_filename to '' if LOG_FILENAME_TEMPLATE = + # strinterp will set metpluslog_filename to '' if template is empty if config.has_option('config', 'LOG_FILENAME_TEMPLATE'): - metpluslog_filename = config.strinterp('config', '{LOG_FILENAME_TEMPLATE}', - LOG_TIMESTAMP_TEMPLATE=log_filenametimestamp) + metpluslog_filename = config.strinterp( + 'config', + '{LOG_FILENAME_TEMPLATE}', + LOG_TIMESTAMP_TEMPLATE=log_filenametimestamp + ) else: metpluslog_filename = '' if metpluslog_filename: @@ -374,15 +342,15 @@ def _set_logvars(config, logger=None): metpluslog = '' # Adding LOG_TIMESTAMP to the final configuration file. - logger.info('Adding: config.LOG_TIMESTAMP=%s' % repr(log_filenametimestamp)) + logger.info('Adding LOG_TIMESTAMP=%s' % repr(log_filenametimestamp)) config.set('config', 'LOG_TIMESTAMP', log_filenametimestamp) # Setting LOG_METPLUS in the configuration object # At this point LOG_METPLUS will have a value or '' the empty string. if user_defined_log_file: - logger.info('Replace [config] LOG_METPLUS with %s' % repr(metpluslog)) + logger.info('Replace LOG_METPLUS with %s' % repr(metpluslog)) else: - logger.info('Adding: config.LOG_METPLUS=%s' % repr(metpluslog)) + logger.info('Adding LOG_METPLUS=%s' % repr(metpluslog)) # expand LOG_METPLUS to ensure it is available config.set('config', 'LOG_METPLUS', metpluslog) @@ -501,25 +469,28 @@ def replace_config_from_section(config, section, required=True): return new_config class METplusConfig(ProdConfig): - """!A replacement for the produtil.config.ProdConfig used throughout - the METplus system. You should never need to instantiate one of - these --- the launch() and load() functions do that for you. This - class is the underlying implementation of most of the - functionality described in launch() and load()""" - - # items that are found in these sections will be moved into the [config] section - OLD_SECTIONS = ['dir', - 'exe', - 'filename_templates', - 'regex_pattern', - ] + """! Configuration class to store configuration values read from + METplus config files. + """ + + # items that are found in these sections + # will be moved into the [config] section + OLD_SECTIONS = ( + 'dir', + 'exe', + 'filename_templates', + 'regex_pattern', + ) def __init__(self, conf=None): """!Creates a new METplusConfig - @param conf The configuration file.""" + @param conf The configuration file + """ # set interpolation to None so you can supply filename template # that contain % to config.set - conf = ConfigParser(strict=False, inline_comment_prefixes=(';',), interpolation=None) if (conf is None) else conf + conf = ConfigParser(strict=False, + inline_comment_prefixes=(';',), + interpolation=None) if (conf is None) else conf super().__init__(conf) self._cycle = None self._logger = logging.getLogger('metplus') @@ -530,11 +501,11 @@ def __init__(self, conf=None): # get the OS environment and store it self.env = os.environ.copy() - # add user_env_vars section to hold environment variables defined by the user + # add section to hold environment variables defined by the user self.add_section('user_env_vars') def log(self, sublog=None): - """!Overrides method in ProdConfig + """! Overrides method in ProdConfig If the sublog argument is provided, then the logger will be under that subdomain of the "metplus" logging domain. Otherwise, this METplusConfig's logger @@ -546,7 +517,7 @@ def log(self, sublog=None): return logging.getLogger('metplus.'+sublog) return self._logger - def move_all_to_config_section(self): + def _move_all_to_config_section(self): """! Move all configuration variables that are found in the previously supported sections into the config section. """ @@ -588,6 +559,7 @@ def move_runtime_configs(self): ] more_run_confs = [item for item in self.keys(from_section) if item.startswith('LOG') or item.endswith('BASE')] + # create destination section if it does not exist if not self.has_section(to_section): self._conf.add_section(to_section) @@ -611,42 +583,19 @@ def remove_current_vars(self): if self.has_option('config', current_var): self._conf.remove_option('config', current_var) - def find_section(self, sec, opt): - """! Search through list of previously supported config sections - to find variable requested. This allows the removal of these - sections to consider all of the variables members of the - [config] section. - Args: - @param sec section requested - look in this section first - @param opt configuration variable to find - @returns section heading name or None if not found - """ - # first check the section requested - if self.has_option(sec, opt): - return sec - - # loop through previously supported sections to find variable opt - # return section name if found - for section in self.OLD_SECTIONS: - if self.has_option(section, opt): - return section - - # return None if variable is not found - return None - # override get methods to perform additional error checking def getraw(self, sec, opt, default='', count=0): """ parse parameter and replace any existing parameters referenced with the value (looking in same section, then config, dir, and os environment) returns raw string, preserving {valid?fmt=%Y} blocks - Args: - @param sec: Section in the conf file to look for variable - @param opt: Variable to interpret - @param default: Default value to use if config is not set - @param count: Counter used to stop recursion to prevent infinite - Returns: - Raw string or empty string if function calls itself too many times + + @param sec: Section in the conf file to look for variable + @param opt: Variable to interpret + @param default: Default value to use if config is not set + @param count: Counter used to stop recursion to prevent infinite + @returns Raw string or empty string if function calls itself too + many times """ if count >= 10: self.logger.error("Could not resolve getraw - check for circular " @@ -689,13 +638,14 @@ def getraw(self, sec, opt, default='', count=0): return in_template.replace('//', '/') def check_default(self, sec, name, default): - """!helper function for get methods, report error and raise NoOptionError if - default is not set. If default is set, set the config variable to the - default value so that the value is stored in the final conf - Args: - @param sec section of config - @param name name of config variable - @param default value to use - if set to None, error and raise exception + """! helper function for get methods, report error and raise + NoOptionError if default is not set. + If default is set, set the config variable to the + default value so that the value is stored in the final conf + + @param sec section of config + @param name name of config variable + @param default value to use - if set to None, error/raise exception """ if default is None: raise @@ -720,8 +670,9 @@ def check_default(self, sec, name, default): self.set(sec, name, default) def getexe(self, exe_name): - """!Wraps produtil exe with checks to see if option is set and if - exe actually exists. Returns None if not found instead of exiting""" + """! Wraps produtil exe with checks to see if option is set and if + exe actually exists. Returns None if not found instead of exiting + """ try: exe_path = super().getstr('config', exe_name) except NoOptionError as e: @@ -734,7 +685,7 @@ def getexe(self, exe_name): full_exe_path = shutil.which(exe_path) if full_exe_path is None: - msg = 'Executable {} does not exist at {}'.format(exe_name, exe_path) + msg = f'Executable {exe_name} does not exist at {exe_path}' if self.logger: self.logger.error(msg) else: @@ -745,8 +696,11 @@ def getexe(self, exe_name): self.set('config', exe_name, full_exe_path) return full_exe_path - def getdir(self, dir_name, default=None, morevars=None,taskvars=None, must_exist=False): - """!Wraps produtil getdir and reports an error if it is set to /path/to""" + def getdir(self, dir_name, default=None, morevars=None,taskvars=None, + must_exist=False): + """! Wraps produtil getdir and reports an error if + it is set to /path/to + """ try: dir_path = super().getstr('config', dir_name, default=None, morevars=morevars, taskvars=taskvars) @@ -755,7 +709,8 @@ def getdir(self, dir_name, default=None, morevars=None,taskvars=None, must_exist dir_path = default if '/path/to' in dir_path: - raise ValueError("[config] " + dir_name + " cannot be set to or contain '/path/to'") + raise ValueError(f"{dir_name} cannot be set to " + "or contain '/path/to'") if '\n' in dir_path: raise ValueError(f"Invalid value for [config] {dir_name} " @@ -769,24 +724,28 @@ def getdir(self, dir_name, default=None, morevars=None,taskvars=None, must_exist return dir_path.replace('//', '/') def getdir_nocheck(self, dir_name, default=None): - return super().getstr('config', dir_name, default=default).replace('//', '/') + return super().getstr('config', dir_name, + default=default).replace('//', '/') def getstr_nocheck(self, sec, name, default=None): - # if requested section is in the list of sections that are no longer used - # look in the [config] section for the variable + # if requested section is in the list of sections that are + # no longer used look in the [config] section for the variable if sec in self.OLD_SECTIONS: sec = 'config' return super().getstr(sec, name, default=default).replace('//', '/') - def getstr(self, sec, name, default=None, badtypeok=False, morevars=None, taskvars=None): - """!Wraps produtil getstr. Config variable is checked with a default value of None - because if the config is not set and a default is specified, it will just return - that value. We want to log that a default was used and set it in the config so - it will show up in the final conf that is generated at the end of execution. - If no default was specified in the call, the NoOptionError is raised again. - Replace double forward slash with single to prevent error that occurs if that - is found inside a MET config file (because it considers // the start of a comment + def getstr(self, sec, name, default=None, badtypeok=False, morevars=None, + taskvars=None): + """! Wraps produtil getstr. Config variable is checked with a default + value of None because if the config is not set and a default is + specified, it will just return that value. + We want to log that a default was used and set it in the config so + it will show up in the final conf that is generated at the end of + execution. If no default was specified in the call, + the NoOptionError is raised again. Replace double forward slash + with single to prevent error that occurs if that is found inside + a MET config file because it considers // the start of a comment """ if sec in self.OLD_SECTIONS: sec = 'config' @@ -800,20 +759,25 @@ def getstr(self, sec, name, default=None, badtypeok=False, morevars=None, taskva self.check_default(sec, name, default) return default.replace('//', '/') - def getbool(self, sec, name, default=None, badtypeok=False, morevars=None, taskvars=None): - """!Wraps produtil getbool. Config variable is checked with a default value of None - because if the config is not set and a default is specified, it will just return - that value. We want to log that a default was used and set it in the config so - it will show up in the final conf that is generated at the end of execution. - If no default was specified in the call, the NoOptionError is raised again. - @returns None if value is not a boolean (or yes/no), value if set, default if not set + def getbool(self, sec, name, default=None, badtypeok=False, morevars=None, + taskvars=None): + """! Wraps produtil getbool. Config variable is checked with a + default value of None because if the config is not set and a + default is specified, it will just return that value. + We want to log that a default was used and set it in the config so + it will show up in the final conf that is generated at the end of + execution. If no default was specified in the call, + the NoOptionError is raised again. + @returns None if value is not a boolean (or yes/no), value if set, + default if not set """ if sec in self.OLD_SECTIONS: sec = 'config' try: return super().getbool(sec, name, default=None, - badtypeok=badtypeok, morevars=morevars, taskvars=taskvars) + badtypeok=badtypeok, morevars=morevars, + taskvars=taskvars) except NoOptionError: # config item was not set self.check_default(sec, name, default) @@ -838,17 +802,21 @@ def getbool(self, sec, name, default=None, badtypeok=False, morevars=None, taskv self.logger.error(f"[{sec}] {name} must be an boolean.") return None - def getint(self, sec, name, default=None, badtypeok=False, morevars=None, taskvars=None): + def getint(self, sec, name, default=None, badtypeok=False, morevars=None, + taskvars=None): """!Wraps produtil getint to gracefully report if variable is not set and no default value is specified - @returns Value if set, default of missing value if not set, None if value is an incorrect type""" + @returns Value if set, default of missing value if not set, + None if value is an incorrect type""" if sec in self.OLD_SECTIONS: sec = 'config' try: - # call ProdConfig function with no default set so we can log and set the default + # call ProdConfig function with no default set so + # we can log and set the default return super().getint(sec, name, default=None, - badtypeok=badtypeok, morevars=morevars, taskvars=taskvars) + badtypeok=badtypeok, morevars=morevars, + taskvars=taskvars) # if config variable is not set except NoOptionError: @@ -860,7 +828,7 @@ def getint(self, sec, name, default=None, badtypeok=False, morevars=None, taskva # if invalid value except ValueError: - # check if it was an empty string and return MISSING_DATA_VALUE if so + # check if it was an empty string and return MISSING_DATA_VALUE if super().getstr(sec, name) == '': return util.MISSING_DATA_VALUE @@ -868,17 +836,21 @@ def getint(self, sec, name, default=None, badtypeok=False, morevars=None, taskva self.logger.error(f"[{sec}] {name} must be an integer.") return None - def getfloat(self, sec, name, default=None, badtypeok=False, morevars=None, taskvars=None): + def getfloat(self, sec, name, default=None, badtypeok=False, morevars=None, + taskvars=None): """!Wraps produtil getint to gracefully report if variable is not set and no default value is specified - @returns Value if set, default of missing value if not set, None if value is an incorrect type""" + @returns Value if set, default of missing value if not set, + None if value is an incorrect type""" if sec in self.OLD_SECTIONS: sec = 'config' try: - # call ProdConfig function with no default set so we can log and set the default + # call ProdConfig function with no default set so + # we can log and set the default return super().getfloat(sec, name, default=None, - badtypeok=badtypeok, morevars=morevars, taskvars=taskvars) + badtypeok=badtypeok, morevars=morevars, + taskvars=taskvars) # if config variable is not set except NoOptionError: @@ -890,7 +862,7 @@ def getfloat(self, sec, name, default=None, badtypeok=False, morevars=None, task # if invalid value except ValueError: - # check if it was an empty string and return MISSING_DATA_VALUE if so + # check if it was an empty string and return MISSING_DATA_VALUE if super().getstr(sec, name) == '': return util.MISSING_DATA_VALUE @@ -898,7 +870,8 @@ def getfloat(self, sec, name, default=None, badtypeok=False, morevars=None, task self.logger.error(f"[{sec}] {name} must be a float.") return None - def getseconds(self, sec, name, default=None, badtypeok=False, morevars=None, taskvars=None): + def getseconds(self, sec, name, default=None, badtypeok=False, + morevars=None, taskvars=None): """!Converts time values ending in H, M, or S to seconds""" if sec in self.OLD_SECTIONS: sec = 'config' @@ -907,7 +880,8 @@ def getseconds(self, sec, name, default=None, badtypeok=False, morevars=None, ta # convert value to seconds # Valid options match format 3600, 3600S, 60M, or 1H value = super().getstr(sec, name, default=None, - badtypeok=badtypeok, morevars=morevars, taskvars=taskvars) + badtypeok=badtypeok, morevars=morevars, + taskvars=taskvars) regex_and_multiplier = {r'(-*)(\d+)S': 1, r'(-*)(\d+)M': 60, r'(-*)(\d+)H': 3600, @@ -921,8 +895,8 @@ def getseconds(self, sec, name, default=None, badtypeok=False, morevars=None, ta return int(match.group(2)) * mult # if value is not in an expected format, error and exit - msg = '[{}] {} does not match expected format. '.format(sec, name) +\ - 'Valid options match 3600, 3600S, 60M, or 1H' + msg = (f'[{sec}] {name} does not match expected format. ' + 'Valid options match 3600, 3600S, 60M, or 1H') if self.logger: self.logger.error(msg) else: @@ -935,14 +909,38 @@ def getseconds(self, sec, name, default=None, badtypeok=False, morevars=None, ta self.check_default(sec, name, default) return default + def get_mp_config_name(self, mp_config): + """! Get first name of METplus config variable that is set. + + @param mp_config list of METplus config keys to check. Can also be a + single item + @returns Name of first METplus config name in list that is set in the + METplusConfig object. None if none keys in the list are set. + """ + if not isinstance(mp_config, list): + mp_configs = [mp_config] + else: + mp_configs = mp_config + + for mp_config_name in mp_configs: + if self.has_option('config', mp_config_name): + return mp_config_name + + return None + + class METplusLogFormatter(logging.Formatter): def __init__(self, config): self.default_fmt = config.getraw('config', 'LOG_LINE_FORMAT') - self.info_fmt = config.getraw('config', 'LOG_INFO_LINE_FORMAT', self.default_fmt) - self.debug_fmt = config.getraw('config', 'LOG_DEBUG_LINE_FORMAT', self.default_fmt) - self.error_fmt = config.getraw('config', 'LOG_ERR_LINE_FORMAT', self.default_fmt) + self.info_fmt = config.getraw('config', 'LOG_INFO_LINE_FORMAT', + self.default_fmt) + self.debug_fmt = config.getraw('config', 'LOG_DEBUG_LINE_FORMAT', + self.default_fmt) + self.error_fmt = config.getraw('config', 'LOG_ERR_LINE_FORMAT', + self.default_fmt) super().__init__(fmt=self.default_fmt, - datefmt=config.getraw('config', 'LOG_LINE_DATE_FORMAT'), + datefmt=config.getraw('config', + 'LOG_LINE_DATE_FORMAT'), style='%') def format(self, record): @@ -959,3 +957,1016 @@ def format(self, record): self._style._fmt = self.default_fmt return output + +def validate_configuration_variables(config, force_check=False): + + all_sed_cmds = [] + # check for deprecated config items and warn user to remove/replace them + deprecated_isOK, sed_cmds = check_for_deprecated_config(config) + all_sed_cmds.extend(sed_cmds) + + # check for deprecated env vars in MET config files and warn user to remove/replace them + deprecatedMET_isOK, sed_cmds = check_for_deprecated_met_config(config) + all_sed_cmds.extend(sed_cmds) + + # validate configuration variables + field_isOK, sed_cmds = validate_field_info_configs(config, force_check) + all_sed_cmds.extend(sed_cmds) + + # check that OUTPUT_BASE is not set to the exact same value as INPUT_BASE + inoutbase_isOK = True + input_real_path = os.path.realpath(config.getdir_nocheck('INPUT_BASE', '')) + output_real_path = os.path.realpath(config.getdir('OUTPUT_BASE')) + if input_real_path == output_real_path: + config.logger.error(f"INPUT_BASE AND OUTPUT_BASE are set to the exact same path: {input_real_path}") + config.logger.error("Please change one of these paths to avoid risk of losing input data") + inoutbase_isOK = False + + check_user_environment(config) + + return deprecated_isOK, field_isOK, inoutbase_isOK, deprecatedMET_isOK, all_sed_cmds + +def check_for_deprecated_config(config): + """!Checks user configuration files and reports errors or warnings if any deprecated variable + is found. If an alternate variable name can be suggested, add it to the 'alt' section + If the alternate cannot be literally substituted for the old name, set copy to False + Args: + @config : METplusConfig object to evaluate + Returns: + A tuple containing a boolean if the configuration is suitable to run or not and + if it is not correct, the 2nd item is a list of sed commands that can be run to help + fix the incorrect configuration variables + """ + + # key is the name of the depreacted variable that is no longer allowed in any config files + # value is a dictionary containing information about what to do with the deprecated config + # 'sec' is the section of the config file where the replacement resides, i.e. config, dir, + # filename_templates + # 'alt' is the alternative name for the deprecated config. this can be a single variable name or + # text to describe multiple variables or how to handle it. Set to None to tell the user to + # just remove the variable + # 'copy' is an optional item (defaults to True). set this to False if one cannot simply replace + # the deprecated config variable name with the value in 'alt' + # 'req' is an optional item (defaults to True). this to False to report a warning for the + # deprecated config and allow execution to continue. this is generally no longer used + # because we are requiring users to update the config files. if used, the developer must + # modify the code to handle both variables accordingly + deprecated_dict = { + 'LOOP_BY_INIT' : {'sec' : 'config', 'alt' : 'LOOP_BY', 'copy': False}, + 'LOOP_METHOD' : {'sec' : 'config', 'alt' : 'LOOP_ORDER'}, + 'PREPBUFR_DIR_REGEX' : {'sec' : 'regex_pattern', 'alt' : None}, + 'PREPBUFR_FILE_REGEX' : {'sec' : 'regex_pattern', 'alt' : None}, + 'OBS_INPUT_DIR_REGEX' : {'sec' : 'regex_pattern', 'alt' : 'OBS_POINT_STAT_INPUT_DIR', 'copy': False}, + 'FCST_INPUT_DIR_REGEX' : {'sec' : 'regex_pattern', 'alt' : 'FCST_POINT_STAT_INPUT_DIR', 'copy': False}, + 'FCST_INPUT_FILE_REGEX' : + {'sec' : 'regex_pattern', 'alt' : 'FCST_POINT_STAT_INPUT_TEMPLATE', 'copy': False}, + 'OBS_INPUT_FILE_REGEX' : {'sec' : 'regex_pattern', 'alt' : 'OBS_POINT_STAT_INPUT_TEMPLATE', 'copy': False}, + 'PREPBUFR_DATA_DIR' : {'sec' : 'dir', 'alt' : 'PB2NC_INPUT_DIR'}, + 'PREPBUFR_MODEL_DIR_NAME' : {'sec' : 'dir', 'alt' : 'PB2NC_INPUT_DIR', 'copy': False}, + 'OBS_INPUT_FILE_TMPL' : + {'sec' : 'filename_templates', 'alt' : 'OBS_POINT_STAT_INPUT_TEMPLATE'}, + 'FCST_INPUT_FILE_TMPL' : + {'sec' : 'filename_templates', 'alt' : 'FCST_POINT_STAT_INPUT_TEMPLATE'}, + 'NC_FILE_TMPL' : {'sec' : 'filename_templates', 'alt' : 'PB2NC_OUTPUT_TEMPLATE'}, + 'FCST_INPUT_DIR' : {'sec' : 'dir', 'alt' : 'FCST_POINT_STAT_INPUT_DIR'}, + 'OBS_INPUT_DIR' : {'sec' : 'dir', 'alt' : 'OBS_POINT_STAT_INPUT_DIR'}, + 'REGRID_TO_GRID' : {'sec' : 'config', 'alt' : 'POINT_STAT_REGRID_TO_GRID'}, + 'FCST_HR_START' : {'sec' : 'config', 'alt' : 'LEAD_SEQ', 'copy': False}, + 'FCST_HR_END' : {'sec' : 'config', 'alt' : 'LEAD_SEQ', 'copy': False}, + 'FCST_HR_INTERVAL' : {'sec' : 'config', 'alt' : 'LEAD_SEQ', 'copy': False}, + 'START_DATE' : {'sec' : 'config', 'alt' : 'INIT_BEG or VALID_BEG', 'copy': False}, + 'END_DATE' : {'sec' : 'config', 'alt' : 'INIT_END or VALID_END', 'copy': False}, + 'INTERVAL_TIME' : {'sec' : 'config', 'alt' : 'INIT_INCREMENT or VALID_INCREMENT', 'copy': False}, + 'BEG_TIME' : {'sec' : 'config', 'alt' : 'INIT_BEG or VALID_BEG', 'copy': False}, + 'END_TIME' : {'sec' : 'config', 'alt' : 'INIT_END or VALID_END', 'copy': False}, + 'START_HOUR' : {'sec' : 'config', 'alt' : 'INIT_BEG or VALID_BEG', 'copy': False}, + 'END_HOUR' : {'sec' : 'config', 'alt' : 'INIT_END or VALID_END', 'copy': False}, + 'OBS_BUFR_VAR_LIST' : {'sec' : 'config', 'alt' : 'PB2NC_OBS_BUFR_VAR_LIST'}, + 'TIME_SUMMARY_FLAG' : {'sec' : 'config', 'alt' : 'PB2NC_TIME_SUMMARY_FLAG'}, + 'TIME_SUMMARY_BEG' : {'sec' : 'config', 'alt' : 'PB2NC_TIME_SUMMARY_BEG'}, + 'TIME_SUMMARY_END' : {'sec' : 'config', 'alt' : 'PB2NC_TIME_SUMMARY_END'}, + 'TIME_SUMMARY_VAR_NAMES' : {'sec' : 'config', 'alt' : 'PB2NC_TIME_SUMMARY_VAR_NAMES'}, + 'TIME_SUMMARY_TYPE' : {'sec' : 'config', 'alt' : 'PB2NC_TIME_SUMMARY_TYPE'}, + 'OVERWRITE_NC_OUTPUT' : {'sec' : 'config', 'alt' : 'PB2NC_SKIP_IF_OUTPUT_EXISTS', 'copy': False}, + 'VERTICAL_LOCATION' : {'sec' : 'config', 'alt' : 'PB2NC_VERTICAL_LOCATION'}, + 'VERIFICATION_GRID' : {'sec' : 'config', 'alt' : 'REGRID_DATA_PLANE_VERIF_GRID'}, + 'WINDOW_RANGE_BEG' : {'sec' : 'config', 'alt' : 'OBS_WINDOW_BEGIN'}, + 'WINDOW_RANGE_END' : {'sec' : 'config', 'alt' : 'OBS_WINDOW_END'}, + 'OBS_EXACT_VALID_TIME' : + {'sec' : 'config', 'alt' : 'OBS_WINDOW_BEGIN and OBS_WINDOW_END', 'copy': False}, + 'FCST_EXACT_VALID_TIME' : + {'sec' : 'config', 'alt' : 'FCST_WINDOW_BEGIN and FCST_WINDOW_END', 'copy': False}, + 'PCP_COMBINE_METHOD' : + {'sec' : 'config', 'alt' : 'FCST_PCP_COMBINE_METHOD and/or OBS_PCP_COMBINE_METHOD', 'copy': False}, + 'FHR_BEG' : {'sec' : 'config', 'alt' : 'LEAD_SEQ', 'copy': False}, + 'FHR_END' : {'sec' : 'config', 'alt' : 'LEAD_SEQ', 'copy': False}, + 'FHR_INC' : {'sec' : 'config', 'alt' : 'LEAD_SEQ', 'copy': False}, + 'FHR_GROUP_BEG' : {'sec' : 'config', 'alt' : 'LEAD_SEQ_[N]', 'copy': False}, + 'FHR_GROUP_END' : {'sec' : 'config', 'alt' : 'LEAD_SEQ_[N]', 'copy': False}, + 'FHR_GROUP_LABELS' : {'sec' : 'config', 'alt' : 'LEAD_SEQ_[N]_LABEL', 'copy': False}, + 'CYCLONE_OUT_DIR' : {'sec' : 'dir', 'alt' : 'CYCLONE_OUTPUT_DIR'}, + 'ENSEMBLE_STAT_OUT_DIR' : {'sec' : 'dir', 'alt' : 'ENSEMBLE_STAT_OUTPUT_DIR'}, + 'EXTRACT_OUT_DIR' : {'sec' : 'dir', 'alt' : 'EXTRACT_TILES_OUTPUT_DIR'}, + 'GRID_STAT_OUT_DIR' : {'sec' : 'dir', 'alt' : 'GRID_STAT_OUTPUT_DIR'}, + 'MODE_OUT_DIR' : {'sec' : 'dir', 'alt' : 'MODE_OUTPUT_DIR'}, + 'MTD_OUT_DIR' : {'sec' : 'dir', 'alt' : 'MTD_OUTPUT_DIR'}, + 'SERIES_INIT_OUT_DIR' : {'sec' : 'dir', 'alt' : 'SERIES_ANALYSIS_OUTPUT_DIR'}, + 'SERIES_LEAD_OUT_DIR' : {'sec' : 'dir', 'alt' : 'SERIES_ANALYSIS_OUTPUT_DIR'}, + 'SERIES_INIT_FILTERED_OUT_DIR' : + {'sec' : 'dir', 'alt' : 'SERIES_ANALYSIS_FILTERED_OUTPUT_DIR'}, + 'SERIES_LEAD_FILTERED_OUT_DIR' : + {'sec' : 'dir', 'alt' : 'SERIES_ANALYSIS_FILTERED_OUTPUT_DIR'}, + 'STAT_ANALYSIS_OUT_DIR' : + {'sec' : 'dir', 'alt' : 'STAT_ANALYSIS_OUTPUT_DIR'}, + 'TCMPR_PLOT_OUT_DIR' : {'sec' : 'dir', 'alt' : 'TCMPR_PLOT_OUTPUT_DIR'}, + 'FCST_MIN_FORECAST' : {'sec' : 'config', 'alt' : 'LEAD_SEQ_MIN'}, + 'FCST_MAX_FORECAST' : {'sec' : 'config', 'alt' : 'LEAD_SEQ_MAX'}, + 'OBS_MIN_FORECAST' : {'sec' : 'config', 'alt' : 'OBS_PCP_COMBINE_MIN_LEAD'}, + 'OBS_MAX_FORECAST' : {'sec' : 'config', 'alt' : 'OBS_PCP_COMBINE_MAX_LEAD'}, + 'FCST_INIT_INTERVAL' : {'sec' : 'config', 'alt' : None}, + 'OBS_INIT_INTERVAL' : {'sec' : 'config', 'alt' : None}, + 'FCST_DATA_INTERVAL' : {'sec' : '', 'alt' : 'FCST_PCP_COMBINE_DATA_INTERVAL'}, + 'OBS_DATA_INTERVAL' : {'sec' : '', 'alt' : 'OBS_PCP_COMBINE_DATA_INTERVAL'}, + 'FCST_IS_DAILY_FILE' : {'sec' : '', 'alt' : 'FCST_PCP_COMBINE_IS_DAILY_FILE'}, + 'OBS_IS_DAILY_FILE' : {'sec' : '', 'alt' : 'OBS_PCP_COMBINE_IS_DAILY_FILE'}, + 'FCST_TIMES_PER_FILE' : {'sec' : '', 'alt' : 'FCST_PCP_COMBINE_TIMES_PER_FILE'}, + 'OBS_TIMES_PER_FILE' : {'sec' : '', 'alt' : 'OBS_PCP_COMBINE_TIMES_PER_FILE'}, + 'FCST_LEVEL' : {'sec' : '', 'alt' : 'FCST_PCP_COMBINE_INPUT_ACCUMS', 'copy': False}, + 'OBS_LEVEL' : {'sec' : '', 'alt' : 'OBS_PCP_COMBINE_INPUT_ACCUMS', 'copy': False}, + 'MODE_FCST_CONV_RADIUS' : {'sec' : 'config', 'alt' : 'FCST_MODE_CONV_RADIUS'}, + 'MODE_FCST_CONV_THRESH' : {'sec' : 'config', 'alt' : 'FCST_MODE_CONV_THRESH'}, + 'MODE_FCST_MERGE_FLAG' : {'sec' : 'config', 'alt' : 'FCST_MODE_MERGE_FLAG'}, + 'MODE_FCST_MERGE_THRESH' : {'sec' : 'config', 'alt' : 'FCST_MODE_MERGE_THRESH'}, + 'MODE_OBS_CONV_RADIUS' : {'sec' : 'config', 'alt' : 'OBS_MODE_CONV_RADIUS'}, + 'MODE_OBS_CONV_THRESH' : {'sec' : 'config', 'alt' : 'OBS_MODE_CONV_THRESH'}, + 'MODE_OBS_MERGE_FLAG' : {'sec' : 'config', 'alt' : 'OBS_MODE_MERGE_FLAG'}, + 'MODE_OBS_MERGE_THRESH' : {'sec' : 'config', 'alt' : 'OBS_MODE_MERGE_THRESH'}, + 'MTD_FCST_CONV_RADIUS' : {'sec' : 'config', 'alt' : 'FCST_MTD_CONV_RADIUS'}, + 'MTD_FCST_CONV_THRESH' : {'sec' : 'config', 'alt' : 'FCST_MTD_CONV_THRESH'}, + 'MTD_OBS_CONV_RADIUS' : {'sec' : 'config', 'alt' : 'OBS_MTD_CONV_RADIUS'}, + 'MTD_OBS_CONV_THRESH' : {'sec' : 'config', 'alt' : 'OBS_MTD_CONV_THRESH'}, + 'RM_EXE' : {'sec' : 'exe', 'alt' : 'RM'}, + 'CUT_EXE' : {'sec' : 'exe', 'alt' : 'CUT'}, + 'TR_EXE' : {'sec' : 'exe', 'alt' : 'TR'}, + 'NCAP2_EXE' : {'sec' : 'exe', 'alt' : 'NCAP2'}, + 'CONVERT_EXE' : {'sec' : 'exe', 'alt' : 'CONVERT'}, + 'NCDUMP_EXE' : {'sec' : 'exe', 'alt' : 'NCDUMP'}, + 'EGREP_EXE' : {'sec' : 'exe', 'alt' : 'EGREP'}, + 'ADECK_TRACK_DATA_DIR' : {'sec' : 'dir', 'alt' : 'TC_PAIRS_ADECK_INPUT_DIR'}, + 'BDECK_TRACK_DATA_DIR' : {'sec' : 'dir', 'alt' : 'TC_PAIRS_BDECK_INPUT_DIR'}, + 'MISSING_VAL_TO_REPLACE' : {'sec' : 'config', 'alt' : 'TC_PAIRS_MISSING_VAL_TO_REPLACE'}, + 'MISSING_VAL' : {'sec' : 'config', 'alt' : 'TC_PAIRS_MISSING_VAL'}, + 'TRACK_DATA_SUBDIR_MOD' : {'sec' : 'dir', 'alt' : None}, + 'ADECK_FILE_PREFIX' : {'sec' : 'config', 'alt' : 'TC_PAIRS_ADECK_TEMPLATE', 'copy': False}, + 'BDECK_FILE_PREFIX' : {'sec' : 'config', 'alt' : 'TC_PAIRS_BDECK_TEMPLATE', 'copy': False}, + 'TOP_LEVEL_DIRS' : {'sec' : 'config', 'alt' : 'TC_PAIRS_READ_ALL_FILES'}, + 'TC_PAIRS_DIR' : {'sec' : 'dir', 'alt' : 'TC_PAIRS_OUTPUT_DIR'}, + 'CYCLONE' : {'sec' : 'config', 'alt' : 'TC_PAIRS_CYCLONE'}, + 'STORM_ID' : {'sec' : 'config', 'alt' : 'TC_PAIRS_STORM_ID'}, + 'BASIN' : {'sec' : 'config', 'alt' : 'TC_PAIRS_BASIN'}, + 'STORM_NAME' : {'sec' : 'config', 'alt' : 'TC_PAIRS_STORM_NAME'}, + 'DLAND_FILE' : {'sec' : 'config', 'alt' : 'TC_PAIRS_DLAND_FILE'}, + 'TRACK_TYPE' : {'sec' : 'config', 'alt' : 'TC_PAIRS_REFORMAT_DECK'}, + 'FORECAST_TMPL' : {'sec' : 'filename_templates', 'alt' : 'TC_PAIRS_ADECK_TEMPLATE'}, + 'REFERENCE_TMPL' : {'sec' : 'filename_templates', 'alt' : 'TC_PAIRS_BDECK_TEMPLATE'}, + 'TRACK_DATA_MOD_FORCE_OVERWRITE' : + {'sec' : 'config', 'alt' : 'TC_PAIRS_SKIP_IF_REFORMAT_EXISTS', 'copy': False}, + 'TC_PAIRS_FORCE_OVERWRITE' : {'sec' : 'config', 'alt' : 'TC_PAIRS_SKIP_IF_OUTPUT_EXISTS', 'copy': False}, + 'GRID_STAT_CONFIG' : {'sec' : 'config', 'alt' : 'GRID_STAT_CONFIG_FILE'}, + 'MODE_CONFIG' : {'sec' : 'config', 'alt': 'MODE_CONFIG_FILE'}, + 'FCST_PCP_COMBINE_INPUT_LEVEL': {'sec': 'config', 'alt' : 'FCST_PCP_COMBINE_INPUT_ACCUMS'}, + 'OBS_PCP_COMBINE_INPUT_LEVEL': {'sec': 'config', 'alt' : 'OBS_PCP_COMBINE_INPUT_ACCUMS'}, + 'TIME_METHOD': {'sec': 'config', 'alt': 'LOOP_BY', 'copy': False}, + 'MODEL_DATA_DIR': {'sec': 'dir', 'alt': 'EXTRACT_TILES_GRID_INPUT_DIR'}, + 'STAT_LIST': {'sec': 'config', 'alt': 'SERIES_ANALYSIS_STAT_LIST'}, + 'NLAT': {'sec': 'config', 'alt': 'EXTRACT_TILES_NLAT'}, + 'NLON': {'sec': 'config', 'alt': 'EXTRACT_TILES_NLON'}, + 'DLAT': {'sec': 'config', 'alt': 'EXTRACT_TILES_DLAT'}, + 'DLON': {'sec': 'config', 'alt': 'EXTRACT_TILES_DLON'}, + 'LON_ADJ': {'sec': 'config', 'alt': 'EXTRACT_TILES_LON_ADJ'}, + 'LAT_ADJ': {'sec': 'config', 'alt': 'EXTRACT_TILES_LAT_ADJ'}, + 'OVERWRITE_TRACK': {'sec': 'config', 'alt': 'EXTRACT_TILES_OVERWRITE_TRACK'}, + 'BACKGROUND_MAP': {'sec': 'config', 'alt': 'SERIES_ANALYSIS_BACKGROUND_MAP'}, + 'GFS_FCST_FILE_TMPL': {'sec': 'filename_templates', 'alt': 'FCST_EXTRACT_TILES_INPUT_TEMPLATE'}, + 'GFS_ANLY_FILE_TMPL': {'sec': 'filename_templates', 'alt': 'OBS_EXTRACT_TILES_INPUT_TEMPLATE'}, + 'SERIES_BY_LEAD_FILTERED_OUTPUT_DIR': {'sec': 'dir', 'alt': 'SERIES_ANALYSIS_FILTERED_OUTPUT_DIR'}, + 'SERIES_BY_INIT_FILTERED_OUTPUT_DIR': {'sec': 'dir', 'alt': 'SERIES_ANALYSIS_FILTERED_OUTPUT_DIR'}, + 'SERIES_BY_LEAD_OUTPUT_DIR': {'sec': 'dir', 'alt': 'SERIES_ANALYSIS_OUTPUT_DIR'}, + 'SERIES_BY_INIT_OUTPUT_DIR': {'sec': 'dir', 'alt': 'SERIES_ANALYSIS_OUTPUT_DIR'}, + 'SERIES_BY_LEAD_GROUP_FCSTS': {'sec': 'config', 'alt': 'SERIES_ANALYSIS_GROUP_FCSTS'}, + 'SERIES_ANALYSIS_BY_LEAD_CONFIG_FILE': {'sec': 'config', 'alt': 'SERIES_ANALYSIS_CONFIG_FILE'}, + 'SERIES_ANALYSIS_BY_INIT_CONFIG_FILE': {'sec': 'config', 'alt': 'SERIES_ANALYSIS_CONFIG_FILE'}, + 'ENSEMBLE_STAT_MET_OBS_ERROR_TABLE': {'sec': 'config', 'alt': 'ENSEMBLE_STAT_MET_OBS_ERR_TABLE'}, + 'VAR_LIST': {'sec': 'config', 'alt': 'BOTH_VAR_NAME BOTH_VAR_LEVELS or SERIES_ANALYSIS_VAR_LIST', 'copy': False}, + 'SERIES_ANALYSIS_VAR_LIST': {'sec': 'config', 'alt': 'BOTH_VAR_NAME BOTH_VAR_LEVELS', 'copy': False}, + 'EXTRACT_TILES_VAR_LIST': {'sec': 'config', 'alt': ''}, + 'STAT_ANALYSIS_LOOKIN_DIR': {'sec': 'dir', 'alt': 'MODEL1_STAT_ANALYSIS_LOOKIN_DIR'}, + 'VALID_HOUR_METHOD': {'sec': 'config', 'alt': None}, + 'VALID_HOUR_BEG': {'sec': 'config', 'alt': None}, + 'VALID_HOUR_END': {'sec': 'config', 'alt': None}, + 'VALID_HOUR_INCREMENT': {'sec': 'config', 'alt': None}, + 'INIT_HOUR_METHOD': {'sec': 'config', 'alt': None}, + 'INIT_HOUR_BEG': {'sec': 'config', 'alt': None}, + 'INIT_HOUR_END': {'sec': 'config', 'alt': None}, + 'INIT_HOUR_INCREMENT': {'sec': 'config', 'alt': None}, + 'STAT_ANALYSIS_CONFIG': {'sec': 'config', 'alt': 'STAT_ANALYSIS_CONFIG_FILE'}, + 'JOB_NAME': {'sec': 'config', 'alt': 'STAT_ANALYSIS_JOB_NAME'}, + 'JOB_ARGS': {'sec': 'config', 'alt': 'STAT_ANALYSIS_JOB_ARGS'}, + 'FCST_LEAD': {'sec': 'config', 'alt': 'FCST_LEAD_LIST'}, + 'FCST_VAR_NAME': {'sec': 'config', 'alt': 'FCST_VAR_LIST'}, + 'FCST_VAR_LEVEL': {'sec': 'config', 'alt': 'FCST_VAR_LEVEL_LIST'}, + 'OBS_VAR_NAME': {'sec': 'config', 'alt': 'OBS_VAR_LIST'}, + 'OBS_VAR_LEVEL': {'sec': 'config', 'alt': 'OBS_VAR_LEVEL_LIST'}, + 'REGION': {'sec': 'config', 'alt': 'VX_MASK_LIST'}, + 'INTERP': {'sec': 'config', 'alt': 'INTERP_LIST'}, + 'INTERP_PTS': {'sec': 'config', 'alt': 'INTERP_PTS_LIST'}, + 'CONV_THRESH': {'sec': 'config', 'alt': 'CONV_THRESH_LIST'}, + 'FCST_THRESH': {'sec': 'config', 'alt': 'FCST_THRESH_LIST'}, + 'LINE_TYPE': {'sec': 'config', 'alt': 'LINE_TYPE_LIST'}, + 'STAT_ANALYSIS_DUMP_ROW_TMPL': {'sec': 'filename_templates', 'alt': 'STAT_ANALYSIS_DUMP_ROW_TEMPLATE'}, + 'STAT_ANALYSIS_OUT_STAT_TMPL': {'sec': 'filename_templates', 'alt': 'STAT_ANALYSIS_OUT_STAT_TEMPLATE'}, + 'PLOTTING_SCRIPTS_DIR': {'sec': 'dir', 'alt': 'MAKE_PLOTS_SCRIPTS_DIR'}, + 'STAT_FILES_INPUT_DIR': {'sec': 'dir', 'alt': 'MAKE_PLOTS_INPUT_DIR'}, + 'PLOTTING_OUTPUT_DIR': {'sec': 'dir', 'alt': 'MAKE_PLOTS_OUTPUT_DIR'}, + 'VERIF_CASE': {'sec': 'config', 'alt': 'MAKE_PLOTS_VERIF_CASE'}, + 'VERIF_TYPE': {'sec': 'config', 'alt': 'MAKE_PLOTS_VERIF_TYPE'}, + 'PLOT_TIME': {'sec': 'config', 'alt': 'DATE_TIME'}, + 'MODEL_NAME': {'sec': 'config', 'alt': 'MODEL'}, + 'MODEL_OBS_NAME': {'sec': 'config', 'alt': 'MODEL_OBTYPE'}, + 'MODEL_STAT_DIR': {'sec': 'dir', 'alt': 'MODEL_STAT_ANALYSIS_LOOKIN_DIR'}, + 'MODEL_NAME_ON_PLOT': {'sec': 'config', 'alt': 'MODEL_REFERENCE_NAME'}, + 'REGION_LIST': {'sec': 'config', 'alt': 'VX_MASK_LIST'}, + 'PLOT_STATS_LIST': {'sec': 'config', 'alt': 'MAKE_PLOT_STATS_LIST'}, + 'CI_METHOD': {'sec': 'config', 'alt': 'MAKE_PLOTS_CI_METHOD'}, + 'VERIF_GRID': {'sec': 'config', 'alt': 'MAKE_PLOTS_VERIF_GRID'}, + 'EVENT_EQUALIZATION': {'sec': 'config', 'alt': 'MAKE_PLOTS_EVENT_EQUALIZATION'}, + 'MTD_CONFIG': {'sec': 'config', 'alt': 'MTD_CONFIG_FILE'}, + 'CLIMO_GRID_STAT_INPUT_DIR': {'sec': 'dir', 'alt': 'GRID_STAT_CLIMO_MEAN_INPUT_DIR'}, + 'CLIMO_GRID_STAT_INPUT_TEMPLATE': {'sec': 'filename_templates', 'alt': 'GRID_STAT_CLIMO_MEAN_INPUT_TEMPLATE'}, + 'CLIMO_POINT_STAT_INPUT_DIR': {'sec': 'dir', 'alt': 'POINT_STAT_CLIMO_MEAN_INPUT_DIR'}, + 'CLIMO_POINT_STAT_INPUT_TEMPLATE': {'sec': 'filename_templates', 'alt': 'POINT_STAT_CLIMO_MEAN_INPUT_TEMPLATE'}, + 'GEMPAKTOCF_CLASSPATH': {'sec': 'exe', 'alt': 'GEMPAKTOCF_JAR', 'copy': False}, + 'CUSTOM_INGEST__OUTPUT_DIR': {'sec': 'dir', 'alt': 'PY_EMBED_INGEST__OUTPUT_DIR'}, + 'CUSTOM_INGEST__OUTPUT_TEMPLATE': {'sec': 'filename_templates', 'alt': 'PY_EMBED_INGEST__OUTPUT_TEMPLATE'}, + 'CUSTOM_INGEST__OUTPUT_GRID': {'sec': 'config', 'alt': 'PY_EMBED_INGEST__OUTPUT_GRID'}, + 'CUSTOM_INGEST__SCRIPT': {'sec': 'config', 'alt': 'PY_EMBED_INGEST__SCRIPT'}, + 'CUSTOM_INGEST__TYPE': {'sec': 'config', 'alt': 'PY_EMBED_INGEST__TYPE'}, + 'TC_STAT_RUN_VIA': {'sec': 'config', 'alt': 'TC_STAT_CONFIG_FILE', + 'copy': False}, + 'TC_STAT_CMD_LINE_JOB': {'sec': 'config', 'alt': 'TC_STAT_JOB_ARGS'}, + 'TC_STAT_JOBS_LIST': {'sec': 'config', 'alt': 'TC_STAT_JOB_ARGS'}, + 'EXTRACT_TILES_OVERWRITE_TRACK': {'sec': 'config', + 'alt': 'EXTRACT_TILES_SKIP_IF_OUTPUT_EXISTS', + 'copy': False}, + 'EXTRACT_TILES_PAIRS_INPUT_DIR': {'sec': 'dir', + 'alt': 'EXTRACT_TILES_STAT_INPUT_DIR', + 'copy': False}, + 'EXTRACT_TILES_FILTERED_OUTPUT_TEMPLATE': {'sec': 'filename_template', + 'alt': 'EXTRACT_TILES_STAT_INPUT_TEMPLATE',}, + 'EXTRACT_TILES_GRID_INPUT_DIR': {'sec': 'dir', + 'alt': 'FCST_EXTRACT_TILES_INPUT_DIR' + 'and ' + 'OBS_EXTRACT_TILES_INPUT_DIR', + 'copy': False}, + 'SERIES_ANALYSIS_FILTER_OPTS': {'sec': 'config', + 'alt': 'TC_STAT_JOB_ARGS', + 'copy': False}, + 'SERIES_ANALYSIS_INPUT_DIR': {'sec': 'dir', + 'alt': 'FCST_SERIES_ANALYSIS_INPUT_DIR ' + 'and ' + 'OBS_SERIES_ANALYSIS_INPUT_DIR'}, + 'FCST_SERIES_ANALYSIS_TILE_INPUT_TEMPLATE': {'sec': 'filename_templates', + 'alt': 'FCST_SERIES_ANALYSIS_INPUT_TEMPLATE '}, + 'OBS_SERIES_ANALYSIS_TILE_INPUT_TEMPLATE': {'sec': 'filename_templates', + 'alt': 'OBS_SERIES_ANALYSIS_INPUT_TEMPLATE '}, + 'EXTRACT_TILES_STAT_INPUT_DIR': {'sec': 'dir', + 'alt': 'EXTRACT_TILES_TC_STAT_INPUT_DIR',}, + 'EXTRACT_TILES_STAT_INPUT_TEMPLATE': {'sec': 'filename_templates', + 'alt': 'EXTRACT_TILES_TC_STAT_INPUT_TEMPLATE',}, + 'SERIES_ANALYSIS_STAT_INPUT_DIR': {'sec': 'dir', + 'alt': 'SERIES_ANALYSIS_TC_STAT_INPUT_DIR', }, + 'SERIES_ANALYSIS_STAT_INPUT_TEMPLATE': {'sec': 'filename_templates', + 'alt': 'SERIES_ANALYSIS_TC_STAT_INPUT_TEMPLATE', }, + } + + # template '' : {'sec' : '', 'alt' : '', 'copy': True}, + + logger = config.logger + + # create list of errors and warnings to report for deprecated configs + e_list = [] + w_list = [] + all_sed_cmds = [] + + for old, depr_info in deprecated_dict.items(): + if isinstance(depr_info, dict): + + # check if is found in the old item, use regex to find variables if found + if '' in old: + old_regex = old.replace('', r'(\d+)') + indices = find_indices_in_config_section(old_regex, + config, + index_index=1).keys() + for index in indices: + old_with_index = old.replace('', index) + if depr_info['alt']: + alt_with_index = depr_info['alt'].replace('', index) + else: + alt_with_index = '' + + handle_deprecated(old_with_index, alt_with_index, depr_info, + config, all_sed_cmds, w_list, e_list) + else: + handle_deprecated(old, depr_info['alt'], depr_info, + config, all_sed_cmds, w_list, e_list) + + + # check all templates and error if any deprecated tags are used + # value of dict is replacement tag, set to None if no replacement exists + # deprecated tags: region (replace with basin) + deprecated_tags = {'region' : 'basin'} + template_vars = config.keys('config') + template_vars = [tvar for tvar in template_vars if tvar.endswith('_TEMPLATE')] + for temp_var in template_vars: + template = config.getraw('filename_templates', temp_var) + tags = get_tags(template) + + for depr_tag, replace_tag in deprecated_tags.items(): + if depr_tag in tags: + e_msg = 'Deprecated tag {{{}}} found in {}.'.format(depr_tag, + temp_var) + if replace_tag is not None: + e_msg += ' Replace with {{{}}}'.format(replace_tag) + + e_list.append(e_msg) + + # if any warning exist, report them + if w_list: + for warning_msg in w_list: + logger.warning(warning_msg) + + # if any errors exist, report them and exit + if e_list: + logger.error('DEPRECATED CONFIG ITEMS WERE FOUND. ' +\ + 'PLEASE REMOVE/REPLACE THEM FROM CONFIG FILES') + for error_msg in e_list: + logger.error(error_msg) + return False, all_sed_cmds + + return True, [] + +def check_for_deprecated_met_config(config): + sed_cmds = [] + all_good = True + + # set CURRENT_* METplus variables in case they are referenced in a + # METplus config variable and not already set + for fcst_or_obs in ['FCST', 'OBS']: + for name_or_level in ['NAME', 'LEVEL']: + current_var = f'CURRENT_{fcst_or_obs}_{name_or_level}' + if not config.has_option('config', current_var): + config.set('config', current_var, '') + + # check if *_CONFIG_FILE if set in the METplus config file and check for + # deprecated environment variables in those files + met_config_keys = [key for key in config.keys('config') + if key.endswith('CONFIG_FILE')] + + for met_config_key in met_config_keys: + met_tool = met_config_key.replace('_CONFIG_FILE', '') + + # get custom loop list to check if multiple config files are used based on the custom string + custom_list = get_custom_string_list(config, met_tool) + + for custom_string in custom_list: + met_config = config.getraw('config', met_config_key) + if not met_config: + continue + + met_config_file = do_string_sub(met_config, custom=custom_string) + + if not check_for_deprecated_met_config_file(config, met_config_file, sed_cmds, met_tool): + all_good = False + + return all_good, sed_cmds + +def check_for_deprecated_met_config_file(config, met_config, sed_cmds, met_tool): + + all_good = True + if not os.path.exists(met_config): + config.logger.error(f"Config file does not exist: {met_config}") + return False + + deprecated_met_list = ['MET_VALID_HHMM', 'GRID_VX', 'CONFIG_DIR'] + deprecated_output_prefix_list = ['FCST_VAR', 'OBS_VAR'] + config.logger.debug(f"Checking for deprecated environment variables in: {met_config}") + + with open(met_config, 'r') as file_handle: + lines = file_handle.read().splitlines() + + for line in lines: + for deprecated_item in deprecated_met_list: + if '${' + deprecated_item + '}' in line: + all_good = False + config.logger.error("Please remove deprecated environment variable " + f"${{{deprecated_item}}} found in MET config file: " + f"{met_config}") + + if deprecated_item == 'MET_VALID_HHMM' and 'file_name' in line: + config.logger.error(f"Set {met_tool}_CLIMO_MEAN_INPUT_[DIR/TEMPLATE] in a " + "METplus config file to set CLIMO_MEAN_FILE in a MET config") + new_line = " file_name = [ ${CLIMO_MEAN_FILE} ];" + + # escape [ and ] because they are special characters in sed commands + old_line = line.rstrip().replace('[', r'\[').replace(']', r'\]') + + sed_cmds.append(f"sed -i 's|^{old_line}|{new_line}|g' {met_config}") + add_line = f"{met_tool}_CLIMO_MEAN_INPUT_TEMPLATE" + sed_cmds.append(f"#Add {add_line}") + break + + if 'to_grid' in line: + config.logger.error("MET to_grid variable should reference " + "${REGRID_TO_GRID} environment variable") + new_line = " to_grid = ${REGRID_TO_GRID};" + + # escape [ and ] because they are special characters in sed commands + old_line = line.rstrip().replace('[', r'\[').replace(']', r'\]') + + sed_cmds.append(f"sed -i 's|^{old_line}|{new_line}|g' {met_config}") + config.logger.info(f"Be sure to set {met_tool}_REGRID_TO_GRID to the correct value.") + add_line = f"{met_tool}_REGRID_TO_GRID" + sed_cmds.append(f"#Add {add_line}") + break + + + for deprecated_item in deprecated_output_prefix_list: + # if deprecated item found in output prefix or to_grid line, replace line to use + # env var OUTPUT_PREFIX or REGRID_TO_GRID + if '${' + deprecated_item + '}' in line and 'output_prefix' in line: + config.logger.error("output_prefix variable should reference " + "${OUTPUT_PREFIX} environment variable") + new_line = "output_prefix = \"${OUTPUT_PREFIX}\";" + + # escape [ and ] because they are special characters in sed commands + old_line = line.rstrip().replace('[', r'\[').replace(']', r'\]') + + sed_cmds.append(f"sed -i 's|^{old_line}|{new_line}|g' {met_config}") + config.logger.info(f"You will need to add {met_tool}_OUTPUT_PREFIX to the METplus config file" + f" that sets {met_tool}_CONFIG_FILE. Set it to:") + output_prefix = _replace_output_prefix(line) + add_line = f"{met_tool}_OUTPUT_PREFIX = {output_prefix}" + config.logger.info(add_line) + sed_cmds.append(f"#Add {add_line}") + all_good = False + break + + return all_good + +def validate_field_info_configs(config, force_check=False): + """!Verify that config variables with _VAR_ in them are valid. Returns True if all are valid. + Returns False if any items are invalid""" + + variable_extensions = ['NAME', 'LEVELS', 'THRESH', 'OPTIONS'] + all_good = True, [] + + if skip_field_info_validation(config) and not force_check: + return True, [] + + # keep track of all sed commands to replace config variable names + all_sed_cmds = [] + + for ext in variable_extensions: + # find all _VAR_ keys in the conf files + data_types_and_indices = find_indices_in_config_section(r"(\w+)_VAR(\d+)_"+ext, + config, + index_index=2, + id_index=1) + + # if BOTH_VAR_ is used, set FCST and OBS to the same value + # if FCST or OBS is used, the other must be present as well + # if BOTH and either FCST or OBS are set, report an error + # get other data type + for index, data_type_list in data_types_and_indices.items(): + + is_valid, err_msgs, sed_cmds = is_var_item_valid(data_type_list, index, ext, config) + if not is_valid: + for err_msg in err_msgs: + config.logger.error(err_msg) + all_sed_cmds.extend(sed_cmds) + all_good = False + + # make sure FCST and OBS have the same number of levels if coming from separate variables + elif ext == 'LEVELS' and all(item in ['FCST', 'OBS'] for item in data_type_list): + fcst_levels = getlist(config.getraw('config', f"FCST_VAR{index}_LEVELS", '')) + + # add empty string if no levels are found because python embedding items do not need + # to include a level, but the other item may have a level and the numbers need to match + if not fcst_levels: + fcst_levels.append('') + + obs_levels = getlist(config.getraw('config', f"OBS_VAR{index}_LEVELS", '')) + if not obs_levels: + obs_levels.append('') + + if len(fcst_levels) != len(obs_levels): + config.logger.error(f"FCST_VAR{index}_LEVELS and OBS_VAR{index}_LEVELS do not have " + "the same number of elements") + all_good = False + + return all_good, all_sed_cmds + +def check_user_environment(config): + """!Check if any environment variables set in [user_env_vars] are already set in + the user's environment. Warn them that it will be overwritten from the conf if it is""" + if not config.has_section('user_env_vars'): + return + + for env_var in config.keys('user_env_vars'): + if env_var in os.environ: + msg = '{} is already set in the environment. '.format(env_var) +\ + 'Overwriting from conf file' + config.logger.warning(msg) + +def find_indices_in_config_section(regex, config, sec='config', + index_index=1, id_index=None): + """! Use regular expression to get all config variables that match and + are set in the user's configuration. This is used to handle config + variables that have multiple indices, i.e. FCST_VAR1_NAME, FCST_VAR2_NAME, + etc. + + @param regex regular expression to use to find variables + @param config METplusConfig object to search + @param sec (optional) config file section to search. Defaults to config + @param index_index 1 based number that is the regex match index for the + index number (default is 1) + @param id_index 1 based number that is the regex match index for the + identifier. Defaults to None which does not extract an indentifier + + number and the first match is used as an identifier + @returns dictionary where keys are the index number and the value is a + list of identifiers (if noID=True) or a list containing None + """ + # regex expression must have 2 () items and the 2nd item must be the index + all_conf = config.keys(sec) + indices = {} + regex = re.compile(regex) + for conf in all_conf: + result = regex.match(conf) + if result is not None: + index = result.group(index_index) + if id_index: + identifier = result.group(id_index) + else: + identifier = None + + if index not in indices: + indices[index] = [identifier] + else: + indices[index].append(identifier) + + return indices + +def handle_deprecated(old, alt, depr_info, config, all_sed_cmds, w_list, e_list): + sec = depr_info['sec'] + config_files = config.getstr('config', 'CONFIG_INPUT', '').split(',') + # if deprecated config item is found + if config.has_option(sec, old): + # if it is not required to remove, add to warning list + if 'req' in depr_info.keys() and depr_info['req'] is False: + msg = '[{}] {} is deprecated and will be '.format(sec, old) + \ + 'removed in a future version of METplus' + if alt: + msg += ". Please replace with {}".format(alt) + w_list.append(msg) + # if it is required to remove, add to error list + else: + if not alt: + e_list.append("[{}] {} should be removed".format(sec, old)) + else: + e_list.append("[{}] {} should be replaced with {}".format(sec, old, alt)) + + if 'copy' not in depr_info.keys() or depr_info['copy']: + for config_file in config_files: + all_sed_cmds.append(f"sed -i 's|^{old}|{alt}|g' {config_file}") + all_sed_cmds.append(f"sed -i 's|{{{old}}}|{{{alt}}}|g' {config_file}") + +def get_custom_string_list(config, met_tool): + var_name = 'CUSTOM_LOOP_LIST' + custom_loop_list = config.getstr_nocheck('config', + f'{met_tool.upper()}_{var_name}', + config.getstr_nocheck('config', + var_name, + '')) + custom_loop_list = getlist(custom_loop_list) + if not custom_loop_list: + custom_loop_list.append('') + + return custom_loop_list + +def _replace_output_prefix(line): + op_replacements = {'${MODEL}': '{MODEL}', + '${FCST_VAR}': '{CURRENT_FCST_NAME}', + '${OBTYPE}': '{OBTYPE}', + '${OBS_VAR}': '{CURRENT_OBS_NAME}', + '${LEVEL}': '{CURRENT_FCST_LEVEL}', + '${FCST_TIME}': '{lead?fmt=%3H}', + } + prefix = line.split('=')[1].strip().rstrip(';').strip('"') + for key, value, in op_replacements.items(): + prefix = prefix.replace(key, value) + + return prefix + +def parse_var_list(config, time_info=None, data_type=None, met_tool=None, + levels_as_list=False): + """ read conf items and populate list of dictionaries containing + information about each variable to be compared + + @param config: METplusConfig object + @param time_info: time object for string sub, optional + @param data_type: data type to find. Can be FCST, OBS, or ENS. + If not set, get FCST/OBS/BOTH + @param met_tool: optional name of MET tool to look for wrapper + specific var items + @param levels_as_list If true, store levels and output names as + a list instead of creating a field info dict for each name/level + @returns list of dictionaries with variable information + """ + + # validate configs again in case wrapper is not running from run_metplus + # this does not need to be done if parsing a specific data type, + # i.e. ENS or FCST + if data_type is None: + if not validate_field_info_configs(config)[0]: + return [] + elif data_type == 'BOTH': + config.logger.error("Cannot request BOTH explicitly in parse_var_list") + return [] + + # var_list is a list containing an list of dictionaries + var_list = [] + + # if specific data type is requested, only get that type + if data_type: + data_types = [data_type] + # otherwise get both FCST and OBS + else: + data_types = ['FCST', 'OBS'] + + # get indices of VAR items for data type and/or met tool + indices = [] + if met_tool: + indices = find_var_name_indices(config, data_types, met_tool).keys() + if not indices: + indices = find_var_name_indices(config, data_types).keys() + + # get config name prefixes for each data type to find + dt_search_prefixes = {} + for current_type in data_types: + # get list of variable prefixes to search + prefixes = get_field_search_prefixes(current_type, met_tool) + dt_search_prefixes[current_type] = prefixes + + # loop over all possible variables and add them to list + for index in indices: + field_info_list = [] + for current_type in data_types: + # get dictionary of existing config variables to use + search_prefixes = dt_search_prefixes[current_type] + field_configs = get_field_config_variables(config, + index, + search_prefixes) + + field_info = format_var_items(field_configs, time_info) + if not isinstance(field_info, dict): + config.logger.error(f'Could not process {current_type}_' + f'VAR{index} variables: {field_info}') + continue + + field_info['data_type'] = current_type.lower() + field_info_list.append(field_info) + + # check that all fields types were found + if not field_info_list or len(data_types) != len(field_info_list): + continue + + # check if number of levels for each field type matches + n_levels = len(field_info_list[0]['levels']) + if len(data_types) > 1: + if (n_levels != len(field_info_list[1]['levels'])): + continue + + # if requested, put all field levels in a single item + if levels_as_list: + var_dict = {} + for field_info in field_info_list: + current_type = field_info.get('data_type') + var_dict[f"{current_type}_name"] = field_info.get('name') + var_dict[f"{current_type}_level"] = field_info.get('levels') + var_dict[f"{current_type}_thresh"] = field_info.get('thresh') + var_dict[f"{current_type}_extra"] = field_info.get('extra') + var_dict[f"{current_type}_output_name"] = field_info.get('output_names') + + var_dict['index'] = index + var_list.append(var_dict) + continue + + # loop over levels and add all values to output dictionary + for level_index in range(n_levels): + var_dict = {} + + # get level values to use for string substitution in name + # used for python embedding calls that read the level value + sub_info = {} + for field_info in field_info_list: + dt_level = f"{field_info.get('data_type')}_level" + sub_info[dt_level] = field_info.get('levels')[level_index] + + for field_info in field_info_list: + current_type = field_info.get('data_type') + name = field_info.get('name') + level = field_info.get('levels')[level_index] + thresh = field_info.get('thresh') + extra = field_info.get('extra') + output_name = field_info.get('output_names')[level_index] + + # substitute level in name if filename template is specified + subbed_name = do_string_sub(name, + skip_missing_tags=True, + **sub_info) + + var_dict[f"{current_type}_name"] = subbed_name + var_dict[f"{current_type}_level"] = level + var_dict[f"{current_type}_thresh"] = thresh + var_dict[f"{current_type}_extra"] = extra + var_dict[f"{current_type}_output_name"] = output_name + + var_dict['index'] = index + var_list.append(var_dict) + + # extra debugging information used for developer debugging only + ''' + for v in var_list: + config.logger.debug(f"VAR{v['index']}:") + if 'fcst_name' in v.keys(): + config.logger.debug(" fcst_name:"+v['fcst_name']) + config.logger.debug(" fcst_level:"+v['fcst_level']) + if 'fcst_thresh' in v.keys(): + config.logger.debug(" fcst_thresh:"+str(v['fcst_thresh'])) + if 'fcst_extra' in v.keys(): + config.logger.debug(" fcst_extra:"+v['fcst_extra']) + if 'fcst_output_name' in v.keys(): + config.logger.debug(" fcst_output_name:"+v['fcst_output_name']) + if 'obs_name' in v.keys(): + config.logger.debug(" obs_name:"+v['obs_name']) + config.logger.debug(" obs_level:"+v['obs_level']) + if 'obs_thresh' in v.keys(): + config.logger.debug(" obs_thresh:"+str(v['obs_thresh'])) + if 'obs_extra' in v.keys(): + config.logger.debug(" obs_extra:"+v['obs_extra']) + if 'obs_output_name' in v.keys(): + config.logger.debug(" obs_output_name:"+v['obs_output_name']) + if 'ens_name' in v.keys(): + config.logger.debug(" ens_name:"+v['ens_name']) + config.logger.debug(" ens_level:"+v['ens_level']) + if 'ens_thresh' in v.keys(): + config.logger.debug(" ens_thresh:"+str(v['ens_thresh'])) + if 'ens_extra' in v.keys(): + config.logger.debug(" ens_extra:"+v['ens_extra']) + if 'ens_output_name' in v.keys(): + config.logger.debug(" ens_output_name:"+v['ens_output_name']) + ''' + return sorted(var_list, key=lambda x: x['index']) + +def find_var_name_indices(config, data_types, met_tool=None): + data_type_regex = f"{'|'.join(data_types)}" + + # if data_types includes FCST or OBS, also search for BOTH + if any([item for item in ['FCST', 'OBS'] if item in data_types]): + data_type_regex += '|BOTH' + + regex_string = f"({data_type_regex})" + + # if MET tool is specified, get tool specific items + if met_tool: + regex_string += f"_{met_tool.upper()}" + + regex_string += r"_VAR(\d+)_(NAME|INPUT_FIELD_NAME|FIELD_NAME)" + + # find all _VAR_NAME keys in the conf files + return find_indices_in_config_section(regex_string, + config, + index_index=2, + id_index=1) + +def skip_field_info_validation(config): + """!Check config to see if having corresponding FCST/OBS variables is necessary. If process list only + contains reformatter wrappers, don't validate field info. Also, if MTD is in the process list and + it is configured to only process either FCST or OBS, validation is unnecessary.""" + + reformatters = ['PCPCombine', 'RegridDataPlane'] + process_list = [item[0] for item in get_process_list(config)] + + # if running MTD in single mode, you don't need matching FCST/OBS + if 'MTD' in process_list and config.getbool('config', 'MTD_SINGLE_RUN'): + return True + + # if running any app other than the reformatters, you need matching FCST/OBS, so don't skip + if [item for item in process_list if item not in reformatters]: + return False + + return True + +def get_process_list(config): + """!Read process list, Extract instance string if specified inside + parenthesis. Remove dashes/underscores and change to lower case, + then map the name to the correct wrapper name + + @param config METplusConfig object to read PROCESS_LIST value + @returns list of tuple containing process name and instance identifier + (None if no instance was set) + """ + # get list of processes + process_list = getlist(config.getstr('config', 'PROCESS_LIST')) + + out_process_list = [] + # for each item remove dashes, underscores, and cast to lower-case + for process in process_list: + # if instance is specified, extract the text inside parenthesis + match = re.match(r'(.*)\((.*)\)', process) + if match: + instance = match.group(2) + process_name = match.group(1) + else: + instance = None + process_name = process + + wrapper_name = get_wrapper_name(process_name) + if wrapper_name is None: + config.logger.warning(f"PROCESS_LIST item {process_name} " + "may be invalid.") + wrapper_name = process_name + + # if MakePlots is in process list, remove it because + # it will be called directly from StatAnalysis + if wrapper_name == 'MakePlots': + continue + + out_process_list.append((wrapper_name, instance)) + + return out_process_list + +def get_field_search_prefixes(data_type, met_tool=None): + """! Get list of prefixes to search for field variables. + + @param data_type type of field to search for, i.e. FCST, OBS, ENS, etc. + Check for BOTH_ variables first only if data type is FCST or OBS + @param met_tool name of tool to search for variable or None if looking + for generic field info + @returns list of prefixes to search, i.e. [BOTH_, FCST_] or + [ENS_] or [BOTH_GRID_STAT_, OBS_GRID_STAT_] + """ + search_prefixes = [] + var_strings = [] + + # if met tool name is set, prioritize + # wrapper-specific configs before generic configs + if met_tool: + var_strings.append(f'{met_tool.upper()}_') + + var_strings.append('') + + for var_string in var_strings: + search_prefixes.append(f"{data_type}_{var_string}") + + # if looking for FCST or OBS, also check for BOTH prefix + if data_type in ['FCST', 'OBS']: + search_prefixes.append(f"BOTH_{var_string}") + + return search_prefixes + +def is_var_item_valid(item_list, index, ext, config): + """!Given a list of data types (FCST, OBS, ENS, or BOTH) check if the + combination is valid. + If BOTH is found, FCST and OBS should not be found. + If FCST or OBS is found, the other must also be found. + @param item_list list of data types that were found for a given index + @param index number following _VAR in the variable name + @param ext extension to check, i.e. NAME, LEVELS, THRESH, or OPTIONS + @param config METplusConfig instance + @returns tuple containing boolean if var item is valid, list of error + messages and list of sed commands to help the user update their old + configuration files + """ + + full_ext = f"_VAR{index}_{ext}" + msg = [] + sed_cmds = [] + if 'BOTH' in item_list and ('FCST' in item_list or 'OBS' in item_list): + + msg.append(f"Cannot set FCST{full_ext} or OBS{full_ext} if BOTH{full_ext} is set.") + elif ext == 'THRESH': + # allow thresholds unless BOTH and (FCST or OBS) are set + pass + + elif 'FCST' in item_list and 'OBS' not in item_list: + # if FCST level has 1 item and OBS name is a python embedding script, + # don't report error + level_list = getlist(config.getraw('config', + f'FCST_VAR{index}_LEVELS', + '')) + other_name = config.getraw('config', f'OBS_VAR{index}_NAME', '') + skip_error_for_py_embed = ext == 'LEVELS' and is_python_script(other_name) and len(level_list) == 1 + # do not report error for OPTIONS since it isn't required to be the same length + if ext not in ['OPTIONS'] and not skip_error_for_py_embed: + msg.append(f"If FCST{full_ext} is set, you must either set OBS{full_ext} or " + f"change FCST{full_ext} to BOTH{full_ext}") + + config_files = config.getstr('config', 'CONFIG_INPUT', '').split(',') + for config_file in config_files: + sed_cmds.append(f"sed -i 's|^FCST{full_ext}|BOTH{full_ext}|g' {config_file}") + sed_cmds.append(f"sed -i 's|{{FCST{full_ext}}}|{{BOTH{full_ext}}}|g' {config_file}") + + elif 'OBS' in item_list and 'FCST' not in item_list: + # if OBS level has 1 item and FCST name is a python embedding script, + # don't report error + level_list = getlist(config.getraw('config', + f'OBS_VAR{index}_LEVELS', + '')) + other_name = config.getraw('config', f'FCST_VAR{index}_NAME', '') + skip_error_for_py_embed = ext == 'LEVELS' and is_python_script(other_name) and len(level_list) == 1 + + if ext not in ['OPTIONS'] and not skip_error_for_py_embed: + msg.append(f"If OBS{full_ext} is set, you must either set FCST{full_ext} or " + f"change OBS{full_ext} to BOTH{full_ext}") + + config_files = config.getstr('config', 'CONFIG_INPUT', '').split(',') + for config_file in config_files: + sed_cmds.append(f"sed -i 's|^OBS{full_ext}|BOTH{full_ext}|g' {config_file}") + sed_cmds.append(f"sed -i 's|{{OBS{full_ext}}}|{{BOTH{full_ext}}}|g' {config_file}") + + return not bool(msg), msg, sed_cmds + +def get_field_config_variables(config, index, search_prefixes): + """! Search for variables that are set in the config that correspond to + the fields requested. Some field info items have + synonyms that can be used if the typical name is not set. This is used + in RegridDataPlane wrapper. + + @param config METplusConfig object to search + @param index of field (VAR) to find + @param search_prefixes list of valid prefixes to search for variables + in the config, i.e. FCST_VAR1_ or OBS_GRID_STAT_VAR2_ + @returns dictionary containing a config variable name to be used for + each field info value. If a valid config variable was not set for a + field info value, the value for that key will be set to None. + """ + # list of field info variables to find from config + # used as keys for dictionaries + field_info_items = ['name', + 'levels', + 'thresh', + 'options', + 'output_names', + ] + + field_configs = {} + search_suffixes = {} + + # initialize field configs dictionary values to None + # initialize dictionary of valid suffixes to search for with + # the capitalized version of field info name + for field_info_item in field_info_items: + field_configs[field_info_item] = None + search_suffixes[field_info_item] = [field_info_item.upper()] + + # add alternate suffixes for config variable names to attempt + search_suffixes['name'].append('INPUT_FIELD_NAME') + search_suffixes['name'].append('FIELD_NAME') + search_suffixes['levels'].append('INPUT_LEVEL') + search_suffixes['levels'].append('FIELD_LEVEL') + search_suffixes['output_names'].append('OUTPUT_FIELD_NAME') + search_suffixes['output_names'].append('FIELD_NAME') + + # look through field config keys and obtain highest priority + # variable name for each field config + for search_var, suffixes in search_suffixes.items(): + for prefix in search_prefixes: + + found = False + for suffix in suffixes: + var_name = f"{prefix}VAR{index}_{suffix}" + # if variable is found in config, + # get the value and break out of suffix loop + if config.has_option('config', var_name): + field_configs[search_var] = config.getraw('config', + var_name) + found = True + break + + # if config variable was found, break out of prefix loop + if found: + break + + return field_configs diff --git a/metplus/util/met_config.py b/metplus/util/met_config.py new file mode 100644 index 0000000000..103bf639ef --- /dev/null +++ b/metplus/util/met_config.py @@ -0,0 +1,700 @@ +""" +Program Name: met_config.py +Contact(s): George McCabe +""" + +import os + +from .met_util import getlist, get_threshold_via_regex, MISSING_DATA_VALUE +from .met_util import remove_quotes as util_remove_quotes +from .config_metplus import find_indices_in_config_section + +class METConfig: + """! Stores information for a member of a MET config variables that + can be used to set the value, the data type of the item, + optional name of environment variable to set (without METPLUS_ prefix) + if it differs from the name, + and any additional requirements such as remove quotes or make uppercase. + output_dict argument is ignored and only added to allow the argument + to the function that creates an instance of this object. + """ + def __init__(self, name, data_type, + env_var_name=None, + metplus_configs=None, + extra_args=None, + children=None, + output_dict=None): + self.name = name + self.data_type = data_type + self.metplus_configs = metplus_configs + self.extra_args = extra_args + self.env_var_name = env_var_name if env_var_name else name + self.children = children + + def __repr__(self): + return (f'{self.__class__.__name__}({self.name}, {self.data_type}, ' + f'{self.env_var_name}, ' + f'{self.metplus_configs}, ' + f'{self.extra_args}' + f', {self.children}' + ')') + + @property + def name(self): + return self._name + + @name.setter + def name(self, name): + if not isinstance(name, str): + raise TypeError("Name must be a string") + self._name = name + + @property + def data_type(self): + return self._data_type + + @data_type.setter + def data_type(self, data_type): + self._data_type = data_type + + @property + def env_var_name(self): + return self._env_var_name + + @env_var_name.setter + def env_var_name(self, env_var_name): + if not isinstance(env_var_name, str): + raise TypeError("Name must be a string") + self._env_var_name = env_var_name + + @property + def metplus_configs(self): + return self._metplus_configs + + @metplus_configs.setter + def metplus_configs(self, metplus_configs): + # convert to a list if input is a single value + config_names = metplus_configs + if config_names and not isinstance(config_names, list): + config_names = [config_names] + + self._metplus_configs = config_names + + @property + def extra_args(self): + return self._extra_args + + @extra_args.setter + def extra_args(self, extra_args): + args = extra_args if extra_args else {} + if not isinstance(args, dict): + raise TypeError("Expected a dictionary") + + self._extra_args = args + + @property + def children(self): + return self._children + + @children.setter + def children(self, children): + if not children and 'dict' in self.data_type: + raise TypeError("Must have children if data_type is dict.") + + if children: + if 'dict' not in self.data_type: + raise TypeError("data_type must be dict to have " + f"children. data_type is {self.data_type}") + + self._children = children + +def get_wrapped_met_config_file(config, app_name, default_config_file=None): + """! Get the MET config file path for the wrapper from the + METplusConfig object. If unset, use the default value if provided. + + @param default_config_file (optional) filename of wrapped MET config + file found in parm/met_config to use if config file is not set + @returns path to wrapped config file or None if no default is provided + """ + config_name = f'{app_name.upper()}_CONFIG_FILE' + config_file = config.getraw('config', config_name, '') + if config_file: + return config_file + + if not default_config_file: + return None + + default_config_path = os.path.join(config.getdir('PARM_BASE'), + 'met_config', + default_config_file) + config.logger.debug(f"{config_name} is not set. " + f"Using {default_config_path}") + return default_config_path + +def add_met_config_dict(config, app_name, output_dict, dict_name, items): + """! Read config variables for MET config dictionary and set + env_var_dict with formatted values + + @params dict_name name of MET dictionary variable + @params items dictionary where the key is name of variable inside MET + dictionary and the value is info about the item (see parse_item_info + function for more information) + """ + dict_items = [] + + # config prefix i.e GRID_STAT_CLIMO_MEAN_ + metplus_prefix = f'{app_name}_{dict_name}_'.upper() + for name, item_info in items.items(): + data_type, extra, kids, nicknames = _parse_item_info(item_info) + + # config name i.e. GRID_STAT_CLIMO_MEAN_FILE_NAME + metplus_name = f'{metplus_prefix}{name.upper()}' + + # change (n) to _N i.e. distance_map.beta_value(n) + metplus_name = metplus_name.replace('(N)', '_N') + metplus_configs = [] + + if 'dict' not in data_type: + children = None + # if variable ends with _BEG, read _BEGIN first + if metplus_name.endswith('BEG'): + metplus_configs.append(f'{metplus_name}IN') + + metplus_configs.append(metplus_name) + if nicknames: + for nickname in nicknames: + metplus_configs.append( + f'{app_name}_{nickname}'.upper() + ) + + # if dictionary, read get children from MET config + else: + children = [] + for kid_name, kid_info in kids.items(): + kid_upper = kid_name.upper() + kid_type, kid_extra, _, _ = _parse_item_info(kid_info) + + metplus_configs.append(f'{metplus_name}_{kid_upper}') + metplus_configs.append(f'{metplus_prefix}{kid_upper}') + + kid_args = _parse_extra_args(kid_extra) + child_item = METConfig( + name=kid_name, + data_type=kid_type, + metplus_configs=metplus_configs.copy(), + extra_args=kid_args, + ) + children.append(child_item) + + # reset metplus config list for next kid + metplus_configs.clear() + + # set metplus_configs + metplus_configs = None + + extra_args = _parse_extra_args(extra) + dict_item = ( + METConfig( + name=name, + data_type=data_type, + metplus_configs=metplus_configs, + extra_args=extra_args, + children=children, + ) + ) + dict_items.append(dict_item) + + final_met_config = METConfig( + name=dict_name, + data_type='dict', + children=dict_items, + ) + + return add_met_config_item(config, + final_met_config, + output_dict) + +def add_met_config_item(config, item, output_dict, depth=0): + """! Reads info from METConfig object, gets value from + METplusConfig, and formats it based on the specifications. Sets + value in output dictionary with key starting with METPLUS_. + + @param item METConfig object to read and determine what to get + @param output_dict dictionary to save formatted output + @param depth counter to check if item being processed is nested within + another variable or not. If depth is 0, it is a top level variable. + This is used internally by this function and shouldn't be supplied + outside of calls within this function. + """ + env_var_name = item.env_var_name.upper() + if not env_var_name.startswith('METPLUS_'): + env_var_name = f'METPLUS_{env_var_name}' + + # handle dictionary or dictionary list item + if 'dict' in item.data_type: + tmp_dict = {} + for child in item.children: + if not add_met_config_item(config, child, tmp_dict, + depth=depth+1): + return False + + dict_string = format_met_config(item.data_type, + tmp_dict, + item.name, + keys=None) + + # if handling dict MET config that is not nested inside another + if not depth and item.data_type == 'dict': + env_var_name = f'{env_var_name}_DICT' + + output_dict[env_var_name] = dict_string + return True + + # handle non-dictionary item + set_met_config = set_met_config_function(item.data_type) + if not set_met_config: + return False + + return set_met_config(config, + output_dict, + item.metplus_configs, + item.name, + c_dict_key=env_var_name, + **item.extra_args) + +def add_met_config_dict_list(config, app_name, output_dict, dict_name, + dict_items): + search_string = f'{app_name}_{dict_name}'.upper() + regex = r'^' + search_string + r'(\d+)_(\w+)$' + indices = find_indices_in_config_section(regex, config, + index_index=1, + id_index=2) + + all_met_config_items = {} + is_ok = True + for index, items in indices.items(): + # read all variables for each index + met_config_items = {} + + # check if any variable found doesn't match valid variables + not_in_dict = [item for item in items + if item.lower() not in dict_items] + if any(not_in_dict): + for item in not_in_dict: + config.logger.error("Invalid variable: " + f"{search_string}{index}_{item}") + is_ok = False + continue + + for name, item_info in dict_items.items(): + data_type, extra, kids, nicknames = _parse_item_info(item_info) + metplus_configs = [f'{search_string}{index}_{name.upper()}'] + extra_args = _parse_extra_args(extra) + item = METConfig(name=name, + data_type=data_type, + metplus_configs=metplus_configs, + extra_args=extra_args, + ) + + if not add_met_config_item(config, item, met_config_items): + is_ok = False + continue + + dict_string = format_met_config('dict', + met_config_items, + name='') + all_met_config_items[index] = dict_string + + # format list of dictionaries + output_string = format_met_config('list', + all_met_config_items, + dict_name) + output_dict[f'METPLUS_{dict_name.upper()}_LIST'] = output_string + return is_ok + +def format_met_config(data_type, c_dict, name, keys=None): + """! Return formatted variable named with any if they + are set to a value. If none of the items are set, return empty string + + @param data_type type of value to format + @param c_dict config dictionary to read values from + @param name name of dictionary to create + @param keys list of c_dict keys to use if they are set. If unset (None) + then read all keys from c_dict + @returns MET config formatted dictionary/list + if any items are set, or empty string if not + """ + values = [] + if keys is None: + keys = c_dict.keys() + + for key in keys: + value = c_dict.get(key) + if value: + values.append(str(value)) + + # if none of the keys are set to a value in dict, return empty string + if not values: + return '' + + output = ''.join(values) + # add curly braces if dictionary + if 'dict' in data_type: + output = f"{{{output}}}" + + # add square braces if list + if 'list' in data_type: + output = f"[{output}];" + + # if name is not empty, add variable name and equals sign + print(f"NAME IS X{name}X and OUTPUT:{output}") + if name: + output = f'{name} = {output}' + return output + +def set_met_config_function(item_type): + """! Return function to use based on item type + + @param item_type type of MET config variable to obtain + Valid values: list, string, int, float, thresh, bool + @returns function to use or None if invalid type provided + """ + if item_type == 'int': + return set_met_config_int + elif item_type == 'string': + return set_met_config_string + elif item_type == 'list': + return set_met_config_list + elif item_type == 'float': + return set_met_config_float + elif item_type == 'thresh': + return set_met_config_thresh + elif item_type == 'bool': + return set_met_config_bool + else: + raise ValueError(f"Invalid argument for item type: {item_type}") + +def _get_config_or_default(mp_config_name, get_function, + default=None): + conf_value = '' + + # if no possible METplus config variables are not set + if mp_config_name is None: + # if no default defined, return without doing anything + if not default: + return None + + # if METplus config variable is set, read the value + else: + conf_value = get_function('config', + mp_config_name, + '') + + # if variable is not set and there is a default defined, set default + if not conf_value and default: + conf_value = default + + return conf_value + +def set_met_config_list(config, c_dict, mp_config, met_config_name, + c_dict_key=None, **kwargs): + """! Get list from METplus configuration file and format it to be passed + into a MET configuration file. Set c_dict item with formatted string. + Args: + @param c_dict configuration dictionary to set + @param mp_config_name METplus configuration variable name. Assumed to be + in the [config] section. Value can be a comma-separated list of items. + @param met_config name of MET configuration variable to set. Also used + to determine the key in c_dict to set (upper-case) + @param c_dict_key optional argument to specify c_dict key to store result. If + set to None (default) then use upper-case of met_config_name + @param allow_empty if True, if METplus config variable is set + but is an empty string, then set the c_dict value to an empty + list. If False, behavior is the same as when the variable is + not set at all, which is to not set anything for the c_dict + value + @param remove_quotes if True, output value without quotes. + Default value is False + @param default (Optional) if set, use this value as default + if config is not set + """ + mp_config_name = config.get_mp_config_name(mp_config) + conf_value = _get_config_or_default( + mp_config_name, + get_function=config.getraw, + default=kwargs.get('default') + ) + if conf_value is None: + return True + + # convert value from config to a list + conf_values = getlist(conf_value) + if conf_values or kwargs.get('allow_empty', False): + out_values = [] + for conf_value in conf_values: + remove_quotes = kwargs.get('remove_quotes', False) + # if not removing quotes, escape any quotes found in list items + if not remove_quotes: + conf_value = conf_value.replace('"', '\\"') + + conf_value = util_remove_quotes(conf_value) + if not remove_quotes: + conf_value = f'"{conf_value}"' + + out_values.append(conf_value) + out_value = f"[{', '.join(out_values)}]" + + if not c_dict_key: + c_key = met_config_name.upper() + else: + c_key = c_dict_key + + if met_config_name: + out_value = f'{met_config_name} = {out_value};' + c_dict[c_key] = out_value + + return True + +def set_met_config_string(config, c_dict, mp_config, met_config_name, + c_dict_key=None, **kwargs): + """! Get string from METplus configuration file and format it to be passed + into a MET configuration file. Set c_dict item with formatted string. + + @param c_dict configuration dictionary to set + @param mp_config METplus configuration variable name. Assumed to be + in the [config] section. Value can be a comma-separated list of items. + @param met_config_name name of MET configuration variable to set. Also used + to determine the key in c_dict to set (upper-case) + @param c_dict_key optional argument to specify c_dict key to store result. If + set to None (default) then use upper-case of met_config_name + @param remove_quotes if True, output value without quotes. + Default value is False + @param to_grid if True, format to_grid value + Default value is False + @param default (Optional) if set, use this value as default + if config is not set + """ + mp_config_name = config.get_mp_config_name(mp_config) + conf_value = _get_config_or_default( + mp_config_name, + get_function=config.getraw, + default=kwargs.get('default') + ) + if not conf_value: + return True + + conf_value = util_remove_quotes(conf_value) + # add quotes back if remote quotes is False + if not kwargs.get('remove_quotes'): + conf_value = f'"{conf_value}"' + + if kwargs.get('uppercase', False): + conf_value = conf_value.upper() + + if kwargs.get('to_grid', False): + conf_value = format_regrid_to_grid(conf_value) + + c_key = c_dict_key if c_dict_key else met_config_name.upper() + if met_config_name: + conf_value = f'{met_config_name} = {conf_value};' + + c_dict[c_key] = conf_value + return True + +def set_met_config_number(config, c_dict, num_type, mp_config, + met_config_name, c_dict_key=None, **kwargs): + """! Get integer from METplus configuration file and format it to be passed + into a MET configuration file. Set c_dict item with formatted string. + Args: + @param c_dict configuration dictionary to set + @param num_type type of number to get from config. If set to 'int', call + getint function. If not, call getfloat function. + @param mp_config METplus configuration variable name. Assumed to be + in the [config] section. Value can be a comma-separated list of items. + @param met_config_name name of MET configuration variable to set. Also used + to determine the key in c_dict to set (upper-case) if c_dict_key is None + @param c_dict_key optional argument to specify c_dict key to store result. If + set to None (default) then use upper-case of met_config_name + @param default (Optional) if set, use this value as default + if config is not set + """ + mp_config_name = config.get_mp_config_name(mp_config) + if mp_config_name is None: + return True + + if num_type == 'int': + conf_value = config.getint('config', mp_config_name) + else: + conf_value = config.getfloat('config', mp_config_name) + + if conf_value is None: + return False + if conf_value != MISSING_DATA_VALUE: + if not c_dict_key: + c_key = met_config_name.upper() + else: + c_key = c_dict_key + + if met_config_name: + out_value = f"{met_config_name} = {str(conf_value)};" + else: + out_value = str(conf_value) + c_dict[c_key] = out_value + + return True + +def set_met_config_int(config, c_dict, mp_config_name, met_config_name, + c_dict_key=None, **kwargs): + return set_met_config_number(config, c_dict, 'int', + mp_config_name, + met_config_name, + c_dict_key=c_dict_key, + **kwargs) + +def set_met_config_float(config, c_dict, mp_config_name, + met_config_name, c_dict_key=None, **kwargs): + return set_met_config_number(config, c_dict, 'float', + mp_config_name, + met_config_name, + c_dict_key=c_dict_key, + **kwargs) + +def set_met_config_thresh(config, c_dict, mp_config, met_config_name, + c_dict_key=None, **kwargs): + mp_config_name = config.get_mp_config_name(mp_config) + if mp_config_name is None: + return True + + conf_value = config.getstr('config', mp_config_name, '') + if conf_value: + if get_threshold_via_regex(conf_value) is None: + config.logger.error(f"Incorrectly formatted threshold: {mp_config_name}") + return False + + if not c_dict_key: + c_key = met_config_name.upper() + else: + c_key = c_dict_key + + if met_config_name: + out_value = f"{met_config_name} = {str(conf_value)};" + else: + out_value = str(conf_value) + + c_dict[c_key] = out_value + return True + +def set_met_config_bool(config, c_dict, mp_config, met_config_name, + c_dict_key=None, **kwargs): + """! Get boolean from METplus configuration file and format it to be + passed into a MET configuration file. Set c_dict item with boolean + value expressed as a string. + Args: + @param c_dict configuration dictionary to set + @param mp_config METplus configuration variable name. + Assumed to be in the [config] section. + @param met_config_name name of MET configuration variable to + set. Also used to determine the key in c_dict to set + (upper-case) + @param c_dict_key optional argument to specify c_dict key to + store result. If set to None (default) then use upper-case of + met_config_name + @param uppercase If true, set value to TRUE or FALSE + """ + mp_config_name = config.get_mp_config_name(mp_config) + if mp_config_name is None: + return True + conf_value = config.getbool('config', mp_config_name, '') + if conf_value is None: + config.logger.error(f'Invalid boolean value set for {mp_config_name}') + return False + + # if not invalid but unset, return without setting c_dict with no error + if conf_value == '': + return True + + conf_value = str(conf_value) + if kwargs.get('uppercase', True): + conf_value = conf_value.upper() + + if not c_dict_key: + c_key = met_config_name.upper() + else: + c_key = c_dict_key + + conf_value = util_remove_quotes(conf_value) + if met_config_name: + conf_value = f'{met_config_name} = {conf_value};' + c_dict[c_key] = conf_value + return True + +def format_regrid_to_grid(to_grid): + to_grid = to_grid.strip('"') + if not to_grid: + to_grid = 'NONE' + + # if NONE, FCST, or OBS force uppercase, otherwise add quotes + if to_grid.upper() in ['NONE', 'FCST', 'OBS']: + to_grid = to_grid.upper() + else: + to_grid = f'"{to_grid}"' + + return to_grid + +def _parse_item_info(item_info): + """! Parses info about a MET config dictionary item. The input can + be a single string that is the data type of the item. It can also be + a tuple containing 2 to 4 values. The additional values must be + supplied in order: + * extra: string of extra information about item, i.e. + 'remove_quotes', 'uppercase', or 'allow_empty' + * kids: dictionary describing child values (used only for dict items) + where the key is the name of the variable and the value is item info + for the child variable in the same format as item_info that is + parsed in this function + * nicknames: list of other METplus config variable name that can be + used to set a value. The app name i.e. GRID_STAT_ is prepended to + each nickname in the list. Used for backwards compatibility for + METplus config variables whose name does not match the MET config + variable name + + @param item_info string or tuple containing information about a + dictionary item + @returns tuple of data type, extra info, children, and nicknames or + None for each tuple value that is not set + """ + if isinstance(item_info, tuple): + data_type, *rest = item_info + else: + data_type = item_info + rest = [] + + extra = rest.pop(0) if rest else None + kids = rest.pop(0) if rest else None + nicknames = rest.pop(0) if rest else None + + return data_type, extra, kids, nicknames + +def _parse_extra_args(extra): + """! Check string for extra option keywords and set them to True in + dictionary if they are found. Supports 'remove_quotes', 'uppercase' + and 'allow_empty' + + @param extra string to parse for keywords + @returns dictionary with extra args set if found in string + """ + extra_args = {} + if not extra: + return extra_args + + VALID_EXTRAS = ( + 'remove_quotes', + 'uppercase', + 'allow_empty', + 'to_grid', + 'default', + ) + for extra_option in VALID_EXTRAS: + if extra_option in extra: + extra_args[extra_option] = True + return extra_args diff --git a/metplus/util/met_dictionary_info.py b/metplus/util/met_dictionary_info.py deleted file mode 100644 index 2decdc0f22..0000000000 --- a/metplus/util/met_dictionary_info.py +++ /dev/null @@ -1,110 +0,0 @@ -""" -Program Name: met_dictionary_info.py -Contact(s): George McCabe -Abstract: -History Log: Initial version -Usage: -Parameters: None -Input Files: N/A -Output Files: N/A -""" - - -class METConfigInfo: - """! Stores information for a member of a MET config variables that - can be used to set the value, the data type of the item, - optional name of environment variable to set (without METPLUS_ prefix) - if it differs from the name, - and any additional requirements such as remove quotes or make uppercase. - output_dict argument is ignored and only added to allow the argument - to the function that creates an instance of this object. - """ - def __init__(self, name, data_type, - env_var_name=None, - metplus_configs=None, - extra_args=None, - children=None, - output_dict=None): - self.name = name - self.data_type = data_type - self.metplus_configs = metplus_configs - self.extra_args = extra_args - self.env_var_name = env_var_name if env_var_name else name - self.children = children - - def __repr__(self): - return (f'{self.__class__.__name__}({self.name}, {self.data_type}, ' - f'{self.env_var_name}, ' - f'{self.metplus_configs}, ' - f'{self.extra_args}' - f', {self.children}' - ')') - - @property - def name(self): - return self._name - - @name.setter - def name(self, name): - if not isinstance(name, str): - raise TypeError("Name must be a string") - self._name = name - - @property - def data_type(self): - return self._data_type - - @data_type.setter - def data_type(self, data_type): - self._data_type = data_type - - @property - def env_var_name(self): - return self._env_var_name - - @env_var_name.setter - def env_var_name(self, env_var_name): - if not isinstance(env_var_name, str): - raise TypeError("Name must be a string") - self._env_var_name = env_var_name - - @property - def metplus_configs(self): - return self._metplus_configs - - @metplus_configs.setter - def metplus_configs(self, metplus_configs): - # convert to a list if input is a single value - config_names = metplus_configs - if config_names and not isinstance(config_names, list): - config_names = [config_names] - - self._metplus_configs = config_names - - @property - def extra_args(self): - return self._extra_args - - @extra_args.setter - def extra_args(self, extra_args): - args = extra_args if extra_args else {} - if not isinstance(args, dict): - raise TypeError("Expected a dictionary") - - self._extra_args = args - - @property - def children(self): - return self._children - - @children.setter - def children(self, children): - if not children and 'dict' in self.data_type: - raise TypeError("Must have children if data_type is dict.") - - if children: - if 'dict' not in self.data_type: - raise TypeError("data_type must be dict to have " - f"children. data_type is {self.data_type}") - - self._children = children diff --git a/metplus/util/met_util.py b/metplus/util/met_util.py index 105740e420..493312daab 100644 --- a/metplus/util/met_util.py +++ b/metplus/util/met_util.py @@ -25,10 +25,7 @@ from .string_template_substitution import do_string_sub from .string_template_substitution import parse_template -from .string_template_substitution import get_tags from . import time_util as time_util -from .doc_util import get_wrapper_name - from .. import get_metplus_version """!@namespace met_util @@ -75,7 +72,7 @@ def pre_run_setup(config_inputs): logger.info(f"Config Input: {config_item}") # validate configuration variables - isOK_A, isOK_B, isOK_C, isOK_D, all_sed_cmds = validate_configuration_variables(config) + isOK_A, isOK_B, isOK_C, isOK_D, all_sed_cmds = config_metplus.validate_configuration_variables(config) if not (isOK_A and isOK_B and isOK_C and isOK_D): # if any sed commands were generated, write them to the sed file if all_sed_cmds: @@ -238,493 +235,6 @@ def write_all_commands(all_commands, config): file_handle.write("COMMAND:\n") file_handle.write(f"{command}\n\n") -def check_for_deprecated_config(config): - """!Checks user configuration files and reports errors or warnings if any deprecated variable - is found. If an alternate variable name can be suggested, add it to the 'alt' section - If the alternate cannot be literally substituted for the old name, set copy to False - Args: - @config : METplusConfig object to evaluate - Returns: - A tuple containing a boolean if the configuration is suitable to run or not and - if it is not correct, the 2nd item is a list of sed commands that can be run to help - fix the incorrect configuration variables - """ - - # key is the name of the depreacted variable that is no longer allowed in any config files - # value is a dictionary containing information about what to do with the deprecated config - # 'sec' is the section of the config file where the replacement resides, i.e. config, dir, - # filename_templates - # 'alt' is the alternative name for the deprecated config. this can be a single variable name or - # text to describe multiple variables or how to handle it. Set to None to tell the user to - # just remove the variable - # 'copy' is an optional item (defaults to True). set this to False if one cannot simply replace - # the deprecated config variable name with the value in 'alt' - # 'req' is an optional item (defaults to True). this to False to report a warning for the - # deprecated config and allow execution to continue. this is generally no longer used - # because we are requiring users to update the config files. if used, the developer must - # modify the code to handle both variables accordingly - deprecated_dict = { - 'LOOP_BY_INIT' : {'sec' : 'config', 'alt' : 'LOOP_BY', 'copy': False}, - 'LOOP_METHOD' : {'sec' : 'config', 'alt' : 'LOOP_ORDER'}, - 'PREPBUFR_DIR_REGEX' : {'sec' : 'regex_pattern', 'alt' : None}, - 'PREPBUFR_FILE_REGEX' : {'sec' : 'regex_pattern', 'alt' : None}, - 'OBS_INPUT_DIR_REGEX' : {'sec' : 'regex_pattern', 'alt' : 'OBS_POINT_STAT_INPUT_DIR', 'copy': False}, - 'FCST_INPUT_DIR_REGEX' : {'sec' : 'regex_pattern', 'alt' : 'FCST_POINT_STAT_INPUT_DIR', 'copy': False}, - 'FCST_INPUT_FILE_REGEX' : - {'sec' : 'regex_pattern', 'alt' : 'FCST_POINT_STAT_INPUT_TEMPLATE', 'copy': False}, - 'OBS_INPUT_FILE_REGEX' : {'sec' : 'regex_pattern', 'alt' : 'OBS_POINT_STAT_INPUT_TEMPLATE', 'copy': False}, - 'PREPBUFR_DATA_DIR' : {'sec' : 'dir', 'alt' : 'PB2NC_INPUT_DIR'}, - 'PREPBUFR_MODEL_DIR_NAME' : {'sec' : 'dir', 'alt' : 'PB2NC_INPUT_DIR', 'copy': False}, - 'OBS_INPUT_FILE_TMPL' : - {'sec' : 'filename_templates', 'alt' : 'OBS_POINT_STAT_INPUT_TEMPLATE'}, - 'FCST_INPUT_FILE_TMPL' : - {'sec' : 'filename_templates', 'alt' : 'FCST_POINT_STAT_INPUT_TEMPLATE'}, - 'NC_FILE_TMPL' : {'sec' : 'filename_templates', 'alt' : 'PB2NC_OUTPUT_TEMPLATE'}, - 'FCST_INPUT_DIR' : {'sec' : 'dir', 'alt' : 'FCST_POINT_STAT_INPUT_DIR'}, - 'OBS_INPUT_DIR' : {'sec' : 'dir', 'alt' : 'OBS_POINT_STAT_INPUT_DIR'}, - 'REGRID_TO_GRID' : {'sec' : 'config', 'alt' : 'POINT_STAT_REGRID_TO_GRID'}, - 'FCST_HR_START' : {'sec' : 'config', 'alt' : 'LEAD_SEQ', 'copy': False}, - 'FCST_HR_END' : {'sec' : 'config', 'alt' : 'LEAD_SEQ', 'copy': False}, - 'FCST_HR_INTERVAL' : {'sec' : 'config', 'alt' : 'LEAD_SEQ', 'copy': False}, - 'START_DATE' : {'sec' : 'config', 'alt' : 'INIT_BEG or VALID_BEG', 'copy': False}, - 'END_DATE' : {'sec' : 'config', 'alt' : 'INIT_END or VALID_END', 'copy': False}, - 'INTERVAL_TIME' : {'sec' : 'config', 'alt' : 'INIT_INCREMENT or VALID_INCREMENT', 'copy': False}, - 'BEG_TIME' : {'sec' : 'config', 'alt' : 'INIT_BEG or VALID_BEG', 'copy': False}, - 'END_TIME' : {'sec' : 'config', 'alt' : 'INIT_END or VALID_END', 'copy': False}, - 'START_HOUR' : {'sec' : 'config', 'alt' : 'INIT_BEG or VALID_BEG', 'copy': False}, - 'END_HOUR' : {'sec' : 'config', 'alt' : 'INIT_END or VALID_END', 'copy': False}, - 'OBS_BUFR_VAR_LIST' : {'sec' : 'config', 'alt' : 'PB2NC_OBS_BUFR_VAR_LIST'}, - 'TIME_SUMMARY_FLAG' : {'sec' : 'config', 'alt' : 'PB2NC_TIME_SUMMARY_FLAG'}, - 'TIME_SUMMARY_BEG' : {'sec' : 'config', 'alt' : 'PB2NC_TIME_SUMMARY_BEG'}, - 'TIME_SUMMARY_END' : {'sec' : 'config', 'alt' : 'PB2NC_TIME_SUMMARY_END'}, - 'TIME_SUMMARY_VAR_NAMES' : {'sec' : 'config', 'alt' : 'PB2NC_TIME_SUMMARY_VAR_NAMES'}, - 'TIME_SUMMARY_TYPE' : {'sec' : 'config', 'alt' : 'PB2NC_TIME_SUMMARY_TYPE'}, - 'OVERWRITE_NC_OUTPUT' : {'sec' : 'config', 'alt' : 'PB2NC_SKIP_IF_OUTPUT_EXISTS', 'copy': False}, - 'VERTICAL_LOCATION' : {'sec' : 'config', 'alt' : 'PB2NC_VERTICAL_LOCATION'}, - 'VERIFICATION_GRID' : {'sec' : 'config', 'alt' : 'REGRID_DATA_PLANE_VERIF_GRID'}, - 'WINDOW_RANGE_BEG' : {'sec' : 'config', 'alt' : 'OBS_WINDOW_BEGIN'}, - 'WINDOW_RANGE_END' : {'sec' : 'config', 'alt' : 'OBS_WINDOW_END'}, - 'OBS_EXACT_VALID_TIME' : - {'sec' : 'config', 'alt' : 'OBS_WINDOW_BEGIN and OBS_WINDOW_END', 'copy': False}, - 'FCST_EXACT_VALID_TIME' : - {'sec' : 'config', 'alt' : 'FCST_WINDOW_BEGIN and FCST_WINDOW_END', 'copy': False}, - 'PCP_COMBINE_METHOD' : - {'sec' : 'config', 'alt' : 'FCST_PCP_COMBINE_METHOD and/or OBS_PCP_COMBINE_METHOD', 'copy': False}, - 'FHR_BEG' : {'sec' : 'config', 'alt' : 'LEAD_SEQ', 'copy': False}, - 'FHR_END' : {'sec' : 'config', 'alt' : 'LEAD_SEQ', 'copy': False}, - 'FHR_INC' : {'sec' : 'config', 'alt' : 'LEAD_SEQ', 'copy': False}, - 'FHR_GROUP_BEG' : {'sec' : 'config', 'alt' : 'LEAD_SEQ_[N]', 'copy': False}, - 'FHR_GROUP_END' : {'sec' : 'config', 'alt' : 'LEAD_SEQ_[N]', 'copy': False}, - 'FHR_GROUP_LABELS' : {'sec' : 'config', 'alt' : 'LEAD_SEQ_[N]_LABEL', 'copy': False}, - 'CYCLONE_OUT_DIR' : {'sec' : 'dir', 'alt' : 'CYCLONE_OUTPUT_DIR'}, - 'ENSEMBLE_STAT_OUT_DIR' : {'sec' : 'dir', 'alt' : 'ENSEMBLE_STAT_OUTPUT_DIR'}, - 'EXTRACT_OUT_DIR' : {'sec' : 'dir', 'alt' : 'EXTRACT_TILES_OUTPUT_DIR'}, - 'GRID_STAT_OUT_DIR' : {'sec' : 'dir', 'alt' : 'GRID_STAT_OUTPUT_DIR'}, - 'MODE_OUT_DIR' : {'sec' : 'dir', 'alt' : 'MODE_OUTPUT_DIR'}, - 'MTD_OUT_DIR' : {'sec' : 'dir', 'alt' : 'MTD_OUTPUT_DIR'}, - 'SERIES_INIT_OUT_DIR' : {'sec' : 'dir', 'alt' : 'SERIES_ANALYSIS_OUTPUT_DIR'}, - 'SERIES_LEAD_OUT_DIR' : {'sec' : 'dir', 'alt' : 'SERIES_ANALYSIS_OUTPUT_DIR'}, - 'SERIES_INIT_FILTERED_OUT_DIR' : - {'sec' : 'dir', 'alt' : 'SERIES_ANALYSIS_FILTERED_OUTPUT_DIR'}, - 'SERIES_LEAD_FILTERED_OUT_DIR' : - {'sec' : 'dir', 'alt' : 'SERIES_ANALYSIS_FILTERED_OUTPUT_DIR'}, - 'STAT_ANALYSIS_OUT_DIR' : - {'sec' : 'dir', 'alt' : 'STAT_ANALYSIS_OUTPUT_DIR'}, - 'TCMPR_PLOT_OUT_DIR' : {'sec' : 'dir', 'alt' : 'TCMPR_PLOT_OUTPUT_DIR'}, - 'FCST_MIN_FORECAST' : {'sec' : 'config', 'alt' : 'LEAD_SEQ_MIN'}, - 'FCST_MAX_FORECAST' : {'sec' : 'config', 'alt' : 'LEAD_SEQ_MAX'}, - 'OBS_MIN_FORECAST' : {'sec' : 'config', 'alt' : 'OBS_PCP_COMBINE_MIN_LEAD'}, - 'OBS_MAX_FORECAST' : {'sec' : 'config', 'alt' : 'OBS_PCP_COMBINE_MAX_LEAD'}, - 'FCST_INIT_INTERVAL' : {'sec' : 'config', 'alt' : None}, - 'OBS_INIT_INTERVAL' : {'sec' : 'config', 'alt' : None}, - 'FCST_DATA_INTERVAL' : {'sec' : '', 'alt' : 'FCST_PCP_COMBINE_DATA_INTERVAL'}, - 'OBS_DATA_INTERVAL' : {'sec' : '', 'alt' : 'OBS_PCP_COMBINE_DATA_INTERVAL'}, - 'FCST_IS_DAILY_FILE' : {'sec' : '', 'alt' : 'FCST_PCP_COMBINE_IS_DAILY_FILE'}, - 'OBS_IS_DAILY_FILE' : {'sec' : '', 'alt' : 'OBS_PCP_COMBINE_IS_DAILY_FILE'}, - 'FCST_TIMES_PER_FILE' : {'sec' : '', 'alt' : 'FCST_PCP_COMBINE_TIMES_PER_FILE'}, - 'OBS_TIMES_PER_FILE' : {'sec' : '', 'alt' : 'OBS_PCP_COMBINE_TIMES_PER_FILE'}, - 'FCST_LEVEL' : {'sec' : '', 'alt' : 'FCST_PCP_COMBINE_INPUT_ACCUMS', 'copy': False}, - 'OBS_LEVEL' : {'sec' : '', 'alt' : 'OBS_PCP_COMBINE_INPUT_ACCUMS', 'copy': False}, - 'MODE_FCST_CONV_RADIUS' : {'sec' : 'config', 'alt' : 'FCST_MODE_CONV_RADIUS'}, - 'MODE_FCST_CONV_THRESH' : {'sec' : 'config', 'alt' : 'FCST_MODE_CONV_THRESH'}, - 'MODE_FCST_MERGE_FLAG' : {'sec' : 'config', 'alt' : 'FCST_MODE_MERGE_FLAG'}, - 'MODE_FCST_MERGE_THRESH' : {'sec' : 'config', 'alt' : 'FCST_MODE_MERGE_THRESH'}, - 'MODE_OBS_CONV_RADIUS' : {'sec' : 'config', 'alt' : 'OBS_MODE_CONV_RADIUS'}, - 'MODE_OBS_CONV_THRESH' : {'sec' : 'config', 'alt' : 'OBS_MODE_CONV_THRESH'}, - 'MODE_OBS_MERGE_FLAG' : {'sec' : 'config', 'alt' : 'OBS_MODE_MERGE_FLAG'}, - 'MODE_OBS_MERGE_THRESH' : {'sec' : 'config', 'alt' : 'OBS_MODE_MERGE_THRESH'}, - 'MTD_FCST_CONV_RADIUS' : {'sec' : 'config', 'alt' : 'FCST_MTD_CONV_RADIUS'}, - 'MTD_FCST_CONV_THRESH' : {'sec' : 'config', 'alt' : 'FCST_MTD_CONV_THRESH'}, - 'MTD_OBS_CONV_RADIUS' : {'sec' : 'config', 'alt' : 'OBS_MTD_CONV_RADIUS'}, - 'MTD_OBS_CONV_THRESH' : {'sec' : 'config', 'alt' : 'OBS_MTD_CONV_THRESH'}, - 'RM_EXE' : {'sec' : 'exe', 'alt' : 'RM'}, - 'CUT_EXE' : {'sec' : 'exe', 'alt' : 'CUT'}, - 'TR_EXE' : {'sec' : 'exe', 'alt' : 'TR'}, - 'NCAP2_EXE' : {'sec' : 'exe', 'alt' : 'NCAP2'}, - 'CONVERT_EXE' : {'sec' : 'exe', 'alt' : 'CONVERT'}, - 'NCDUMP_EXE' : {'sec' : 'exe', 'alt' : 'NCDUMP'}, - 'EGREP_EXE' : {'sec' : 'exe', 'alt' : 'EGREP'}, - 'ADECK_TRACK_DATA_DIR' : {'sec' : 'dir', 'alt' : 'TC_PAIRS_ADECK_INPUT_DIR'}, - 'BDECK_TRACK_DATA_DIR' : {'sec' : 'dir', 'alt' : 'TC_PAIRS_BDECK_INPUT_DIR'}, - 'MISSING_VAL_TO_REPLACE' : {'sec' : 'config', 'alt' : 'TC_PAIRS_MISSING_VAL_TO_REPLACE'}, - 'MISSING_VAL' : {'sec' : 'config', 'alt' : 'TC_PAIRS_MISSING_VAL'}, - 'TRACK_DATA_SUBDIR_MOD' : {'sec' : 'dir', 'alt' : None}, - 'ADECK_FILE_PREFIX' : {'sec' : 'config', 'alt' : 'TC_PAIRS_ADECK_TEMPLATE', 'copy': False}, - 'BDECK_FILE_PREFIX' : {'sec' : 'config', 'alt' : 'TC_PAIRS_BDECK_TEMPLATE', 'copy': False}, - 'TOP_LEVEL_DIRS' : {'sec' : 'config', 'alt' : 'TC_PAIRS_READ_ALL_FILES'}, - 'TC_PAIRS_DIR' : {'sec' : 'dir', 'alt' : 'TC_PAIRS_OUTPUT_DIR'}, - 'CYCLONE' : {'sec' : 'config', 'alt' : 'TC_PAIRS_CYCLONE'}, - 'STORM_ID' : {'sec' : 'config', 'alt' : 'TC_PAIRS_STORM_ID'}, - 'BASIN' : {'sec' : 'config', 'alt' : 'TC_PAIRS_BASIN'}, - 'STORM_NAME' : {'sec' : 'config', 'alt' : 'TC_PAIRS_STORM_NAME'}, - 'DLAND_FILE' : {'sec' : 'config', 'alt' : 'TC_PAIRS_DLAND_FILE'}, - 'TRACK_TYPE' : {'sec' : 'config', 'alt' : 'TC_PAIRS_REFORMAT_DECK'}, - 'FORECAST_TMPL' : {'sec' : 'filename_templates', 'alt' : 'TC_PAIRS_ADECK_TEMPLATE'}, - 'REFERENCE_TMPL' : {'sec' : 'filename_templates', 'alt' : 'TC_PAIRS_BDECK_TEMPLATE'}, - 'TRACK_DATA_MOD_FORCE_OVERWRITE' : - {'sec' : 'config', 'alt' : 'TC_PAIRS_SKIP_IF_REFORMAT_EXISTS', 'copy': False}, - 'TC_PAIRS_FORCE_OVERWRITE' : {'sec' : 'config', 'alt' : 'TC_PAIRS_SKIP_IF_OUTPUT_EXISTS', 'copy': False}, - 'GRID_STAT_CONFIG' : {'sec' : 'config', 'alt' : 'GRID_STAT_CONFIG_FILE'}, - 'MODE_CONFIG' : {'sec' : 'config', 'alt': 'MODE_CONFIG_FILE'}, - 'FCST_PCP_COMBINE_INPUT_LEVEL': {'sec': 'config', 'alt' : 'FCST_PCP_COMBINE_INPUT_ACCUMS'}, - 'OBS_PCP_COMBINE_INPUT_LEVEL': {'sec': 'config', 'alt' : 'OBS_PCP_COMBINE_INPUT_ACCUMS'}, - 'TIME_METHOD': {'sec': 'config', 'alt': 'LOOP_BY', 'copy': False}, - 'MODEL_DATA_DIR': {'sec': 'dir', 'alt': 'EXTRACT_TILES_GRID_INPUT_DIR'}, - 'STAT_LIST': {'sec': 'config', 'alt': 'SERIES_ANALYSIS_STAT_LIST'}, - 'NLAT': {'sec': 'config', 'alt': 'EXTRACT_TILES_NLAT'}, - 'NLON': {'sec': 'config', 'alt': 'EXTRACT_TILES_NLON'}, - 'DLAT': {'sec': 'config', 'alt': 'EXTRACT_TILES_DLAT'}, - 'DLON': {'sec': 'config', 'alt': 'EXTRACT_TILES_DLON'}, - 'LON_ADJ': {'sec': 'config', 'alt': 'EXTRACT_TILES_LON_ADJ'}, - 'LAT_ADJ': {'sec': 'config', 'alt': 'EXTRACT_TILES_LAT_ADJ'}, - 'OVERWRITE_TRACK': {'sec': 'config', 'alt': 'EXTRACT_TILES_OVERWRITE_TRACK'}, - 'BACKGROUND_MAP': {'sec': 'config', 'alt': 'SERIES_ANALYSIS_BACKGROUND_MAP'}, - 'GFS_FCST_FILE_TMPL': {'sec': 'filename_templates', 'alt': 'FCST_EXTRACT_TILES_INPUT_TEMPLATE'}, - 'GFS_ANLY_FILE_TMPL': {'sec': 'filename_templates', 'alt': 'OBS_EXTRACT_TILES_INPUT_TEMPLATE'}, - 'SERIES_BY_LEAD_FILTERED_OUTPUT_DIR': {'sec': 'dir', 'alt': 'SERIES_ANALYSIS_FILTERED_OUTPUT_DIR'}, - 'SERIES_BY_INIT_FILTERED_OUTPUT_DIR': {'sec': 'dir', 'alt': 'SERIES_ANALYSIS_FILTERED_OUTPUT_DIR'}, - 'SERIES_BY_LEAD_OUTPUT_DIR': {'sec': 'dir', 'alt': 'SERIES_ANALYSIS_OUTPUT_DIR'}, - 'SERIES_BY_INIT_OUTPUT_DIR': {'sec': 'dir', 'alt': 'SERIES_ANALYSIS_OUTPUT_DIR'}, - 'SERIES_BY_LEAD_GROUP_FCSTS': {'sec': 'config', 'alt': 'SERIES_ANALYSIS_GROUP_FCSTS'}, - 'SERIES_ANALYSIS_BY_LEAD_CONFIG_FILE': {'sec': 'config', 'alt': 'SERIES_ANALYSIS_CONFIG_FILE'}, - 'SERIES_ANALYSIS_BY_INIT_CONFIG_FILE': {'sec': 'config', 'alt': 'SERIES_ANALYSIS_CONFIG_FILE'}, - 'ENSEMBLE_STAT_MET_OBS_ERROR_TABLE': {'sec': 'config', 'alt': 'ENSEMBLE_STAT_MET_OBS_ERR_TABLE'}, - 'VAR_LIST': {'sec': 'config', 'alt': 'BOTH_VAR_NAME BOTH_VAR_LEVELS or SERIES_ANALYSIS_VAR_LIST', 'copy': False}, - 'SERIES_ANALYSIS_VAR_LIST': {'sec': 'config', 'alt': 'BOTH_VAR_NAME BOTH_VAR_LEVELS', 'copy': False}, - 'EXTRACT_TILES_VAR_LIST': {'sec': 'config', 'alt': ''}, - 'STAT_ANALYSIS_LOOKIN_DIR': {'sec': 'dir', 'alt': 'MODEL1_STAT_ANALYSIS_LOOKIN_DIR'}, - 'VALID_HOUR_METHOD': {'sec': 'config', 'alt': None}, - 'VALID_HOUR_BEG': {'sec': 'config', 'alt': None}, - 'VALID_HOUR_END': {'sec': 'config', 'alt': None}, - 'VALID_HOUR_INCREMENT': {'sec': 'config', 'alt': None}, - 'INIT_HOUR_METHOD': {'sec': 'config', 'alt': None}, - 'INIT_HOUR_BEG': {'sec': 'config', 'alt': None}, - 'INIT_HOUR_END': {'sec': 'config', 'alt': None}, - 'INIT_HOUR_INCREMENT': {'sec': 'config', 'alt': None}, - 'STAT_ANALYSIS_CONFIG': {'sec': 'config', 'alt': 'STAT_ANALYSIS_CONFIG_FILE'}, - 'JOB_NAME': {'sec': 'config', 'alt': 'STAT_ANALYSIS_JOB_NAME'}, - 'JOB_ARGS': {'sec': 'config', 'alt': 'STAT_ANALYSIS_JOB_ARGS'}, - 'FCST_LEAD': {'sec': 'config', 'alt': 'FCST_LEAD_LIST'}, - 'FCST_VAR_NAME': {'sec': 'config', 'alt': 'FCST_VAR_LIST'}, - 'FCST_VAR_LEVEL': {'sec': 'config', 'alt': 'FCST_VAR_LEVEL_LIST'}, - 'OBS_VAR_NAME': {'sec': 'config', 'alt': 'OBS_VAR_LIST'}, - 'OBS_VAR_LEVEL': {'sec': 'config', 'alt': 'OBS_VAR_LEVEL_LIST'}, - 'REGION': {'sec': 'config', 'alt': 'VX_MASK_LIST'}, - 'INTERP': {'sec': 'config', 'alt': 'INTERP_LIST'}, - 'INTERP_PTS': {'sec': 'config', 'alt': 'INTERP_PTS_LIST'}, - 'CONV_THRESH': {'sec': 'config', 'alt': 'CONV_THRESH_LIST'}, - 'FCST_THRESH': {'sec': 'config', 'alt': 'FCST_THRESH_LIST'}, - 'LINE_TYPE': {'sec': 'config', 'alt': 'LINE_TYPE_LIST'}, - 'STAT_ANALYSIS_DUMP_ROW_TMPL': {'sec': 'filename_templates', 'alt': 'STAT_ANALYSIS_DUMP_ROW_TEMPLATE'}, - 'STAT_ANALYSIS_OUT_STAT_TMPL': {'sec': 'filename_templates', 'alt': 'STAT_ANALYSIS_OUT_STAT_TEMPLATE'}, - 'PLOTTING_SCRIPTS_DIR': {'sec': 'dir', 'alt': 'MAKE_PLOTS_SCRIPTS_DIR'}, - 'STAT_FILES_INPUT_DIR': {'sec': 'dir', 'alt': 'MAKE_PLOTS_INPUT_DIR'}, - 'PLOTTING_OUTPUT_DIR': {'sec': 'dir', 'alt': 'MAKE_PLOTS_OUTPUT_DIR'}, - 'VERIF_CASE': {'sec': 'config', 'alt': 'MAKE_PLOTS_VERIF_CASE'}, - 'VERIF_TYPE': {'sec': 'config', 'alt': 'MAKE_PLOTS_VERIF_TYPE'}, - 'PLOT_TIME': {'sec': 'config', 'alt': 'DATE_TIME'}, - 'MODEL_NAME': {'sec': 'config', 'alt': 'MODEL'}, - 'MODEL_OBS_NAME': {'sec': 'config', 'alt': 'MODEL_OBTYPE'}, - 'MODEL_STAT_DIR': {'sec': 'dir', 'alt': 'MODEL_STAT_ANALYSIS_LOOKIN_DIR'}, - 'MODEL_NAME_ON_PLOT': {'sec': 'config', 'alt': 'MODEL_REFERENCE_NAME'}, - 'REGION_LIST': {'sec': 'config', 'alt': 'VX_MASK_LIST'}, - 'PLOT_STATS_LIST': {'sec': 'config', 'alt': 'MAKE_PLOT_STATS_LIST'}, - 'CI_METHOD': {'sec': 'config', 'alt': 'MAKE_PLOTS_CI_METHOD'}, - 'VERIF_GRID': {'sec': 'config', 'alt': 'MAKE_PLOTS_VERIF_GRID'}, - 'EVENT_EQUALIZATION': {'sec': 'config', 'alt': 'MAKE_PLOTS_EVENT_EQUALIZATION'}, - 'MTD_CONFIG': {'sec': 'config', 'alt': 'MTD_CONFIG_FILE'}, - 'CLIMO_GRID_STAT_INPUT_DIR': {'sec': 'dir', 'alt': 'GRID_STAT_CLIMO_MEAN_INPUT_DIR'}, - 'CLIMO_GRID_STAT_INPUT_TEMPLATE': {'sec': 'filename_templates', 'alt': 'GRID_STAT_CLIMO_MEAN_INPUT_TEMPLATE'}, - 'CLIMO_POINT_STAT_INPUT_DIR': {'sec': 'dir', 'alt': 'POINT_STAT_CLIMO_MEAN_INPUT_DIR'}, - 'CLIMO_POINT_STAT_INPUT_TEMPLATE': {'sec': 'filename_templates', 'alt': 'POINT_STAT_CLIMO_MEAN_INPUT_TEMPLATE'}, - 'GEMPAKTOCF_CLASSPATH': {'sec': 'exe', 'alt': 'GEMPAKTOCF_JAR', 'copy': False}, - 'CUSTOM_INGEST__OUTPUT_DIR': {'sec': 'dir', 'alt': 'PY_EMBED_INGEST__OUTPUT_DIR'}, - 'CUSTOM_INGEST__OUTPUT_TEMPLATE': {'sec': 'filename_templates', 'alt': 'PY_EMBED_INGEST__OUTPUT_TEMPLATE'}, - 'CUSTOM_INGEST__OUTPUT_GRID': {'sec': 'config', 'alt': 'PY_EMBED_INGEST__OUTPUT_GRID'}, - 'CUSTOM_INGEST__SCRIPT': {'sec': 'config', 'alt': 'PY_EMBED_INGEST__SCRIPT'}, - 'CUSTOM_INGEST__TYPE': {'sec': 'config', 'alt': 'PY_EMBED_INGEST__TYPE'}, - 'TC_STAT_RUN_VIA': {'sec': 'config', 'alt': 'TC_STAT_CONFIG_FILE', - 'copy': False}, - 'TC_STAT_CMD_LINE_JOB': {'sec': 'config', 'alt': 'TC_STAT_JOB_ARGS'}, - 'TC_STAT_JOBS_LIST': {'sec': 'config', 'alt': 'TC_STAT_JOB_ARGS'}, - 'EXTRACT_TILES_OVERWRITE_TRACK': {'sec': 'config', - 'alt': 'EXTRACT_TILES_SKIP_IF_OUTPUT_EXISTS', - 'copy': False}, - 'EXTRACT_TILES_PAIRS_INPUT_DIR': {'sec': 'dir', - 'alt': 'EXTRACT_TILES_STAT_INPUT_DIR', - 'copy': False}, - 'EXTRACT_TILES_FILTERED_OUTPUT_TEMPLATE': {'sec': 'filename_template', - 'alt': 'EXTRACT_TILES_STAT_INPUT_TEMPLATE',}, - 'EXTRACT_TILES_GRID_INPUT_DIR': {'sec': 'dir', - 'alt': 'FCST_EXTRACT_TILES_INPUT_DIR' - 'and ' - 'OBS_EXTRACT_TILES_INPUT_DIR', - 'copy': False}, - 'SERIES_ANALYSIS_FILTER_OPTS': {'sec': 'config', - 'alt': 'TC_STAT_JOB_ARGS', - 'copy': False}, - 'SERIES_ANALYSIS_INPUT_DIR': {'sec': 'dir', - 'alt': 'FCST_SERIES_ANALYSIS_INPUT_DIR ' - 'and ' - 'OBS_SERIES_ANALYSIS_INPUT_DIR'}, - 'FCST_SERIES_ANALYSIS_TILE_INPUT_TEMPLATE': {'sec': 'filename_templates', - 'alt': 'FCST_SERIES_ANALYSIS_INPUT_TEMPLATE '}, - 'OBS_SERIES_ANALYSIS_TILE_INPUT_TEMPLATE': {'sec': 'filename_templates', - 'alt': 'OBS_SERIES_ANALYSIS_INPUT_TEMPLATE '}, - 'EXTRACT_TILES_STAT_INPUT_DIR': {'sec': 'dir', - 'alt': 'EXTRACT_TILES_TC_STAT_INPUT_DIR',}, - 'EXTRACT_TILES_STAT_INPUT_TEMPLATE': {'sec': 'filename_templates', - 'alt': 'EXTRACT_TILES_TC_STAT_INPUT_TEMPLATE',}, - 'SERIES_ANALYSIS_STAT_INPUT_DIR': {'sec': 'dir', - 'alt': 'SERIES_ANALYSIS_TC_STAT_INPUT_DIR', }, - 'SERIES_ANALYSIS_STAT_INPUT_TEMPLATE': {'sec': 'filename_templates', - 'alt': 'SERIES_ANALYSIS_TC_STAT_INPUT_TEMPLATE', }, - } - - # template '' : {'sec' : '', 'alt' : '', 'copy': True}, - - logger = config.logger - - # create list of errors and warnings to report for deprecated configs - e_list = [] - w_list = [] - all_sed_cmds = [] - - for old, depr_info in deprecated_dict.items(): - if isinstance(depr_info, dict): - - # check if is found in the old item, use regex to find variables if found - if '' in old: - old_regex = old.replace('', r'(\d+)') - indicies = find_indices_in_config_section(old_regex, - config, - index_index=1).keys() - for index in indicies: - old_with_index = old.replace('', index) - if depr_info['alt']: - alt_with_index = depr_info['alt'].replace('', index) - else: - alt_with_index = '' - - handle_deprecated(old_with_index, alt_with_index, depr_info, - config, all_sed_cmds, w_list, e_list) - else: - handle_deprecated(old, depr_info['alt'], depr_info, - config, all_sed_cmds, w_list, e_list) - - - # check all templates and error if any deprecated tags are used - # value of dict is replacement tag, set to None if no replacement exists - # deprecated tags: region (replace with basin) - deprecated_tags = {'region' : 'basin'} - template_vars = config.keys('config') - template_vars = [tvar for tvar in template_vars if tvar.endswith('_TEMPLATE')] - for temp_var in template_vars: - template = config.getraw('filename_templates', temp_var) - tags = get_tags(template) - - for depr_tag, replace_tag in deprecated_tags.items(): - if depr_tag in tags: - e_msg = 'Deprecated tag {{{}}} found in {}.'.format(depr_tag, - temp_var) - if replace_tag is not None: - e_msg += ' Replace with {{{}}}'.format(replace_tag) - - e_list.append(e_msg) - - # if any warning exist, report them - if w_list: - for warning_msg in w_list: - logger.warning(warning_msg) - - # if any errors exist, report them and exit - if e_list: - logger.error('DEPRECATED CONFIG ITEMS WERE FOUND. ' +\ - 'PLEASE REMOVE/REPLACE THEM FROM CONFIG FILES') - for error_msg in e_list: - logger.error(error_msg) - return False, all_sed_cmds - - return True, [] - -def handle_deprecated(old, alt, depr_info, config, all_sed_cmds, w_list, e_list): - sec = depr_info['sec'] - config_files = config.getstr('config', 'CONFIG_INPUT', '').split(',') - # if deprecated config item is found - if config.has_option(sec, old): - # if it is not required to remove, add to warning list - if 'req' in depr_info.keys() and depr_info['req'] is False: - msg = '[{}] {} is deprecated and will be '.format(sec, old) + \ - 'removed in a future version of METplus' - if alt: - msg += ". Please replace with {}".format(alt) - w_list.append(msg) - # if it is required to remove, add to error list - else: - if not alt: - e_list.append("[{}] {} should be removed".format(sec, old)) - else: - e_list.append("[{}] {} should be replaced with {}".format(sec, old, alt)) - - if 'copy' not in depr_info.keys() or depr_info['copy']: - for config_file in config_files: - all_sed_cmds.append(f"sed -i 's|^{old}|{alt}|g' {config_file}") - all_sed_cmds.append(f"sed -i 's|{{{old}}}|{{{alt}}}|g' {config_file}") - -def check_for_deprecated_met_config(config): - sed_cmds = [] - all_good = True - - # set CURRENT_* METplus variables in case they are referenced in a - # METplus config variable and not already set - for fcst_or_obs in ['FCST', 'OBS']: - for name_or_level in ['NAME', 'LEVEL']: - current_var = f'CURRENT_{fcst_or_obs}_{name_or_level}' - if not config.has_option('config', current_var): - config.set('config', current_var, '') - - # check if *_CONFIG_FILE if set in the METplus config file and check for - # deprecated environment variables in those files - met_config_keys = [key for key in config.keys('config') - if key.endswith('CONFIG_FILE')] - - for met_config_key in met_config_keys: - met_tool = met_config_key.replace('_CONFIG_FILE', '') - - # get custom loop list to check if multiple config files are used based on the custom string - custom_list = get_custom_string_list(config, met_tool) - - for custom_string in custom_list: - met_config = config.getraw('config', met_config_key) - if not met_config: - continue - - met_config_file = do_string_sub(met_config, custom=custom_string) - - if not check_for_deprecated_met_config_file(config, met_config_file, sed_cmds, met_tool): - all_good = False - - return all_good, sed_cmds - -def check_for_deprecated_met_config_file(config, met_config, sed_cmds, met_tool): - - all_good = True - if not os.path.exists(met_config): - config.logger.error(f"Config file does not exist: {met_config}") - return False - - deprecated_met_list = ['MET_VALID_HHMM', 'GRID_VX', 'CONFIG_DIR'] - deprecated_output_prefix_list = ['FCST_VAR', 'OBS_VAR'] - config.logger.debug(f"Checking for deprecated environment variables in: {met_config}") - - with open(met_config, 'r') as file_handle: - lines = file_handle.read().splitlines() - - for line in lines: - for deprecated_item in deprecated_met_list: - if '${' + deprecated_item + '}' in line: - all_good = False - config.logger.error("Please remove deprecated environment variable " - f"${{{deprecated_item}}} found in MET config file: " - f"{met_config}") - - if deprecated_item == 'MET_VALID_HHMM' and 'file_name' in line: - config.logger.error(f"Set {met_tool}_CLIMO_MEAN_INPUT_[DIR/TEMPLATE] in a " - "METplus config file to set CLIMO_MEAN_FILE in a MET config") - new_line = " file_name = [ ${CLIMO_MEAN_FILE} ];" - - # escape [ and ] because they are special characters in sed commands - old_line = line.rstrip().replace('[', r'\[').replace(']', r'\]') - - sed_cmds.append(f"sed -i 's|^{old_line}|{new_line}|g' {met_config}") - add_line = f"{met_tool}_CLIMO_MEAN_INPUT_TEMPLATE" - sed_cmds.append(f"#Add {add_line}") - break - - if 'to_grid' in line: - config.logger.error("MET to_grid variable should reference " - "${REGRID_TO_GRID} environment variable") - new_line = " to_grid = ${REGRID_TO_GRID};" - - # escape [ and ] because they are special characters in sed commands - old_line = line.rstrip().replace('[', r'\[').replace(']', r'\]') - - sed_cmds.append(f"sed -i 's|^{old_line}|{new_line}|g' {met_config}") - config.logger.info(f"Be sure to set {met_tool}_REGRID_TO_GRID to the correct value.") - add_line = f"{met_tool}_REGRID_TO_GRID" - sed_cmds.append(f"#Add {add_line}") - break - - - for deprecated_item in deprecated_output_prefix_list: - # if deprecated item found in output prefix or to_grid line, replace line to use - # env var OUTPUT_PREFIX or REGRID_TO_GRID - if '${' + deprecated_item + '}' in line and 'output_prefix' in line: - config.logger.error("output_prefix variable should reference " - "${OUTPUT_PREFIX} environment variable") - new_line = "output_prefix = \"${OUTPUT_PREFIX}\";" - - # escape [ and ] because they are special characters in sed commands - old_line = line.rstrip().replace('[', r'\[').replace(']', r'\]') - - sed_cmds.append(f"sed -i 's|^{old_line}|{new_line}|g' {met_config}") - config.logger.info(f"You will need to add {met_tool}_OUTPUT_PREFIX to the METplus config file" - f" that sets {met_tool}_CONFIG_FILE. Set it to:") - output_prefix = replace_output_prefix(line) - add_line = f"{met_tool}_OUTPUT_PREFIX = {output_prefix}" - config.logger.info(add_line) - sed_cmds.append(f"#Add {add_line}") - all_good = False - break - - return all_good - -def replace_output_prefix(line): - op_replacements = {'${MODEL}': '{MODEL}', - '${FCST_VAR}': '{CURRENT_FCST_NAME}', - '${OBTYPE}': '{OBTYPE}', - '${OBS_VAR}': '{CURRENT_OBS_NAME}', - '${LEVEL}': '{CURRENT_FCST_LEVEL}', - '${FCST_TIME}': '{lead?fmt=%3H}', - } - prefix = line.split('=')[1].strip().rstrip(';').strip('"') - for key, value, in op_replacements.items(): - prefix = prefix.replace(key, value) - - return prefix - -def get_custom_string_list(config, met_tool): - custom_loop_list = config.getstr_nocheck('config', - f'{met_tool.upper()}_CUSTOM_LOOP_LIST', - config.getstr_nocheck('config', - 'CUSTOM_LOOP_LIST', - '')) - custom_loop_list = getlist(custom_loop_list) - if not custom_loop_list: - custom_loop_list.append('') - - return custom_loop_list - def handle_tmp_dir(config): """! if env var MET_TMP_DIR is set, override config TMP_DIR with value if it differs from what is set @@ -1276,18 +786,6 @@ def round_0p5(val): return round(val * 2) / 2 -def round_to_int(val): - """! Round to integer value - Args: - @param val: The value to round up - Returns: - rval: The rounded up value. - """ - val += 0.5 - rval = int(val) - return rval - - def mkdir_p(path): """! From stackoverflow.com/questions/600268/mkdir-p-functionality-in-python @@ -1301,138 +799,6 @@ def mkdir_p(path): """ Path(path).mkdir(parents=True, exist_ok=True) -def _rmtree_onerr(function, path, exc_info, logger=None): - """!Internal function used to log errors. - This is an internal implementation function called by - shutil.rmtree when an underlying function call failed. See - the Python documentation of shutil.rmtree for details. - @param function the funciton that failed - @param path the path to the function that caused problems - @param exc_info the exception information - @protected""" - if logger: - logger.warning('%s: %s failed: %s' % ( - str(path), str(function), str(exc_info))) - - -def rmtree(tree, logger=None): - """!Deletes the tree, if possible. - @protected - @param tree the directory tree to delete" - @param logger the logger, optional - """ - try: - # If it is a file, special file or symlink we can just - # delete it via unlink: - os.unlink(tree) - return - except EnvironmentError: - pass - # We get here for directories. - if logger: - logger.info('%s: rmtree' % (tree,)) - shutil.rmtree(tree, ignore_errors=False) - -def file_exists(filename): - """! Determines if a file exists - NOTE: Simply using os.path.isfile() is not a Pythonic way - to check if a file exists. You can - still encounter a TOCTTOU bug - "time of check to time of use" - Instead, use the raising of - exceptions, which is a Pythonic - approach: - try: - with open(filename) as fileobj: - pass # or do something fruitful - except IOError as e: - logger.error("your helpful error message goes here") - Args: - @param filename: the full filename (full path) - Returns: - boolean : True if file exists, False otherwise - """ - - try: - return os.path.isfile(filename) - except IOError: - pass - - -def is_dir_empty(directory): - """! Determines if a directory exists and is not empty - Args: - @param directory: The directory to check for existence - and for contents. - Returns: - True: If the directory is empty - False: If the directory exists and isn't empty - """ - return not os.listdir(directory) - -def grep(pattern, infile): - """! Python version of grep, searches the file line-by-line - to find a match to the pattern. Returns upon finding the - first match. - Args: - @param pattern: The pattern to be matched - @param infile: The filename with full filepath in which to - search for the pattern - Returns: - line (string): The matching string - """ - - matching_lines = [] - with open(infile, 'r') as file_handle: - for line in file_handle: - match = re.search(pattern, line) - if match: - matching_lines.append(line) - # if you got here, you didn't find anything - return matching_lines - - -def get_filepaths_for_grbfiles(base_dir): - """! Generates the grb2 file names in a directory tree - by walking the tree either top-down or bottom-up. - For each directory in the tree rooted at - the directory top (including top itself), it - produces a tuple: (dirpath, dirnames, filenames). - This solution was found on Stack Overflow: - http://stackoverflow.com/questions/3207219/how-to-list-all-files-of-a- - directory-in-python#3207973 - **scroll down to the section with "Getting Full File Paths From a - Directory and All Its Subdirectories" - Args: - @param base_dir: The base directory from which we - begin the search for grib2 filenames. - Returns: - file_paths (list): A list of the full filepaths - of the data to be processed. - """ - - # Create an empty list which will eventually store - # all the full filenames - file_paths = [] - - # pylint:disable=unused-variable - # os.walk returns tuple, we don't need to utilize all the returned - # values in the tuple. - - # Walk the tree - for root, directories, files in os.walk(base_dir): - for filename in files: - # add it to the list only if it is a grib file - match = re.match(r'.*(grib|grb|grib2|grb2)$', filename) - if match: - # Join the two strings to form the full - # filepath. - filepath = os.path.join(root, filename) - file_paths.append(filepath) - else: - continue - return file_paths - def get_storms(filter_filename, id_only=False, sort_column='STORM_ID'): """! Get each storm as identified by a column in the input file. Create dictionary storm ID as the key and a list of lines for that @@ -1476,18 +842,6 @@ def get_storms(filter_filename, id_only=False, sort_column='STORM_ID'): return storm_dict -def get_storm_ids(filter_filename): - """! Get each storm as identified by its STORM_ID in the filter file - save these in a set so we only save the unique ids and sort them. - Args: - @param filter_filename: The name of the filter file to read - and extract the storm id - @param logger: The name of the logger for logging useful info - Returns: - sorted_storms (List): a list of unique, sorted storm ids - """ - return get_storms(filter_filename, id_only=True) - def get_files(filedir, filename_regex, logger=None): """! Get all the files (with a particular naming format) by walking @@ -1742,68 +1096,6 @@ def camel_to_underscore(camel): s1 = re.sub(r'([^\d])([A-Z][a-z]+)', r'\1_\2', camel) return re.sub(r'([a-z])([A-Z])', r'\1_\2', s1).lower() -def get_process_list(config): - """!Read process list, Extract instance string if specified inside - parenthesis. Remove dashes/underscores and change to lower case, - then map the name to the correct wrapper name - - @param config METplusConfig object to read PROCESS_LIST value - @returns list of tuple containing process name and instance identifier - (None if no instance was set) - """ - # get list of processes - process_list = getlist(config.getstr('config', 'PROCESS_LIST')) - - out_process_list = [] - # for each item remove dashes, underscores, and cast to lower-case - for process in process_list: - # if instance is specified, extract the text inside parenthesis - match = re.match(r'(.*)\((.*)\)', process) - if match: - instance = match.group(2) - process_name = match.group(1) - else: - instance = None - process_name = process - - wrapper_name = get_wrapper_name(process_name) - if wrapper_name is None: - config.logger.warning(f"PROCESS_LIST item {process_name} " - "may be invalid.") - wrapper_name = process_name - - # if MakePlots is in process list, remove it because - # it will be called directly from StatAnalysis - if wrapper_name == 'MakePlots': - continue - - out_process_list.append((wrapper_name, instance)) - - return out_process_list - -# minutes -def shift_time(time_str, shift): - """ Adjust time by shift hours. Format is %Y%m%d%H%M%S - Args: - @param time_str: Start time in %Y%m%d%H%M%S - @param shift: Amount to adjust time in hours - Returns: - New time in format %Y%m%d%H%M%S - """ - return (datetime.datetime.strptime(time_str, "%Y%m%d%H%M%S") + - datetime.timedelta(hours=shift)).strftime("%Y%m%d%H%M%S") - -def shift_time_minutes(time_str, shift): - """ Adjust time by shift minutes. Format is %Y%m%d%H%M%S - Args: - @param time_str: Start time in %Y%m%d%H%M%S - @param shift: Amount to adjust time in minutes - Returns: - New time in format %Y%m%d%H%M%S - """ - return (datetime.datetime.strptime(time_str, "%Y%m%d%H%M%S") + - datetime.timedelta(minutes=shift)).strftime("%Y%m%d%H%M%S") - def shift_time_seconds(time_str, shift): """ Adjust time by shift seconds. Format is %Y%m%d%H%M%S Args: @@ -1903,298 +1195,6 @@ def write_list_to_file(filename, output_list): for line in output_list: f.write(f"{line}\n") -def validate_configuration_variables(config, force_check=False): - - all_sed_cmds = [] - # check for deprecated config items and warn user to remove/replace them - deprecated_isOK, sed_cmds = check_for_deprecated_config(config) - all_sed_cmds.extend(sed_cmds) - - # check for deprecated env vars in MET config files and warn user to remove/replace them - deprecatedMET_isOK, sed_cmds = check_for_deprecated_met_config(config) - all_sed_cmds.extend(sed_cmds) - - # validate configuration variables - field_isOK, sed_cmds = validate_field_info_configs(config, force_check) - all_sed_cmds.extend(sed_cmds) - - # check that OUTPUT_BASE is not set to the exact same value as INPUT_BASE - inoutbase_isOK = True - input_real_path = os.path.realpath(config.getdir_nocheck('INPUT_BASE', '')) - output_real_path = os.path.realpath(config.getdir('OUTPUT_BASE')) - if input_real_path == output_real_path: - config.logger.error(f"INPUT_BASE AND OUTPUT_BASE are set to the exact same path: {input_real_path}") - config.logger.error("Please change one of these paths to avoid risk of losing input data") - inoutbase_isOK = False - - check_user_environment(config) - - return deprecated_isOK, field_isOK, inoutbase_isOK, deprecatedMET_isOK, all_sed_cmds - -def skip_field_info_validation(config): - """!Check config to see if having corresponding FCST/OBS variables is necessary. If process list only - contains reformatter wrappers, don't validate field info. Also, if MTD is in the process list and - it is configured to only process either FCST or OBS, validation is unnecessary.""" - - reformatters = ['PCPCombine', 'RegridDataPlane'] - process_list = [item[0] for item in get_process_list(config)] - - # if running MTD in single mode, you don't need matching FCST/OBS - if 'MTD' in process_list and config.getbool('config', 'MTD_SINGLE_RUN'): - return True - - # if running any app other than the reformatters, you need matching FCST/OBS, so don't skip - if [item for item in process_list if item not in reformatters]: - return False - - return True - -def find_indices_in_config_section(regex, config, sec='config', - index_index=1, id_index=None): - """! Use regular expression to get all config variables that match and - are set in the user's configuration. This is used to handle config - variables that have multiple indices, i.e. FCST_VAR1_NAME, FCST_VAR2_NAME, - etc. - - @param regex regular expression to use to find variables - @param config METplusConfig object to search - @param sec (optional) config file section to search. Defaults to config - @param index_index 1 based number that is the regex match index for the - index number (default is 1) - @param id_index 1 based number that is the regex match index for the - identifier. Defaults to None which does not extract an indentifier - - number and the first match is used as an identifier - @returns dictionary where keys are the index number and the value is a - list of identifiers (if noID=True) or a list containing None - """ - # regex expression must have 2 () items and the 2nd item must be the index - all_conf = config.keys(sec) - indices = {} - regex = re.compile(regex) - for conf in all_conf: - result = regex.match(conf) - if result is not None: - index = result.group(index_index) - if id_index: - identifier = result.group(id_index) - else: - identifier = None - - if index not in indices: - indices[index] = [identifier] - else: - indices[index].append(identifier) - - return indices - -def is_var_item_valid(item_list, index, ext, config): - """!Given a list of data types (FCST, OBS, ENS, or BOTH) check if the - combination is valid. - If BOTH is found, FCST and OBS should not be found. - If FCST or OBS is found, the other must also be found. - @param item_list list of data types that were found for a given index - @param index number following _VAR in the variable name - @param ext extension to check, i.e. NAME, LEVELS, THRESH, or OPTIONS - @param config METplusConfig instance - @returns tuple containing boolean if var item is valid, list of error - messages and list of sed commands to help the user update their old - configuration files - """ - - full_ext = f"_VAR{index}_{ext}" - msg = [] - sed_cmds = [] - if 'BOTH' in item_list and ('FCST' in item_list or 'OBS' in item_list): - - msg.append(f"Cannot set FCST{full_ext} or OBS{full_ext} if BOTH{full_ext} is set.") - elif ext == 'THRESH': - # allow thresholds unless BOTH and (FCST or OBS) are set - pass - - elif 'FCST' in item_list and 'OBS' not in item_list: - # if FCST level has 1 item and OBS name is a python embedding script, - # don't report error - level_list = getlist(config.getraw('config', - f'FCST_VAR{index}_LEVELS', - '')) - other_name = config.getraw('config', f'OBS_VAR{index}_NAME', '') - skip_error_for_py_embed = ext == 'LEVELS' and is_python_script(other_name) and len(level_list) == 1 - # do not report error for OPTIONS since it isn't required to be the same length - if ext not in ['OPTIONS'] and not skip_error_for_py_embed: - msg.append(f"If FCST{full_ext} is set, you must either set OBS{full_ext} or " - f"change FCST{full_ext} to BOTH{full_ext}") - - config_files = config.getstr('config', 'CONFIG_INPUT', '').split(',') - for config_file in config_files: - sed_cmds.append(f"sed -i 's|^FCST{full_ext}|BOTH{full_ext}|g' {config_file}") - sed_cmds.append(f"sed -i 's|{{FCST{full_ext}}}|{{BOTH{full_ext}}}|g' {config_file}") - - elif 'OBS' in item_list and 'FCST' not in item_list: - # if OBS level has 1 item and FCST name is a python embedding script, - # don't report error - level_list = getlist(config.getraw('config', - f'OBS_VAR{index}_LEVELS', - '')) - other_name = config.getraw('config', f'FCST_VAR{index}_NAME', '') - skip_error_for_py_embed = ext == 'LEVELS' and is_python_script(other_name) and len(level_list) == 1 - - if ext not in ['OPTIONS'] and not skip_error_for_py_embed: - msg.append(f"If OBS{full_ext} is set, you must either set FCST{full_ext} or " - f"change OBS{full_ext} to BOTH{full_ext}") - - config_files = config.getstr('config', 'CONFIG_INPUT', '').split(',') - for config_file in config_files: - sed_cmds.append(f"sed -i 's|^OBS{full_ext}|BOTH{full_ext}|g' {config_file}") - sed_cmds.append(f"sed -i 's|{{OBS{full_ext}}}|{{BOTH{full_ext}}}|g' {config_file}") - - return not bool(msg), msg, sed_cmds - -def validate_field_info_configs(config, force_check=False): - """!Verify that config variables with _VAR_ in them are valid. Returns True if all are valid. - Returns False if any items are invalid""" - - variable_extensions = ['NAME', 'LEVELS', 'THRESH', 'OPTIONS'] - all_good = True, [] - - if skip_field_info_validation(config) and not force_check: - return True, [] - - # keep track of all sed commands to replace config variable names - all_sed_cmds = [] - - for ext in variable_extensions: - # find all _VAR_ keys in the conf files - data_types_and_indices = find_indices_in_config_section(r"(\w+)_VAR(\d+)_"+ext, - config, - index_index=2, - id_index=1) - - # if BOTH_VAR_ is used, set FCST and OBS to the same value - # if FCST or OBS is used, the other must be present as well - # if BOTH and either FCST or OBS are set, report an error - # get other data type - for index, data_type_list in data_types_and_indices.items(): - - is_valid, err_msgs, sed_cmds = is_var_item_valid(data_type_list, index, ext, config) - if not is_valid: - for err_msg in err_msgs: - config.logger.error(err_msg) - all_sed_cmds.extend(sed_cmds) - all_good = False - - # make sure FCST and OBS have the same number of levels if coming from separate variables - elif ext == 'LEVELS' and all(item in ['FCST', 'OBS'] for item in data_type_list): - fcst_levels = getlist(config.getraw('config', f"FCST_VAR{index}_LEVELS", '')) - - # add empty string if no levels are found because python embedding items do not need - # to include a level, but the other item may have a level and the numbers need to match - if not fcst_levels: - fcst_levels.append('') - - obs_levels = getlist(config.getraw('config', f"OBS_VAR{index}_LEVELS", '')) - if not obs_levels: - obs_levels.append('') - - if len(fcst_levels) != len(obs_levels): - config.logger.error(f"FCST_VAR{index}_LEVELS and OBS_VAR{index}_LEVELS do not have " - "the same number of elements") - all_good = False - - return all_good, all_sed_cmds - -def get_field_search_prefixes(data_type, met_tool=None): - """! Get list of prefixes to search for field variables. - - @param data_type type of field to search for, i.e. FCST, OBS, ENS, etc. - Check for BOTH_ variables first only if data type is FCST or OBS - @param met_tool name of tool to search for variable or None if looking - for generic field info - @returns list of prefixes to search, i.e. [BOTH_, FCST_] or - [ENS_] or [BOTH_GRID_STAT_, OBS_GRID_STAT_] - """ - search_prefixes = [] - var_strings = [] - - # if met tool name is set, prioritize - # wrapper-specific configs before generic configs - if met_tool: - var_strings.append(f'{met_tool.upper()}_') - - var_strings.append('') - - for var_string in var_strings: - search_prefixes.append(f"{data_type}_{var_string}") - - # if looking for FCST or OBS, also check for BOTH prefix - if data_type in ['FCST', 'OBS']: - search_prefixes.append(f"BOTH_{var_string}") - - return search_prefixes - -def get_field_config_variables(config, index, search_prefixes): - """! Search for variables that are set in the config that correspond to - the fields requested. Some field info items have - synonyms that can be used if the typical name is not set. This is used - in RegridDataPlane wrapper. - - @param config METplusConfig object to search - @param index of field (VAR) to find - @param search_prefixes list of valid prefixes to search for variables - in the config, i.e. FCST_VAR1_ or OBS_GRID_STAT_VAR2_ - @returns dictionary containing a config variable name to be used for - each field info value. If a valid config variable was not set for a - field info value, the value for that key will be set to None. - """ - # list of field info variables to find from config - # used as keys for dictionaries - field_info_items = ['name', - 'levels', - 'thresh', - 'options', - 'output_names', - ] - - field_configs = {} - search_suffixes = {} - - # initialize field configs dictionary values to None - # initialize dictionary of valid suffixes to search for with - # the capitalized version of field info name - for field_info_item in field_info_items: - field_configs[field_info_item] = None - search_suffixes[field_info_item] = [field_info_item.upper()] - - # add alternate suffixes for config variable names to attempt - search_suffixes['name'].append('INPUT_FIELD_NAME') - search_suffixes['name'].append('FIELD_NAME') - search_suffixes['levels'].append('INPUT_LEVEL') - search_suffixes['levels'].append('FIELD_LEVEL') - search_suffixes['output_names'].append('OUTPUT_FIELD_NAME') - search_suffixes['output_names'].append('FIELD_NAME') - - # look through field config keys and obtain highest priority - # variable name for each field config - for search_var, suffixes in search_suffixes.items(): - for prefix in search_prefixes: - - found = False - for suffix in suffixes: - var_name = f"{prefix}VAR{index}_{suffix}" - # if variable is found in config, - # get the value and break out of suffix loop - if config.has_option('config', var_name): - field_configs[search_var] = config.getraw('config', - var_name) - found = True - break - - # if config variable was found, break out of prefix loop - if found: - break - - return field_configs - def format_var_items(field_configs, time_info=None): """! Substitute time information into field information and format values. @@ -2281,188 +1281,6 @@ def format_var_items(field_configs, time_info=None): return var_items -def find_var_name_indices(config, data_types, met_tool=None): - data_type_regex = f"{'|'.join(data_types)}" - - # if data_types includes FCST or OBS, also search for BOTH - if any([item for item in ['FCST', 'OBS'] if item in data_types]): - data_type_regex += '|BOTH' - - regex_string = f"({data_type_regex})" - - # if MET tool is specified, get tool specific items - if met_tool: - regex_string += f"_{met_tool.upper()}" - - regex_string += r"_VAR(\d+)_(NAME|INPUT_FIELD_NAME|FIELD_NAME)" - - # find all _VAR_NAME keys in the conf files - return find_indices_in_config_section(regex_string, - config, - index_index=2, - id_index=1) - -def parse_var_list(config, time_info=None, data_type=None, met_tool=None, - levels_as_list=False): - """ read conf items and populate list of dictionaries containing - information about each variable to be compared - - @param config: METplusConfig object - @param time_info: time object for string sub, optional - @param data_type: data type to find. Can be FCST, OBS, or ENS. - If not set, get FCST/OBS/BOTH - @param met_tool: optional name of MET tool to look for wrapper - specific var items - @param levels_as_list If true, store levels and output names as - a list instead of creating a field info dict for each name/level - @returns list of dictionaries with variable information - """ - - # validate configs again in case wrapper is not running from run_metplus - # this does not need to be done if parsing a specific data type, - # i.e. ENS or FCST - if data_type is None: - if not validate_field_info_configs(config)[0]: - return [] - elif data_type == 'BOTH': - config.logger.error("Cannot request BOTH explicitly in parse_var_list") - return [] - - # var_list is a list containing an list of dictionaries - var_list = [] - - # if specific data type is requested, only get that type - if data_type: - data_types = [data_type] - # otherwise get both FCST and OBS - else: - data_types = ['FCST', 'OBS'] - - # get indices of VAR items for data type and/or met tool - indices = [] - if met_tool: - indices = find_var_name_indices(config, data_types, met_tool).keys() - if not indices: - indices = find_var_name_indices(config, data_types).keys() - - # get config name prefixes for each data type to find - dt_search_prefixes = {} - for current_type in data_types: - # get list of variable prefixes to search - prefixes = get_field_search_prefixes(current_type, met_tool) - dt_search_prefixes[current_type] = prefixes - - # loop over all possible variables and add them to list - for index in indices: - field_info_list = [] - for current_type in data_types: - # get dictionary of existing config variables to use - search_prefixes = dt_search_prefixes[current_type] - field_configs = get_field_config_variables(config, - index, - search_prefixes) - - field_info = format_var_items(field_configs, time_info) - if not isinstance(field_info, dict): - config.logger.error(f'Could not process {current_type}_' - f'VAR{index} variables: {field_info}') - continue - - field_info['data_type'] = current_type.lower() - field_info_list.append(field_info) - - # check that all fields types were found - if not field_info_list or len(data_types) != len(field_info_list): - continue - - # check if number of levels for each field type matches - n_levels = len(field_info_list[0]['levels']) - if len(data_types) > 1: - if (n_levels != len(field_info_list[1]['levels'])): - continue - - # if requested, put all field levels in a single item - if levels_as_list: - var_dict = {} - for field_info in field_info_list: - current_type = field_info.get('data_type') - var_dict[f"{current_type}_name"] = field_info.get('name') - var_dict[f"{current_type}_level"] = field_info.get('levels') - var_dict[f"{current_type}_thresh"] = field_info.get('thresh') - var_dict[f"{current_type}_extra"] = field_info.get('extra') - var_dict[f"{current_type}_output_name"] = field_info.get('output_names') - - var_dict['index'] = index - var_list.append(var_dict) - continue - - # loop over levels and add all values to output dictionary - for level_index in range(n_levels): - var_dict = {} - - # get level values to use for string substitution in name - # used for python embedding calls that read the level value - sub_info = {} - for field_info in field_info_list: - dt_level = f"{field_info.get('data_type')}_level" - sub_info[dt_level] = field_info.get('levels')[level_index] - - for field_info in field_info_list: - current_type = field_info.get('data_type') - name = field_info.get('name') - level = field_info.get('levels')[level_index] - thresh = field_info.get('thresh') - extra = field_info.get('extra') - output_name = field_info.get('output_names')[level_index] - - # substitute level in name if filename template is specified - subbed_name = do_string_sub(name, - skip_missing_tags=True, - **sub_info) - - var_dict[f"{current_type}_name"] = subbed_name - var_dict[f"{current_type}_level"] = level - var_dict[f"{current_type}_thresh"] = thresh - var_dict[f"{current_type}_extra"] = extra - var_dict[f"{current_type}_output_name"] = output_name - - var_dict['index'] = index - var_list.append(var_dict) - - # extra debugging information used for developer debugging only - ''' - for v in var_list: - config.logger.debug(f"VAR{v['index']}:") - if 'fcst_name' in v.keys(): - config.logger.debug(" fcst_name:"+v['fcst_name']) - config.logger.debug(" fcst_level:"+v['fcst_level']) - if 'fcst_thresh' in v.keys(): - config.logger.debug(" fcst_thresh:"+str(v['fcst_thresh'])) - if 'fcst_extra' in v.keys(): - config.logger.debug(" fcst_extra:"+v['fcst_extra']) - if 'fcst_output_name' in v.keys(): - config.logger.debug(" fcst_output_name:"+v['fcst_output_name']) - if 'obs_name' in v.keys(): - config.logger.debug(" obs_name:"+v['obs_name']) - config.logger.debug(" obs_level:"+v['obs_level']) - if 'obs_thresh' in v.keys(): - config.logger.debug(" obs_thresh:"+str(v['obs_thresh'])) - if 'obs_extra' in v.keys(): - config.logger.debug(" obs_extra:"+v['obs_extra']) - if 'obs_output_name' in v.keys(): - config.logger.debug(" obs_output_name:"+v['obs_output_name']) - if 'ens_name' in v.keys(): - config.logger.debug(" ens_name:"+v['ens_name']) - config.logger.debug(" ens_level:"+v['ens_level']) - if 'ens_thresh' in v.keys(): - config.logger.debug(" ens_thresh:"+str(v['ens_thresh'])) - if 'ens_extra' in v.keys(): - config.logger.debug(" ens_extra:"+v['ens_extra']) - if 'ens_output_name' in v.keys(): - config.logger.debug(" ens_output_name:"+v['ens_output_name']) - ''' - return sorted(var_list, key=lambda x: x['index']) - def sub_var_info(var_info, time_info): if not var_info: return {} @@ -2812,18 +1630,6 @@ def is_python_script(name): return False -def check_user_environment(config): - """!Check if any environment variables set in [user_env_vars] are already set in - the user's environment. Warn them that it will be overwritten from the conf if it is""" - if not config.has_section('user_env_vars'): - return - - for env_var in config.keys('user_env_vars'): - if env_var in os.environ: - msg = '{} is already set in the environment. '.format(env_var) +\ - 'Overwriting from conf file' - config.logger.warning(msg) - def expand_int_string_to_list(int_string): """! Expand string into a list of integer values. Items are separated by commas. Items that are formatted X-Y will be expanded into each number diff --git a/metplus/util/string_template_substitution.py b/metplus/util/string_template_substitution.py index 05dd19fc73..4438a1642c 100644 --- a/metplus/util/string_template_substitution.py +++ b/metplus/util/string_template_substitution.py @@ -827,11 +827,3 @@ def add_offset_matches_to_output_dict(match_dict, output_dict): offset = int(value) output_dict['offset_hours'] = offset - -def extract_lead(template, filename): - new_template = template - new_template = new_template.replace('/', '\/').replace('.', '\.') - match_tags = re.findall(r'{(.*?)}', new_template) - for match_tag in match_tags: - if match_tag.split('?') != 'lead': - new_template = new_template.replace('{' + match_tag + '}', '.*') diff --git a/metplus/wrappers/ascii2nc_wrapper.py b/metplus/wrappers/ascii2nc_wrapper.py index 01e675833f..e68d78ac23 100755 --- a/metplus/wrappers/ascii2nc_wrapper.py +++ b/metplus/wrappers/ascii2nc_wrapper.py @@ -76,11 +76,8 @@ def create_c_dict(self): ) # MET config variables - self.handle_time_summary_legacy(c_dict, - ['TIME_SUMMARY_GRIB_CODES', - 'TIME_SUMMARY_VAR_NAMES', - 'TIME_SUMMARY_TYPES'] - ) + self.handle_time_summary_dict() + self.handle_time_summary_legacy() # handle file window variables for edge in ['BEGIN', 'END']: @@ -100,34 +97,115 @@ def create_c_dict(self): return c_dict + def handle_time_summary_legacy(self): + """! Read METplusConfig variables for the MET config time_summary + dictionary and format values into environment variable + METPLUS_TIME_SUMMARY_DICT as well as other environment variables + that contain individuals items of the time_summary dictionary + that were referenced in wrapped MET config files prior to METplus 4.0. + Developer note: If we discontinue support for legacy wrapped MET + config files + + @param c_dict dictionary to store time_summary item values + @param remove_bracket_list (optional) list of items that need the + square brackets around the value removed because the legacy (pre 4.0) + wrapped MET config includes square braces around the environment + variable. + """ + # handle legacy time summary variables + self.add_met_config(name='', + data_type='bool', + env_var_name='TIME_SUMMARY_FLAG', + metplus_configs=['ASCII2NC_TIME_SUMMARY_FLAG']) + + self.add_met_config(name='', + data_type='bool', + env_var_name='TIME_SUMMARY_RAW_DATA', + metplus_configs=['ASCII2NC_TIME_SUMMARY_RAW_DATA']) + + self.add_met_config(name='', + data_type='string', + env_var_name='TIME_SUMMARY_BEG', + metplus_configs=['ASCII2NC_TIME_SUMMARY_BEG']) + + self.add_met_config(name='', + data_type='string', + env_var_name='TIME_SUMMARY_END', + metplus_configs=['ASCII2NC_TIME_SUMMARY_END']) + + self.add_met_config(name='', + data_type='int', + env_var_name='TIME_SUMMARY_STEP', + metplus_configs=['ASCII2NC_TIME_SUMMARY_STEP']) + + self.add_met_config(name='', + data_type='string', + env_var_name='TIME_SUMMARY_WIDTH', + metplus_configs=['ASCII2NC_TIME_SUMMARY_WIDTH'], + extra_args={'remove_quotes': True}) + + self.add_met_config(name='', + data_type='list', + env_var_name='TIME_SUMMARY_GRIB_CODES', + metplus_configs=['ASCII2NC_TIME_SUMMARY_GRIB_CODES', + 'ASCII2NC_TIME_SUMMARY_GRIB_CODE'], + extra_args={'remove_quotes': True, + 'allow_empty': True}) + + self.add_met_config(name='', + data_type='list', + env_var_name='TIME_SUMMARY_VAR_NAMES', + metplus_configs=['ASCII2NC_TIME_SUMMARY_OBS_VAR', + 'ASCII2NC_TIME_SUMMARY_VAR_NAMES'], + extra_args={'allow_empty': True}) + + self.add_met_config(name='', + data_type='list', + env_var_name='TIME_SUMMARY_TYPES', + metplus_configs=['ASCII2NC_TIME_SUMMARY_TYPE', + 'ASCII2NC_TIME_SUMMARY_TYPES'], + extra_args={'allow_empty': True}) + + self.add_met_config(name='', + data_type='int', + env_var_name='TIME_SUMMARY_VALID_FREQ', + metplus_configs=['ASCII2NC_TIME_SUMMARY_VLD_FREQ', + 'ASCII2NC_TIME_SUMMARY_VALID_FREQ']) + + self.add_met_config(name='', + data_type='float', + env_var_name='TIME_SUMMARY_VALID_THRESH', + metplus_configs=['ASCII2NC_TIME_SUMMARY_VLD_THRESH', + 'ASCII2NC_TIME_SUMMARY_VALID_THRESH']) + def set_environment_variables(self, time_info): """!Set environment variables that will be read by the MET config file. Reformat as needed. Print list of variables that were set and their values. Args: @param time_info dictionary containing timing info from current run""" - # set environment variables needed for MET application + # set environment variables needed for legacy MET config file self.add_env_var('TIME_SUMMARY_FLAG', - self.c_dict['TIME_SUMMARY_FLAG']) + self.env_var_dict.get('METPLUS_TIME_SUMMARY_FLAG', '')) self.add_env_var('TIME_SUMMARY_RAW_DATA', - self.c_dict['TIME_SUMMARY_RAW_DATA']) + self.env_var_dict.get('METPLUS_TIME_SUMMARY_RAW_DATA', '')) self.add_env_var('TIME_SUMMARY_BEG', - self.c_dict['TIME_SUMMARY_BEG']) + self.env_var_dict.get('METPLUS_TIME_SUMMARY_BEG', '')) self.add_env_var('TIME_SUMMARY_END', - self.c_dict['TIME_SUMMARY_END']) + self.env_var_dict.get('METPLUS_TIME_SUMMARY_END', '')) self.add_env_var('TIME_SUMMARY_STEP', - self.c_dict['TIME_SUMMARY_STEP']) + self.env_var_dict.get('METPLUS_TIME_SUMMARY_STEP', '')) self.add_env_var('TIME_SUMMARY_WIDTH', - self.c_dict['TIME_SUMMARY_WIDTH']) + self.env_var_dict.get('METPLUS_TIME_SUMMARY_WIDTH', '')) self.add_env_var('TIME_SUMMARY_GRIB_CODES', - self.c_dict['TIME_SUMMARY_GRIB_CODES']) + self.env_var_dict.get('METPLUS_TIME_SUMMARY_GRIB_CODES', '').strip('[]')) self.add_env_var('TIME_SUMMARY_VAR_NAMES', - self.c_dict['TIME_SUMMARY_VAR_NAMES']) + self.env_var_dict.get('METPLUS_TIME_SUMMARY_VAR_NAMES', '').strip('[]')) self.add_env_var('TIME_SUMMARY_TYPES', - self.c_dict['TIME_SUMMARY_TYPES']) + self.env_var_dict.get('METPLUS_TIME_SUMMARY_TYPES', '').strip('[]')) self.add_env_var('TIME_SUMMARY_VALID_FREQ', - self.c_dict['TIME_SUMMARY_VALID_FREQ']) + self.env_var_dict.get('METPLUS_TIME_SUMMARY_VALID_FREQ', '')) self.add_env_var('TIME_SUMMARY_VALID_THRESH', - self.c_dict['TIME_SUMMARY_VALID_THRESH']) + self.env_var_dict.get('METPLUS_TIME_SUMMARY_VALID_THRESH', '')) # set user environment variables super().set_environment_variables(time_info) diff --git a/metplus/wrappers/command_builder.py b/metplus/wrappers/command_builder.py index 22b1052721..47208ae2fc 100755 --- a/metplus/wrappers/command_builder.py +++ b/metplus/wrappers/command_builder.py @@ -21,8 +21,11 @@ from ..util import met_util as util from ..util import do_string_sub, ti_calculate, get_seconds_from_string from ..util import config_metplus -from ..util import METConfigInfo as met_config +from ..util import METConfig from ..util import MISSING_DATA_VALUE +from ..util import get_custom_string_list +from ..util import get_wrapped_met_config_file, add_met_config_item, format_met_config +from ..util.met_config import format_regrid_to_grid, add_met_config_dict # pylint:disable=pointless-string-statement '''!@namespace CommandBuilder @@ -176,8 +179,8 @@ def create_c_dict(self): if hasattr(self, 'app_name'): app_name = self.app_name - c_dict['CUSTOM_LOOP_LIST'] = util.get_custom_string_list(self.config, - app_name) + c_dict['CUSTOM_LOOP_LIST'] = get_custom_string_list(self.config, + app_name) c_dict['SKIP_TIMES'] = util.get_skip_times(self.config, app_name) @@ -280,20 +283,6 @@ def set_user_environment(self, time_info): **time_info) self.add_env_var(env_var, env_var_value) - @staticmethod - def format_regrid_to_grid(to_grid): - to_grid = to_grid.strip('"') - if not to_grid: - to_grid = 'NONE' - - # if NONE, FCST, or OBS force uppercase, otherwise add quotes - if to_grid.upper() in ['NONE', 'FCST', 'OBS']: - to_grid = to_grid.upper() - else: - to_grid = f'"{to_grid}"' - - return to_grid - def print_all_envs(self, print_copyable=True, print_each_item=True): """! Create list of log messages that output all environment variables that were set by this wrapper. @@ -1448,297 +1437,6 @@ def set_time_dict_for_single_runtime(self): return time_info - def _get_config_or_default(self, mp_config_name, get_function, - default=None): - conf_value = '' - - # if no possible METplus config variables are not set - if mp_config_name is None: - # if no default defined, return without doing anything - if not default: - return None - - # if METplus config variable is set, read the value - else: - conf_value = get_function('config', - mp_config_name, - '') - - # if variable is not set and there is a default defined, set default - if not conf_value and default: - conf_value = default - - return conf_value - - def set_met_config_list(self, c_dict, mp_config, met_config_name, - c_dict_key=None, **kwargs): - """! Get list from METplus configuration file and format it to be passed - into a MET configuration file. Set c_dict item with formatted string. - Args: - @param c_dict configuration dictionary to set - @param mp_config_name METplus configuration variable name. Assumed to be - in the [config] section. Value can be a comma-separated list of items. - @param met_config name of MET configuration variable to set. Also used - to determine the key in c_dict to set (upper-case) - @param c_dict_key optional argument to specify c_dict key to store result. If - set to None (default) then use upper-case of met_config_name - @param allow_empty if True, if METplus config variable is set - but is an empty string, then set the c_dict value to an empty - list. If False, behavior is the same as when the variable is - not set at all, which is to not set anything for the c_dict - value - @param remove_quotes if True, output value without quotes. - Default value is False - @param default (Optional) if set, use this value as default - if config is not set - """ - mp_config_name = self.get_mp_config_name(mp_config) - conf_value = self._get_config_or_default( - mp_config_name, - get_function=self.config.getraw, - default=kwargs.get('default') - ) - if conf_value is None: - return - - # convert value from config to a list - conf_values = util.getlist(conf_value) - if conf_values or kwargs.get('allow_empty', False): - out_values = [] - for conf_value in conf_values: - remove_quotes = kwargs.get('remove_quotes', False) - # if not removing quotes, escape any quotes found in list items - if not remove_quotes: - conf_value = conf_value.replace('"', '\\"') - - conf_value = util.remove_quotes(conf_value) - if not remove_quotes: - conf_value = f'"{conf_value}"' - - out_values.append(conf_value) - out_value = f"[{', '.join(out_values)}]" - - if not c_dict_key: - c_key = met_config_name.upper() - else: - c_key = c_dict_key - - conf_value = f'{met_config_name} = {out_value};' - c_dict[c_key] = conf_value - - def set_met_config_string(self, c_dict, mp_config, met_config_name, - c_dict_key=None, **kwargs): - """! Get string from METplus configuration file and format it to be passed - into a MET configuration file. Set c_dict item with formatted string. - - @param c_dict configuration dictionary to set - @param mp_config METplus configuration variable name. Assumed to be - in the [config] section. Value can be a comma-separated list of items. - @param met_config_name name of MET configuration variable to set. Also used - to determine the key in c_dict to set (upper-case) - @param c_dict_key optional argument to specify c_dict key to store result. If - set to None (default) then use upper-case of met_config_name - @param remove_quotes if True, output value without quotes. - Default value is False - @param to_grid if True, format to_grid value - Default value is False - @param default (Optional) if set, use this value as default - if config is not set - """ - mp_config_name = self.get_mp_config_name(mp_config) - conf_value = self._get_config_or_default( - mp_config_name, - get_function=self.config.getraw, - default=kwargs.get('default') - ) - if not conf_value: - return - - conf_value = util.remove_quotes(conf_value) - # add quotes back if remote quotes is False - if not kwargs.get('remove_quotes'): - conf_value = f'"{conf_value}"' - - if kwargs.get('uppercase', False): - conf_value = conf_value.upper() - - if kwargs.get('to_grid', False): - conf_value = self.format_regrid_to_grid(conf_value) - - c_key = c_dict_key if c_dict_key else met_config_name.upper() - c_dict[c_key] = f'{met_config_name} = {conf_value};' - - def set_met_config_number(self, c_dict, num_type, mp_config, - met_config_name, c_dict_key=None, **kwargs): - """! Get integer from METplus configuration file and format it to be passed - into a MET configuration file. Set c_dict item with formatted string. - Args: - @param c_dict configuration dictionary to set - @param num_type type of number to get from config. If set to 'int', call - getint function. If not, call getfloat function. - @param mp_config METplus configuration variable name. Assumed to be - in the [config] section. Value can be a comma-separated list of items. - @param met_config_name name of MET configuration variable to set. Also used - to determine the key in c_dict to set (upper-case) if c_dict_key is None - @param c_dict_key optional argument to specify c_dict key to store result. If - set to None (default) then use upper-case of met_config_name - @param default (Optional) if set, use this value as default - if config is not set - """ - mp_config_name = self.get_mp_config_name(mp_config) - if mp_config_name is None: - return - - if num_type == 'int': - conf_value = self.config.getint('config', mp_config_name) - else: - conf_value = self.config.getfloat('config', mp_config_name) - - if conf_value is None: - self.isOK = False - elif conf_value != util.MISSING_DATA_VALUE: - if not c_dict_key: - c_key = met_config_name.upper() - else: - c_key = c_dict_key - - c_dict[c_key] = f"{met_config_name} = {str(conf_value)};" - - def set_met_config_int(self, c_dict, mp_config_name, met_config_name, - c_dict_key=None, **kwargs): - self.set_met_config_number(c_dict, 'int', - mp_config_name, - met_config_name, - c_dict_key=c_dict_key, - **kwargs) - - def set_met_config_float(self, c_dict, mp_config_name, - met_config_name, c_dict_key=None, **kwargs): - self.set_met_config_number(c_dict, 'float', - mp_config_name, - met_config_name, - c_dict_key=c_dict_key, - **kwargs) - - def set_met_config_thresh(self, c_dict, mp_config, met_config_name, - c_dict_key=None, **kwargs): - mp_config_name = self.get_mp_config_name(mp_config) - if mp_config_name is None: - return - - conf_value = self.config.getstr('config', mp_config_name, '') - if conf_value: - if util.get_threshold_via_regex(conf_value) is None: - self.log_error(f"Incorrectly formatted threshold: {mp_config_name}") - return - - if not c_dict_key: - c_key = met_config_name.upper() - else: - c_key = c_dict_key - - c_dict[c_key] = f"{met_config_name} = {str(conf_value)};" - - def set_met_config_bool(self, c_dict, mp_config, met_config_name, - c_dict_key=None, **kwargs): - """! Get boolean from METplus configuration file and format it to be - passed into a MET configuration file. Set c_dict item with boolean - value expressed as a string. - Args: - @param c_dict configuration dictionary to set - @param mp_config METplus configuration variable name. - Assumed to be in the [config] section. - @param met_config_name name of MET configuration variable to - set. Also used to determine the key in c_dict to set - (upper-case) - @param c_dict_key optional argument to specify c_dict key to - store result. If set to None (default) then use upper-case of - met_config_name - @param uppercase If true, set value to TRUE or FALSE - """ - mp_config_name = self.get_mp_config_name(mp_config) - if mp_config_name is None: - return - conf_value = self.config.getbool('config', mp_config_name, '') - if conf_value is None: - self.log_error(f'Invalid boolean value set for {mp_config_name}') - return - - # if not invalid but unset, return without setting c_dict with no error - if conf_value == '': - return - - conf_value = str(conf_value) - if kwargs.get('uppercase', True): - conf_value = conf_value.upper() - - if not c_dict_key: - c_key = met_config_name.upper() - else: - c_key = c_dict_key - - c_dict[c_key] = (f'{met_config_name} = ' - f'{util.remove_quotes(conf_value)};') - - def get_mp_config_name(self, mp_config): - """! Get first name of METplus config variable that is set. - - @param mp_config list of METplus config keys to check. Can also be a - single item - @returns Name of first METplus config name in list that is set in the - METplusConfig object. None if none keys in the list are set. - """ - if not isinstance(mp_config, list): - mp_configs = [mp_config] - else: - mp_configs = mp_config - - for mp_config_name in mp_configs: - if self.config.has_option('config', mp_config_name): - return mp_config_name - - return None - - @staticmethod - def format_met_config(data_type, c_dict, name, keys=None): - """! Return formatted variable named with any if they - are set to a value. If none of the items are set, return empty string - - @param data_type type of value to format - @param c_dict config dictionary to read values from - @param name name of dictionary to create - @param keys list of c_dict keys to use if they are set. If unset (None) - then read all keys from c_dict - @returns MET config formatted dictionary/list - if any items are set, or empty string if not - """ - values = [] - if keys is None: - keys = c_dict.keys() - - for key in keys: - value = c_dict.get(key) - if value: - values.append(str(value)) - - # if none of the keys are set to a value in dict, return empty string - if not values: - return '' - - output = ''.join(values) - # add curly braces if dictionary - if 'dict' in data_type: - output = f"{{{output}}}" - - # add square braces if list - if 'list' in data_type: - output = f"[{output}];" - - # if name is not empty, add variable name and equals sign - if name: - output = f'{name} = {output}' - return output - @staticmethod def format_met_config_dict(c_dict, name, keys=None): """! Return formatted dictionary named with any if they @@ -1751,59 +1449,28 @@ def format_met_config_dict(c_dict, name, keys=None): @returns MET config formatted dictionary if any items are set, or empty string if not """ - return CommandBuilder.format_met_config('dict', c_dict=c_dict, name=name, keys=keys) + return format_met_config('dict', c_dict=c_dict, name=name, keys=keys) def handle_regrid(self, c_dict, set_to_grid=True): - app_name_upper = self.app_name.upper() - - # dictionary to hold regrid values as they are read - tmp_dict = {} - + dict_items = {} if set_to_grid: - conf_value = ( - self.config.getstr('config', - f'{app_name_upper}_REGRID_TO_GRID', '') + dict_items['to_grid'] = ('string', 'to_grid') + + # handle legacy format of to_grid + self.add_met_config( + name='', + data_type='string', + env_var_name='REGRID_TO_GRID', + metplus_configs=[f'{self.app_name.upper()}_REGRID_TO_GRID'], + extra_args={'to_grid': True}, + output_dict=c_dict, ) - # set to_grid without formatting for backwards compatibility - formatted_to_grid = self.format_regrid_to_grid(conf_value) - c_dict['REGRID_TO_GRID'] = formatted_to_grid - - if conf_value: - tmp_dict['REGRID_TO_GRID'] = ( - f"to_grid = {formatted_to_grid};" - ) - - self.set_met_config_string(tmp_dict, - f'{app_name_upper}_REGRID_METHOD', - 'method', - c_dict_key='REGRID_METHOD', - remove_quotes=True) - - self.set_met_config_int(tmp_dict, - f'{app_name_upper}_REGRID_WIDTH', - 'width', - c_dict_key='REGRID_WIDTH') - - self.set_met_config_float(tmp_dict, - f'{app_name_upper}_REGRID_VLD_THRESH', - 'vld_thresh', - c_dict_key='REGRID_VLD_THRESH') - self.set_met_config_string(tmp_dict, - f'{app_name_upper}_REGRID_SHAPE', - 'shape', - c_dict_key='REGRID_SHAPE', - remove_quotes=True) - - regrid_string = self.format_met_config_dict(tmp_dict, - 'regrid', - ['REGRID_TO_GRID', - 'REGRID_METHOD', - 'REGRID_WIDTH', - 'REGRID_VLD_THRESH', - 'REGRID_SHAPE', - ]) - self.env_var_dict['METPLUS_REGRID_DICT'] = regrid_string + dict_items['method'] = ('string', 'uppercase,remove_quotes') + dict_items['width'] = 'int' + dict_items['vld_thresh'] = 'float' + dict_items['shape'] = ('string', 'uppercase,remove_quotes') + self.add_met_config_dict('regrid', dict_items) def handle_description(self): """! Get description from config. If _DESC is set, use @@ -1859,29 +1526,6 @@ def get_output_prefix(self, time_info=None, set_env_vars=True): return output_prefix - def _parse_extra_args(self, extra): - """! Check string for extra option keywords and set them to True in - dictionary if they are found. Supports 'remove_quotes', 'uppercase' - and 'allow_empty' - - @param extra string to parse for keywords - @returns dictionary with extra args set if found in string - """ - extra_args = {} - if not extra: - return extra_args - - VALID_EXTRAS = ( - 'remove_quotes', - 'uppercase', - 'allow_empty', - 'to_grid', - ) - for extra_option in VALID_EXTRAS: - if extra_option in extra: - extra_args[extra_option] = True - return extra_args - def handle_climo_dict(self): """! Read climo mean/stdev variables with and set env_var_dict appropriately. Handle previous environment variables that are used @@ -1908,7 +1552,7 @@ def handle_climo_dict(self): # make sure _FILE_NAME is set from INPUT_TEMPLATE/DIR if used self.read_climo_file_name(climo_type) - self.handle_met_config_dict(dict_name, items) + self.add_met_config_dict(dict_name, items) # handle deprecated env vars CLIMO_MEAN_FILE and CLIMO_STDEV_FILE # that are used by pre v4.0.0 wrapped MET config files @@ -2021,41 +1665,24 @@ def handle_flags(self, flag_type): if not hasattr(self, f'{flag_type_upper}_FLAGS'): return - tmp_dict = {} - flag_list = [] + flag_info_dict = {} for flag in getattr(self, f'{flag_type_upper}_FLAGS'): - flag_name = f'{flag_type_upper}_FLAG_{flag.upper()}' - flag_list.append(flag_name) - self.set_met_config_string(tmp_dict, - f'{self.app_name.upper()}_{flag_name}', - flag, - c_dict_key=f'{flag_name}', - remove_quotes=True, - uppercase=True) - - flag_fmt = ( - self.format_met_config_dict(tmp_dict, - f'{flag_type_lower}_flag', - flag_list) - ) - self.env_var_dict[f'METPLUS_{flag_type_upper}_FLAG_DICT'] = flag_fmt + flag_info_dict[flag] = ('string', 'remove_quotes,uppercase') + + self.add_met_config_dict(f'{flag_type_lower}_flag', flag_info_dict) def handle_censor_val_and_thresh(self): """! Read {APP_NAME}_CENSOR_[VAL/THRESH] and set METPLUS_CENSOR_[VAL/THRESH] in self.env_var_dict so it can be referenced in a MET config file """ - self.set_met_config_list(self.env_var_dict, - f'{self.app_name.upper()}_CENSOR_THRESH', - 'censor_thresh', - c_dict_key='METPLUS_CENSOR_THRESH', - remove_quotes=True) - - self.set_met_config_list(self.env_var_dict, - f'{self.app_name.upper()}_CENSOR_VAL', - 'censor_val', - c_dict_key='METPLUS_CENSOR_VAL', - remove_quotes=True) + self.add_met_config(name='censor_thresh', + data_type='list', + extra_args={'remove_quotes': True}) + + self.add_met_config(name='censor_val', + data_type='list', + extra_args={'remove_quotes': True}) def get_env_var_value(self, env_var_name, read_dict=None, item_type=None): """! Read env var value, get text after the equals sign and remove the @@ -2086,7 +1713,7 @@ def handle_time_summary_dict(self): METPLUS_TIME_SUMMARY_DICT that is referenced in the wrapped MET config files. """ - self.handle_met_config_dict('time_summary', { + self.add_met_config_dict('time_summary', { 'flag': 'bool', 'raw_data': 'bool', 'beg': 'string', @@ -2102,105 +1729,6 @@ def handle_time_summary_dict(self): 'vld_thresh': ('float', None, None, ['TIME_SUMMARY_VALID_THRESH']), }) - def handle_time_summary_legacy(self, c_dict, remove_bracket_list=None): - """! Read METplusConfig variables for the MET config time_summary - dictionary and format values into environment variable - METPLUS_TIME_SUMMARY_DICT as well as other environment variables - that contain individuals items of the time_summary dictionary - that were referenced in wrapped MET config files prior to METplus 4.0. - Developer note: If we discontinue support for legacy wrapped MET - config files - - @param c_dict dictionary to store time_summary item values - @param remove_bracket_list (optional) list of items that need the - square brackets around the value removed because the legacy (pre 4.0) - wrapped MET config includes square braces around the environment - variable. - """ - tmp_dict = {} - app = self.app_name.upper() - self.set_met_config_bool(tmp_dict, - f'{app}_TIME_SUMMARY_FLAG', - 'flag', - 'TIME_SUMMARY_FLAG') - - self.set_met_config_bool(tmp_dict, - f'{app}_TIME_SUMMARY_RAW_DATA', - 'raw_data', - 'TIME_SUMMARY_RAW_DATA') - - self.set_met_config_string(tmp_dict, - f'{app}_TIME_SUMMARY_BEG', - 'beg', - 'TIME_SUMMARY_BEG') - - self.set_met_config_string(tmp_dict, - f'{app}_TIME_SUMMARY_END', - 'end', - 'TIME_SUMMARY_END') - - self.set_met_config_int(tmp_dict, - f'{app}_TIME_SUMMARY_STEP', - 'step', - 'TIME_SUMMARY_STEP') - - self.set_met_config_string(tmp_dict, - f'{app}_TIME_SUMMARY_WIDTH', - 'width', - 'TIME_SUMMARY_WIDTH', - remove_quotes=True) - - self.set_met_config_list(tmp_dict, - [f'{app}_TIME_SUMMARY_GRIB_CODES', - f'{app}_TIME_SUMMARY_GRIB_CODE'], - 'grib_code', - 'TIME_SUMMARY_GRIB_CODES', - remove_quotes=True, - allow_empty=True) - - self.set_met_config_list(tmp_dict, - [f'{app}_TIME_SUMMARY_OBS_VAR', - f'{app}_TIME_SUMMARY_VAR_NAMES'], - 'obs_var', - 'TIME_SUMMARY_VAR_NAMES', - allow_empty=True) - - self.set_met_config_list(tmp_dict, - [f'{app}_TIME_SUMMARY_TYPE', - f'{app}_TIME_SUMMARY_TYPES'], - 'type', - 'TIME_SUMMARY_TYPES', - allow_empty=True) - - self.set_met_config_int(tmp_dict, - [f'{app}_TIME_SUMMARY_VLD_FREQ', - f'{app}_TIME_SUMMARY_VALID_FREQ'], - 'vld_freq', - 'TIME_SUMMARY_VALID_FREQ') - - self.set_met_config_float(tmp_dict, - [f'{app}_TIME_SUMMARY_VLD_THRESH', - f'{app}_TIME_SUMMARY_VALID_THRESH'], - 'vld_thresh', - 'TIME_SUMMARY_VALID_THRESH') - - time_summary = self.format_met_config_dict(tmp_dict, - 'time_summary', - keys=None) - self.env_var_dict['METPLUS_TIME_SUMMARY_DICT'] = time_summary - - # set c_dict values to support old method of setting env vars - for key, value in tmp_dict.items(): - c_dict[key] = self.get_env_var_value(key, read_dict=tmp_dict) - - # remove brackets [] from lists - if not remove_bracket_list: - return - - for list_value in remove_bracket_list: - if c_dict.get(list_value): - c_dict[list_value] = c_dict[list_value].strip('[]') - def handle_mask(self, single_value=False, get_flags=False): """! Read mask dictionary values and set them into env_var_list @@ -2221,85 +1749,9 @@ def handle_mask(self, single_value=False, get_flags=False): items['grid_flag'] = ('string', 'remove_quotes,uppercase') items['poly_flag'] = ('string', 'remove_quotes,uppercase') - self.handle_met_config_dict('mask', items) + self.add_met_config_dict('mask', items) - def set_met_config_function(self, item_type): - """! Return function to use based on item type - - @param item_type type of MET config variable to obtain - Valid values: list, string, int, float, thresh, bool - @returns function to use or None if invalid type provided - """ - if item_type == 'int': - return self.set_met_config_int - elif item_type == 'string': - return self.set_met_config_string - elif item_type == 'list': - return self.set_met_config_list - elif item_type == 'float': - return self.set_met_config_float - elif item_type == 'thresh': - return self.set_met_config_thresh - elif item_type == 'bool': - return self.set_met_config_bool - else: - self.log_error("Invalid argument for item type: " - f"{item_type}") - return None - - def handle_met_config_item(self, item, output_dict=None, depth=0): - """! Reads info from METConfigInfo object, gets value from - METplusConfig, and formats it based on the specifications. Sets - value in output dictionary with key starting with METPLUS_. - - @param item METConfigInfo object to read and determine what to get - @param output_dict (optional) dictionary to save formatted output - If unset, use self.env_var_dict. - @param depth counter to check if item being processed is nested within - another variable or not. If depth is 0, it is a top level variable. - This is used internally by this function and shouldn't be supplied - outside of calls within this function. - """ - if output_dict is None: - output_dict = self.env_var_dict - - env_var_name = item.env_var_name.upper() - if not env_var_name.startswith('METPLUS_'): - env_var_name = f'METPLUS_{env_var_name}' - - # handle dictionary or dictionary list item - if 'dict' in item.data_type: - tmp_dict = {} - for child in item.children: - if not self.handle_met_config_item(child, tmp_dict, - depth=depth+1): - return False - - dict_string = self.format_met_config(item.data_type, - tmp_dict, - item.name, - keys=None) - - # if handling dict MET config that is not nested inside another - if not depth and item.data_type == 'dict': - env_var_name = f'{env_var_name}_DICT' - - output_dict[env_var_name] = dict_string - return True - - # handle non-dictionary item - set_met_config = self.set_met_config_function(item.data_type) - if not set_met_config: - return False - - set_met_config(output_dict, - item.metplus_configs, - item.name, - c_dict_key=env_var_name, - **item.extra_args) - return True - - def handle_met_config_dict(self, dict_name, items): + def add_met_config_dict(self, dict_name, items): """! Read config variables for MET config dictionary and set env_var_dict with formatted values @@ -2308,127 +1760,30 @@ def handle_met_config_dict(self, dict_name, items): dictionary and the value is info about the item (see parse_item_info function for more information) """ - dict_items = [] - - # config prefix i.e GRID_STAT_CLIMO_MEAN_ - metplus_prefix = f'{self.app_name}_{dict_name}_'.upper() - for name, item_info in items.items(): - data_type, extra, kids, nicknames = self.parse_item_info(item_info) - - # config name i.e. GRID_STAT_CLIMO_MEAN_FILE_NAME - metplus_name = f'{metplus_prefix}{name.upper()}' - - # change (n) to _N i.e. distance_map.beta_value(n) - metplus_name = metplus_name.replace('(N)', '_N') - metplus_configs = [] - - if 'dict' not in data_type: - children = None - # if variable ends with _BEG, read _BEGIN first - if metplus_name.endswith('BEG'): - metplus_configs.append(f'{metplus_name}IN') - - metplus_configs.append(metplus_name) - if nicknames: - for nickname in nicknames: - metplus_configs.append( - f'{self.app_name}_{nickname}'.upper() - ) - - # if dictionary, read get children from MET config - else: - children = [] - for kid_name, kid_info in kids.items(): - kid_upper = kid_name.upper() - kid_type, kid_extra, _, _ = self.parse_item_info(kid_info) - - metplus_configs.append(f'{metplus_name}_{kid_upper}') - metplus_configs.append(f'{metplus_prefix}{kid_upper}') - - kid_args = self._parse_extra_args(kid_extra) - child_item = self.get_met_config( - name=kid_name, - data_type=kid_type, - metplus_configs=metplus_configs.copy(), - extra_args=kid_args, - ) - children.append(child_item) - - # reset metplus config list for next kid - metplus_configs.clear() - - # set metplus_configs - metplus_configs = None - - extra_args = self._parse_extra_args(extra) - dict_item = ( - self.get_met_config( - name=name, - data_type=data_type, - metplus_configs=metplus_configs, - extra_args=extra_args, - children=children, - ) - ) - dict_items.append(dict_item) - - final_met_config = self.get_met_config( - name=dict_name, - data_type='dict', - children=dict_items, - ) - - return self.handle_met_config_item(final_met_config, self.env_var_dict) - - @staticmethod - def parse_item_info(item_info): - """! Parses info about a MET config dictionary item. The input can - be a single string that is the data type of the item. It can also be - a tuple containing 2 to 4 values. The additional values must be - supplied in order: - * extra: string of extra information about item, i.e. - 'remove_quotes', 'uppercase', or 'allow_empty' - * kids: dictionary describing child values (used only for dict items) - where the key is the name of the variable and the value is item info - for the child variable in the same format as item_info that is - parsed in this function - * nicknames: list of other METplus config variable name that can be - used to set a value. The app name i.e. GRID_STAT_ is prepended to - each nickname in the list. Used for backwards compatibility for - METplus config variables whose name does not match the MET config - variable name - - @param item_info string or tuple containing information about a - dictionary item - @returns tuple of data type, extra info, children, and nicknames or - None for each tuple value that is not set - """ - if isinstance(item_info, tuple): - data_type, *rest = item_info - else: - data_type = item_info - rest = [] - - extra = rest.pop(0) if rest else None - kids = rest.pop(0) if rest else None - nicknames = rest.pop(0) if rest else None + return_code = add_met_config_dict(config=self.config, + app_name=self.app_name, + output_dict=self.env_var_dict, + dict_name=dict_name, + items=items) + if not return_code: + self.isOK = False - return data_type, extra, kids, nicknames + return return_code - def handle_met_config_window(self, dict_name): + def add_met_config_window(self, dict_name): """! Handle a MET config window dictionary. It is assumed that the dictionary only contains 'beg' and 'end' entries that are integers. @param dict_name name of MET dictionary """ - self.handle_met_config_dict(dict_name, { + self.add_met_config_dict(dict_name, { 'beg': 'int', 'end': 'int', }) def add_met_config(self, **kwargs): - """! Create METConfigInfo object from arguments and process - @param kwargs key arguments that should match METConfigInfo + """! Create METConfig object from arguments and process + @param kwargs key arguments that should match METConfig arguments, which includes the following: @param name MET config variable name to set @param data_type type of variable to set, i.e. string, list, bool @@ -2444,17 +1799,10 @@ def add_met_config(self, **kwargs): kwargs['metplus_configs'] = [ f"{self.app_name}_{kwargs.get('name')}".upper() ] - item = met_config(**kwargs) - output_dict = kwargs.get('output_dict') - self.handle_met_config_item(item, output_dict) - - def get_met_config(self, **kwargs): - """! Get METConfigInfo object from arguments and return it - @param kwargs key arguments that should match METConfigInfo - arguments - @returns METConfigInfo object - """ - return met_config(**kwargs) + item = METConfig(**kwargs) + output_dict = kwargs.get('output_dict', self.env_var_dict) + if not add_met_config_item(self.config, item, output_dict): + self.isOK = False def get_config_file(self, default_config_file=None): """! Get the MET config file path for the wrapper from the @@ -2464,20 +1812,9 @@ def get_config_file(self, default_config_file=None): file found in parm/met_config to use if config file is not set @returns path to wrapped config file or None if no default is provided """ - config_name = f'{self.app_name.upper()}_CONFIG_FILE' - config_file = self.config.getraw('config', config_name, '') - if config_file: - return config_file - - if not default_config_file: - return None - - default_config_path = os.path.join(self.config.getdir('PARM_BASE'), - 'met_config', + return get_wrapped_met_config_file(self.config, + self.app_name, default_config_file) - self.logger.debug(f"{config_name} is not set. " - f"Using {default_config_path}") - return default_config_path def get_start_time_input_dict(self): """! Get the first run time specified in config. Used if only running diff --git a/metplus/wrappers/compare_gridded_wrapper.py b/metplus/wrappers/compare_gridded_wrapper.py index 244f95e99e..5a20d06530 100755 --- a/metplus/wrappers/compare_gridded_wrapper.py +++ b/metplus/wrappers/compare_gridded_wrapper.py @@ -14,6 +14,7 @@ from ..util import met_util as util from ..util import do_string_sub, ti_calculate +from ..util import parse_var_list from . import CommandBuilder '''!@namespace CompareGriddedWrapper @@ -50,8 +51,13 @@ def create_c_dict(self): which config variables are used in the wrapper""" c_dict = super().create_c_dict() - self.set_met_config_string(self.env_var_dict, 'MODEL', 'model', 'METPLUS_MODEL') - self.set_met_config_string(self.env_var_dict, 'OBTYPE', 'obtype', 'METPLUS_OBTYPE') + self.add_met_config(name='model', + data_type='string', + metplus_configs=['MODEL']) + + self.add_met_config(name='obtype', + data_type='string', + metplus_configs=['OBTYPE']) # set old MET config items for backwards compatibility c_dict['MODEL_OLD'] = self.config.getstr('config', 'MODEL', 'FCST') @@ -88,13 +94,11 @@ def create_c_dict(self): # handle window variables [FCST/OBS]_[FILE_]_WINDOW_[BEGIN/END] self.handle_file_window_variables(c_dict) - self.set_met_config_string(self.env_var_dict, - f'{self.app_name.upper()}_OUTPUT_PREFIX', - 'output_prefix', - 'METPLUS_OUTPUT_PREFIX') + self.add_met_config(name='output_prefix', + data_type='string') - c_dict['VAR_LIST_TEMP'] = util.parse_var_list(self.config, - met_tool=self.app_name) + c_dict['VAR_LIST_TEMP'] = parse_var_list(self.config, + met_tool=self.app_name) return c_dict @@ -111,7 +115,7 @@ def set_environment_variables(self, time_info): self.add_env_var('MODEL', self.c_dict.get('MODEL_OLD', '')) self.add_env_var('OBTYPE', self.c_dict.get('OBTYPE_OLD', '')) self.add_env_var('REGRID_TO_GRID', - self.c_dict.get('REGRID_TO_GRID', + self.c_dict.get('METPLUS_REGRID_TO_GRID', 'NONE')) super().set_environment_variables(time_info) @@ -400,7 +404,7 @@ def get_command(self): return cmd def handle_climo_cdf_dict(self): - self.handle_met_config_dict('climo_cdf', { + self.add_met_config_dict('climo_cdf', { 'cdf_bins': ('float', None, None, ['CLIMO_CDF_BINS']), 'center_bins': 'bool', 'write_bins': 'bool', @@ -425,4 +429,4 @@ def handle_interp_dict(self, uses_field=False): if uses_field: items['field'] = ('string', 'remove_quotes') - self.handle_met_config_dict('interp', items) + self.add_met_config_dict('interp', items) diff --git a/metplus/wrappers/ensemble_stat_wrapper.py b/metplus/wrappers/ensemble_stat_wrapper.py index 2c532ec05f..fea77b273d 100755 --- a/metplus/wrappers/ensemble_stat_wrapper.py +++ b/metplus/wrappers/ensemble_stat_wrapper.py @@ -16,6 +16,7 @@ from ..util import met_util as util from . import CompareGriddedWrapper from ..util import do_string_sub +from ..util import parse_var_list """!@namespace EnsembleStatWrapper @brief Wraps the MET tool ensemble_stat to compare ensemble datasets @@ -220,43 +221,39 @@ def create_c_dict(self): c_dict['MET_OBS_ERR_TABLE'] = \ self.config.getstr('config', 'ENSEMBLE_STAT_MET_OBS_ERR_TABLE', '') - self.set_met_config_float(self.env_var_dict, - 'ENSEMBLE_STAT_ENS_VLD_THRESH', - 'vld_thresh', - 'METPLUS_ENS_VLD_THRESH') - - self.set_met_config_list(self.env_var_dict, - 'ENSEMBLE_STAT_ENS_OBS_THRESH', - 'obs_thresh', - 'METPLUS_ENS_OBS_THRESH', - remove_quotes=True) - - self.set_met_config_float(self.env_var_dict, - 'ENSEMBLE_STAT_ENS_SSVAR_BIN_SIZE', - 'ens_ssvar_bin_size', - 'METPLUS_ENS_SSVAR_BIN_SIZE') - self.set_met_config_float(self.env_var_dict, - 'ENSEMBLE_STAT_ENS_PHIST_BIN_SIZE', - 'ens_phist_bin_size', - 'METPLUS_ENS_PHIST_BIN_SIZE') + self.add_met_config(name='vld_thresh', + data_type='float', + env_var_name='METPLUS_ENS_VLD_THRESH', + metplus_configs=['ENSEMBLE_STAT_ENS_VLD_THRESH', + 'ENSEMBLE_STAT_VLD_THRESH', + 'ENSEMBLE_STAT_ENS_VALID_THRESH', + 'ENSEMBLE_STAT_VALID_THRESH', + ]) + + self.add_met_config(name='obs_thresh', + data_type='list', + env_var_name='METPLUS_ENS_OBS_THRESH', + metplus_configs=['ENSEMBLE_STAT_ENS_OBS_THRESH', + 'ENSEMBLE_STAT_OBS_THRESH'], + extra_args={'remove_quotes': True}) + + self.add_met_config(name='ens_ssvar_bin_size', + data_type='float') + + self.add_met_config(name='ens_phist_bin_size', + data_type='float') self.handle_nbrhd_prob_dict() - self.set_met_config_float(self.env_var_dict, - 'ENSEMBLE_STAT_ENS_THRESH', - 'ens_thresh', - 'METPLUS_ENS_THRESH') + self.add_met_config(name='ens_thresh', + data_type='float') - self.set_met_config_string(self.env_var_dict, - 'ENSEMBLE_STAT_DUPLICATE_FLAG', - 'duplicate_flag', - 'METPLUS_DUPLICATE_FLAG', - remove_quotes=True) + self.add_met_config(name='duplicate_flag', + data_type='string', + extra_args={'remove_quotes': True}) - self.set_met_config_bool(self.env_var_dict, - 'ENSEMBLE_STAT_SKIP_CONST', - 'skip_const', - 'METPLUS_SKIP_CONST') + self.add_met_config(name='skip_const', + data_type='bool') # set climo_cdf dictionary variables self.handle_climo_cdf_dict() @@ -270,37 +267,32 @@ def create_c_dict(self): self.handle_flags('OUTPUT') self.handle_flags('ENSEMBLE') - self.set_met_config_bool(self.env_var_dict, - 'ENSEMBLE_STAT_OBS_ERROR_FLAG', - 'flag', - 'METPLUS_OBS_ERROR_FLAG') - self.set_met_config_list(self.env_var_dict, - 'ENSEMBLE_STAT_MASK_GRID', - 'grid', - 'METPLUS_MASK_GRID', - allow_empty=True) - self.set_met_config_list(self.env_var_dict, - 'ENSEMBLE_STAT_CI_ALPHA', - 'ci_alpha', - 'METPLUS_CI_ALPHA', - remove_quotes=True) - - self.set_met_config_list(self.env_var_dict, - 'ENSEMBLE_STAT_CENSOR_THRESH', - 'censor_thresh', - 'METPLUS_CENSOR_THRESH', - remove_quotes=True) - self.set_met_config_list(self.env_var_dict, - 'ENSEMBLE_STAT_CENSOR_VAL', - 'censor_val', - 'METPLUS_CENSOR_VAL', - remove_quotes=True) - - self.set_met_config_list(self.env_var_dict, - 'ENSEMBLE_STAT_MESSAGE_TYPE', - 'message_type', - 'METPLUS_MESSAGE_TYPE', - allow_empty=True) + self.add_met_config(name='flag', + data_type='bool', + env_var_name='METPLUS_OBS_ERROR_FLAG', + metplus_configs=['ENSEMBLE_STAT_OBS_ERROR_FLAG']) + + self.add_met_config(name='grid', + data_type='list', + env_var_name='METPLUS_MASK_GRID', + metplus_configs=['ENSEMBLE_STAT_MASK_GRID'], + extra_args={'allow_empty': True}) + + self.add_met_config(name='ci_alpha', + data_type='list', + extra_args={'remove_quotes': True}) + + self.add_met_config(name='censor_thresh', + data_type='list', + extra_args={'remove_quotes': True}) + + self.add_met_config(name='censor_val', + data_type='list', + extra_args={'remove_quotes': True}) + + self.add_met_config(name='message_type', + data_type='list', + extra_args={'allow_empty': True}) self.handle_obs_window_variables(c_dict) @@ -329,14 +321,14 @@ def create_c_dict(self): c_dict['VAR_LIST_OPTIONAL'] = True # parse var list for ENS fields - c_dict['ENS_VAR_LIST_TEMP'] = util.parse_var_list( + c_dict['ENS_VAR_LIST_TEMP'] = parse_var_list( self.config, data_type='ENS', met_tool=self.app_name ) # parse optional var list for FCST and/or OBS fields - c_dict['VAR_LIST_TEMP'] = util.parse_var_list( + c_dict['VAR_LIST_TEMP'] = parse_var_list( self.config, met_tool=self.app_name ) @@ -344,7 +336,7 @@ def create_c_dict(self): return c_dict def handle_nmep_smooth_dict(self): - self.handle_met_config_dict('nmep_smooth', { + self.add_met_config_dict('nmep_smooth', { 'vld_thresh': 'float', 'shape': ('string', 'uppercase,remove_quotes'), 'gaussian_dx': 'float', @@ -357,7 +349,7 @@ def handle_nmep_smooth_dict(self): }) def handle_nbrhd_prob_dict(self): - self.handle_met_config_dict('nbrhd_prob', { + self.add_met_config_dict('nbrhd_prob', { 'width': ('list', 'remove_quotes'), 'shape': ('string', 'uppercase,remove_quotes'), 'vld_thresh': 'float', diff --git a/metplus/wrappers/extract_tiles_wrapper.py b/metplus/wrappers/extract_tiles_wrapper.py index f9ad6dcc8a..6dd37a9684 100755 --- a/metplus/wrappers/extract_tiles_wrapper.py +++ b/metplus/wrappers/extract_tiles_wrapper.py @@ -15,6 +15,7 @@ from ..util import met_util as util from ..util import do_string_sub, ti_calculate +from ..util import parse_var_list from .regrid_data_plane_wrapper import RegridDataPlaneWrapper from . import CommandBuilder @@ -152,8 +153,8 @@ def create_c_dict(self): c_dict['LON_ADJ'] = self.config.getfloat('config', 'EXTRACT_TILES_LON_ADJ') - c_dict['VAR_LIST_TEMP'] = util.parse_var_list(self.config, - met_tool=self.app_name) + c_dict['VAR_LIST_TEMP'] = parse_var_list(self.config, + met_tool=self.app_name) return c_dict def regrid_data_plane_init(self): diff --git a/metplus/wrappers/gen_ens_prod_wrapper.py b/metplus/wrappers/gen_ens_prod_wrapper.py index 3beaebc66e..e8011fb0bd 100755 --- a/metplus/wrappers/gen_ens_prod_wrapper.py +++ b/metplus/wrappers/gen_ens_prod_wrapper.py @@ -118,7 +118,7 @@ def create_c_dict(self): metplus_configs=['DESC', 'GEN_ENS_PROD_DESC'], ) - self.handle_met_config_dict('regrid', { + self.add_met_config_dict('regrid', { 'to_grid': ('string', 'to_grid'), 'method': ('string', 'uppercase,remove_quotes'), 'width': 'int', @@ -176,13 +176,13 @@ def create_c_dict(self): extra_args={'remove_quotes': True, 'uppercase': True}) - self.handle_met_config_dict('nbrhd_prob', { + self.add_met_config_dict('nbrhd_prob', { 'width': ('list', 'remove_quotes'), 'shape': ('string', 'uppercase,remove_quotes'), 'vld_thresh': 'float', }) - self.handle_met_config_dict('nmep_smooth', { + self.add_met_config_dict('nmep_smooth', { 'vld_thresh': 'float', 'shape': ('string', 'uppercase,remove_quotes'), 'gaussian_dx': 'float', diff --git a/metplus/wrappers/grid_diag_wrapper.py b/metplus/wrappers/grid_diag_wrapper.py index b17de1dfd7..d6d306f758 100755 --- a/metplus/wrappers/grid_diag_wrapper.py +++ b/metplus/wrappers/grid_diag_wrapper.py @@ -16,6 +16,7 @@ from ..util import time_util from . import RuntimeFreqWrapper from ..util import do_string_sub +from ..util import parse_var_list '''!@namespace GridDiagWrapper @brief Wraps the Grid-Diag tool @@ -78,9 +79,9 @@ def create_c_dict(self): self.handle_censor_val_and_thresh() - c_dict['VAR_LIST_TEMP'] = util.parse_var_list(self.config, - data_type='FCST', - met_tool=self.app_name) + c_dict['VAR_LIST_TEMP'] = parse_var_list(self.config, + data_type='FCST', + met_tool=self.app_name) c_dict['MASK_POLY_TEMPLATE'] = self.read_mask_poly() diff --git a/metplus/wrappers/grid_stat_wrapper.py b/metplus/wrappers/grid_stat_wrapper.py index c020ae2dd0..2ad1d011a2 100755 --- a/metplus/wrappers/grid_stat_wrapper.py +++ b/metplus/wrappers/grid_stat_wrapper.py @@ -160,25 +160,31 @@ def create_c_dict(self): c_dict['ALLOW_MULTIPLE_FILES'] = False + self.add_met_config(name='cov_thresh', + data_type='list', + env_var_name='METPLUS_NBRHD_COV_THRESH', + metplus_configs=[ + 'GRID_STAT_NEIGHBORHOOD_COV_THRESH' + ], + extra_args={'remove_quotes': True}) + + self.add_met_config(name='width', + data_type='list', + env_var_name='METPLUS_NBRHD_WIDTH', + metplus_configs=[ + 'GRID_STAT_NEIGHBORHOOD_WIDTH' + ], + extra_args={'remove_quotes': True}) + + self.add_met_config(name='shape', + data_type='string', + env_var_name='METPLUS_NBRHD_SHAPE', + metplus_configs=[ + 'GRID_STAT_NEIGHBORHOOD_SHAPE' + ], + extra_args={'remove_quotes': True}) - self.set_met_config_list(self.env_var_dict, - f'GRID_STAT_NEIGHBORHOOD_COV_THRESH', - 'cov_thresh', - 'METPLUS_NBRHD_COV_THRESH', - remove_quotes=True) - - self.set_met_config_list(self.env_var_dict, - f'GRID_STAT_NEIGHBORHOOD_WIDTH', - 'width', - 'METPLUS_NBRHD_WIDTH', - remove_quotes=True) - - self.set_met_config_string(self.env_var_dict, - 'GRID_STAT_NEIGHBORHOOD_SHAPE', - 'shape', - 'METPLUS_NBRHD_SHAPE', - remove_quotes=True) - + # handle legacy environment variables used by old MET configs c_dict['NEIGHBORHOOD_WIDTH'] = ( self.config.getstr('config', 'GRID_STAT_NEIGHBORHOOD_WIDTH', '1') @@ -232,7 +238,7 @@ def create_c_dict(self): data_type='float', metplus_configs=['GRID_STAT_HSS_EC_VALUE']) - self.handle_met_config_dict('distance_map', { + self.add_met_config_dict('distance_map', { 'baddeley_p': 'int', 'baddeley_max_dist': 'float', 'fom_alpha': 'float', @@ -262,8 +268,9 @@ def set_environment_variables(self, time_info): self.add_env_var('NEIGHBORHOOD_SHAPE', self.c_dict['NEIGHBORHOOD_SHAPE']) + cov_thresh = self.get_env_var_value('METPLUS_NBRHD_COV_THRESH') self.add_env_var('NEIGHBORHOOD_COV_THRESH', - self.c_dict.get('NBRHD_COV_THRESH', '')) + cov_thresh) self.add_env_var('VERIF_MASK', self.c_dict.get('VERIFICATION_MASK', '')) diff --git a/metplus/wrappers/ioda2nc_wrapper.py b/metplus/wrappers/ioda2nc_wrapper.py index 3fecb4a4b0..bfdd2e42df 100755 --- a/metplus/wrappers/ioda2nc_wrapper.py +++ b/metplus/wrappers/ioda2nc_wrapper.py @@ -86,10 +86,10 @@ def create_c_dict(self): self.add_met_config(name='message_type_group_map', data_type='list', extra_args={'remove_quotes': True}) self.add_met_config(name='station_id', data_type='list') - self.handle_met_config_window('obs_window') + self.add_met_config_window('obs_window') self.handle_mask(single_value=True) - self.handle_met_config_window('elevation_range') - self.handle_met_config_window('level_range') + self.add_met_config_window('elevation_range') + self.add_met_config_window('level_range') self.add_met_config(name='obs_var', data_type='list') self.add_met_config(name='obs_name_map', data_type='list', extra_args={'remove_quotes': True}) diff --git a/metplus/wrappers/make_plots_wrapper.py b/metplus/wrappers/make_plots_wrapper.py index f33419f7ea..b73d660028 100755 --- a/metplus/wrappers/make_plots_wrapper.py +++ b/metplus/wrappers/make_plots_wrapper.py @@ -19,6 +19,7 @@ import itertools from ..util import met_util as util +from ..util import parse_var_list from . import CommandBuilder # handle if module can't be loaded to run wrapper @@ -120,7 +121,7 @@ def create_c_dict(self): c_dict['LOOP_LIST_ITEMS'] = util.getlist( self.config.getstr('config', 'LOOP_LIST_ITEMS') ) - c_dict['VAR_LIST'] = util.parse_var_list(self.config) + c_dict['VAR_LIST'] = parse_var_list(self.config) c_dict['MODEL_LIST'] = util.getlist( self.config.getstr('config', 'MODEL_LIST', '') ) diff --git a/metplus/wrappers/mode_wrapper.py b/metplus/wrappers/mode_wrapper.py index 0fd39b5387..af4237c4d7 100755 --- a/metplus/wrappers/mode_wrapper.py +++ b/metplus/wrappers/mode_wrapper.py @@ -150,15 +150,11 @@ def create_c_dict(self): ) c_dict['ONCE_PER_FIELD'] = True - self.set_met_config_bool(self.env_var_dict, - 'MODE_QUILT', - 'quilt', - 'METPLUS_QUILT') + self.add_met_config(name='quilt', + data_type='bool') - self.set_met_config_float(self.env_var_dict, - 'MODE_GRID_RES', - 'grid_res', - 'METPLUS_GRID_RES') + self.add_met_config(name='grid_res', + data_type='float') # if MODE_GRID_RES is not set, then unset the default values defaults = self.DEFAULT_VALUES.copy() @@ -168,83 +164,124 @@ def create_c_dict(self): # read forecast and observation field variables for data_type in ['FCST', 'OBS']: - self.set_met_config_list( - self.env_var_dict, - [f'{data_type}_MODE_CONV_RADIUS', - f'MODE_{data_type}_CONV_RADIUS', - 'MODE_CONV_RADIUS'], - 'conv_radius', - f'METPLUS_{data_type}_CONV_RADIUS', - remove_quotes=True, - default=defaults.get(f'{data_type}_CONV_RADIUS'), + self.add_met_config( + name='conv_radius', + data_type='list', + env_var_name=f'METPLUS_{data_type}_CONV_RADIUS', + metplus_configs=[f'{data_type}_MODE_CONV_RADIUS', + f'MODE_{data_type}_CONV_RADIUS', + 'MODE_CONV_RADIUS' + ], + extra_args={ + 'remove_quotes': True, + 'default': defaults.get(f'{data_type}_CONV_RADIUS') + } ) - self.set_met_config_list(self.env_var_dict, - [f'{data_type}_MODE_CONV_THRESH', - f'MODE_{data_type}_CONV_THRESH', - 'MODE_CONV_THRESH'], - 'conv_thresh', - f'METPLUS_{data_type}_CONV_THRESH', - remove_quotes=True) - - self.set_met_config_list(self.env_var_dict, - [f'{data_type}_MODE_MERGE_THRESH', - f'MODE_{data_type}_MERGE_THRESH', - 'MODE_MERGE_THRESH'], - 'merge_thresh', - f'METPLUS_{data_type}_MERGE_THRESH', - remove_quotes=True) - - self.set_met_config_string(self.env_var_dict, - [f'{data_type}_MODE_MERGE_FLAG', - f'MODE_{data_type}_MERGE_FLAG', - 'MODE_MERGE_FLAG'], - 'merge_flag', - f'METPLUS_{data_type}_MERGE_FLAG', - remove_quotes=True, - uppercase=True) - - self.set_met_config_list(self.env_var_dict, - [f'{data_type}_MODE_FILTER_ATTR_NAME', - f'MODE_{data_type}_FILTER_ATTR_NAME', - 'MODE_FILTER_ATTR_NAME'], - 'filter_attr_name', - f'METPLUS_{data_type}_FILTER_ATTR_NAME') - - self.set_met_config_list(self.env_var_dict, - [f'{data_type}_MODE_FILTER_ATTR_THRESH', - f'MODE_{data_type}_FILTER_ATTR_THRESH', - 'MODE_FILTER_ATTR_THRESH'], - 'filter_attr_thresh', - f'METPLUS_{data_type}_FILTER_ATTR_THRESH', - remove_quotes=True) - - self.set_met_config_list(self.env_var_dict, - [f'{data_type}_MODE_CENSOR_THRESH', - f'MODE_{data_type}_CENSOR_THRESH', - 'MODE_CENSOR_THRESH'], - 'censor_thresh', - f'METPLUS_{data_type}_CENSOR_THRESH', - remove_quotes=True) - - self.set_met_config_list(self.env_var_dict, - [f'{data_type}_MODE_CENSOR_VAL', - f'MODE_{data_type}_CENSOR_VAL', - f'{data_type}_MODE_CENSOR_VALUE', - f'MODE_{data_type}_CENSOR_VALUE', - 'MODE_CENSOR_VAL', - 'MODE_CENSOR_VALUE'], - 'censor_val', - f'METPLUS_{data_type}_CENSOR_VAL', - remove_quotes=True) - - self.set_met_config_float(self.env_var_dict, - [f'{data_type}_MODE_VLD_THRESH', - f'{data_type}_MODE_VALID_THRESH', - f'MODE_{data_type}_VLD_THRESH', - f'MODE_{data_type}_VALID_THRESH'], - 'vld_thresh', - f'METPLUS_{data_type}_VLD_THRESH') + self.add_met_config( + name='conv_thresh', + data_type='list', + env_var_name=f'METPLUS_{data_type}_CONV_THRESH', + metplus_configs=[f'{data_type}_MODE_CONV_THRESH', + f'MODE_{data_type}_CONV_THRESH', + 'MODE_CONV_THRESH' + ], + extra_args={ + 'remove_quotes': True, + } + ) + + self.add_met_config( + name='merge_thresh', + data_type='list', + env_var_name=f'METPLUS_{data_type}_MERGE_THRESH', + metplus_configs=[f'{data_type}_MODE_MERGE_THRESH', + f'MODE_{data_type}_MERGE_THRESH', + 'MODE_MERGE_THRESH' + ], + extra_args={ + 'remove_quotes': True, + } + ) + + self.add_met_config( + name='merge_flag', + data_type='string', + env_var_name=f'METPLUS_{data_type}_MERGE_FLAG', + metplus_configs=[f'{data_type}_MODE_MERGE_FLAG', + f'MODE_{data_type}_MERGE_FLAG', + 'MODE_MERGE_FLAG' + ], + extra_args={ + 'remove_quotes': True, + 'uppercase': True, + } + ) + + self.add_met_config( + name='filter_attr_name', + data_type='list', + env_var_name=f'METPLUS_{data_type}_FILTER_ATTR_NAME', + metplus_configs=[f'{data_type}_MODE_FILTER_ATTR_NAME', + f'MODE_{data_type}_FILTER_ATTR_NAME', + 'MODE_FILTER_ATTR_NAME' + ], + ) + + self.add_met_config( + name='filter_attr_thresh', + data_type='list', + env_var_name=f'METPLUS_{data_type}_FILTER_ATTR_THRESH', + metplus_configs=[f'{data_type}_MODE_FILTER_ATTR_THRESH', + f'MODE_{data_type}_FILTER_ATTR_THRESH', + 'MODE_FILTER_ATTR_THRESH' + ], + extra_args={ + 'remove_quotes': True, + } + ) + + self.add_met_config( + name='censor_thresh', + data_type='list', + env_var_name=f'METPLUS_{data_type}_CENSOR_THRESH', + metplus_configs=[f'{data_type}_MODE_CENSOR_THRESH', + f'MODE_{data_type}_CENSOR_THRESH', + 'MODE_CENSOR_THRESH' + ], + extra_args={ + 'remove_quotes': True, + } + ) + + self.add_met_config( + name='censor_val', + data_type='list', + env_var_name=f'METPLUS_{data_type}_CENSOR_VAL', + metplus_configs=[f'{data_type}_MODE_CENSOR_VAL', + f'MODE_{data_type}_CENSOR_VAL', + f'{data_type}_MODE_CENSOR_VALUE', + f'MODE_{data_type}_CENSOR_VALUE', + 'MODE_CENSOR_VAL', + 'MODE_CENSOR_VALUE', + ], + extra_args={ + 'remove_quotes': True, + } + ) + + self.add_met_config( + name='vld_thresh', + data_type='float', + env_var_name=f'METPLUS_{data_type}_VLD_THRESH', + metplus_configs=[f'{data_type}_MODE_VLD_THRESH', + f'MODE_{data_type}_VLD_THRESH', + f'{data_type}_MODE_VALID_THRESH', + f'MODE_{data_type}_VALID_THRESH', + 'MODE_VLD_THRESH', + 'MODE_VALID_THRESH' + ], + ) # set c_dict values for old method of setting env vars for name in ['CONV_RADIUS', @@ -254,14 +291,16 @@ def create_c_dict(self): value = self.get_env_var_value(f'METPLUS_{data_type}_{name}') c_dict[f'{data_type}_{name}'] = value - self.set_met_config_string(self.env_var_dict, - ['MODE_MATCH_FLAG'], - 'match_flag', - 'METPLUS_MATCH_FLAG', - remove_quotes=True, - uppercase=True) + self.add_met_config( + name='match_flag', + data_type='string', + extra_args={ + 'remove_quotes': True, + 'uppercase': True, + } + ) - self.handle_met_config_dict('weight', self.WEIGHTS) + self.add_met_config_dict('weight', self.WEIGHTS) self.handle_flags('nc_pairs') self.add_met_config(name='total_interest_thresh', diff --git a/metplus/wrappers/mtd_wrapper.py b/metplus/wrappers/mtd_wrapper.py index 328824ecbf..6bf2bc19be 100755 --- a/metplus/wrappers/mtd_wrapper.py +++ b/metplus/wrappers/mtd_wrapper.py @@ -15,6 +15,7 @@ from ..util import met_util as util from ..util import time_util from ..util import do_string_sub +from ..util import parse_var_list from . import CompareGriddedWrapper class MTDWrapper(CompareGriddedWrapper): @@ -66,8 +67,8 @@ def create_c_dict(self): c_dict['CONFIG_FILE'] = self.get_config_file('MTDConfig_wrapped') # new method of reading/setting MET config values - self.set_met_config_int(self.env_var_dict, 'MTD_MIN_VOLUME', - 'min_volume', 'METPLUS_MIN_VOLUME') + self.add_met_config(name='min_volume', + data_type='int') # old approach to reading/setting MET config values c_dict['MIN_VOLUME'] = self.config.getstr('config', @@ -124,9 +125,9 @@ def create_c_dict(self): self.read_field_values(c_dict, 'OBS', 'OBS') c_dict['VAR_LIST_TEMP'] = ( - util.parse_var_list(self.config, - data_type=c_dict.get('SINGLE_DATA_SRC'), - met_tool=self.app_name) + parse_var_list(self.config, + data_type=c_dict.get('SINGLE_DATA_SRC'), + met_tool=self.app_name) ) return c_dict @@ -137,17 +138,17 @@ def read_field_values(self, c_dict, read_type, write_type): self.config.getstr('config', f'{read_type}_MTD_INPUT_DATATYPE', '') ) - self.set_met_config_int(self.env_var_dict, - [f'{read_type}_MTD_CONV_RADIUS', - 'MTD_CONV_RADIUS'], - 'conv_radius', - f'METPLUS_{write_type}_CONV_RADIUS') - - self.set_met_config_thresh(self.env_var_dict, - [f'{read_type}_MTD_CONV_THRESH', - 'MTD_CONV_THRESH'], - 'conv_thresh', - f'METPLUS_{write_type}_CONV_THRESH') + self.add_met_config(name='conv_radius', + data_type='int', + env_var_name=f'METPLUS_{write_type}_CONV_RADIUS', + metplus_configs=[f'{read_type}_MTD_CONV_RADIUS', + 'MTD_CONV_RADIUS']) + + self.add_met_config(name='conv_thresh', + data_type='thresh', + env_var_name=f'METPLUS_{write_type}_CONV_THRESH', + metplus_configs=[f'{read_type}_MTD_CONV_THRESH', + 'MTD_CONV_THRESH']) # support old method of setting env vars conf_value = ( diff --git a/metplus/wrappers/pb2nc_wrapper.py b/metplus/wrappers/pb2nc_wrapper.py index 891fc367bc..fa59440aa4 100755 --- a/metplus/wrappers/pb2nc_wrapper.py +++ b/metplus/wrappers/pb2nc_wrapper.py @@ -91,27 +91,54 @@ def create_c_dict(self): # get the MET config file path or use default c_dict['CONFIG_FILE'] = self.get_config_file('PB2NCConfig_wrapped') - self.set_met_config_list(self.env_var_dict, - 'PB2NC_MESSAGE_TYPE', - 'message_type', - 'METPLUS_MESSAGE_TYPE',) + self.add_met_config(name='message_type', + data_type='list') - self.set_met_config_list(self.env_var_dict, - 'PB2NC_STATION_ID', - 'station_id', - 'METPLUS_STATION_ID',) + self.add_met_config(name='station_id', + data_type='list') self.handle_obs_window_variables(c_dict) self.handle_mask(single_value=True) - self.set_met_config_list(self.env_var_dict, - 'PB2NC_OBS_BUFR_VAR_LIST', - 'obs_bufr_var', - 'METPLUS_OBS_BUFR_VAR', - allow_empty=True) + self.add_met_config(name='obs_bufr_var', + data_type='list', + metplus_configs=['PB2NC_OBS_BUFR_VAR_LIST', + 'PB2NC_OBS_BUFR_VAR'], + extra_args={'allow_empty': True}) + + #self.handle_time_summary_legacy(c_dict) + self.handle_time_summary_dict() + + # handle legacy time summary variables + self.add_met_config(name='', + data_type='bool', + env_var_name='TIME_SUMMARY_FLAG', + metplus_configs=['PB2NC_TIME_SUMMARY_FLAG']) + + self.add_met_config(name='', + data_type='string', + env_var_name='TIME_SUMMARY_BEG', + metplus_configs=['PB2NC_TIME_SUMMARY_BEG']) + + self.add_met_config(name='', + data_type='string', + env_var_name='TIME_SUMMARY_END', + metplus_configs=['PB2NC_TIME_SUMMARY_END']) + + self.add_met_config(name='', + data_type='list', + env_var_name='TIME_SUMMARY_VAR_NAMES', + metplus_configs=['PB2NC_TIME_SUMMARY_OBS_VAR', + 'PB2NC_TIME_SUMMARY_VAR_NAMES'], + extra_args={'allow_empty': True}) - self.handle_time_summary_legacy(c_dict) + self.add_met_config(name='', + data_type='list', + env_var_name='TIME_SUMMARY_TYPES', + metplus_configs=['PB2NC_TIME_SUMMARY_TYPE', + 'PB2NC_TIME_SUMMARY_TYPES'], + extra_args={'allow_empty': True}) self.handle_file_window_variables(c_dict, dtypes=['OBS']) @@ -147,22 +174,7 @@ def create_c_dict(self): extra_args={'remove_quotes': True}) # get level_range beg and end - level_range_items = [] - level_range_items.append( - self.get_met_config(name='beg', - data_type='int', - metplus_configs=['PB2NC_LEVEL_RANGE_BEG', - 'PB2NC_LEVEL_RANGE_BEGIN']) - ) - level_range_items.append( - self.get_met_config(name='end', - data_type='int', - metplus_configs=['PB2NC_LEVEL_RANGE_END']) - ) - - self.add_met_config(name='level_range', - data_type='dict', - children=level_range_items) + self.add_met_config_window('level_range') self.add_met_config(name='level_category', data_type='list', @@ -173,7 +185,6 @@ def create_c_dict(self): data_type='int', metplus_configs=['PB2NC_QUALITY_MARK_THRESH']) - return c_dict def set_environment_variables(self, time_info): @@ -196,16 +207,10 @@ def set_environment_variables(self, time_info): self.add_env_var("OBS_BUFR_VAR_LIST", self.c_dict.get('BUFR_VAR_LIST', '')) - self.add_env_var('TIME_SUMMARY_FLAG', - self.c_dict.get('TIME_SUMMARY_FLAG', '')) - self.add_env_var('TIME_SUMMARY_BEG', - self.c_dict.get('TIME_SUMMARY_BEG', '')) - self.add_env_var('TIME_SUMMARY_END', - self.c_dict.get('TIME_SUMMARY_END', '')) - self.add_env_var('TIME_SUMMARY_VAR_NAMES', - self.c_dict.get('TIME_SUMMARY_VAR_NAMES', '')) - self.add_env_var('TIME_SUMMARY_TYPES', - self.c_dict.get('TIME_SUMMARY_TYPES', '')) + for item in ['FLAG', 'BEG', 'END', 'VAR_NAMES', 'TYPES']: + ts_item = f'TIME_SUMMARY_{item}' + self.add_env_var(f'{ts_item}', + self.env_var_dict.get(f'METPLUS_{ts_item}', '')) super().set_environment_variables(time_info) diff --git a/metplus/wrappers/pcp_combine_wrapper.py b/metplus/wrappers/pcp_combine_wrapper.py index 9d90a494ae..d34eaa609b 100755 --- a/metplus/wrappers/pcp_combine_wrapper.py +++ b/metplus/wrappers/pcp_combine_wrapper.py @@ -12,6 +12,7 @@ from ..util import get_seconds_from_string, ti_get_lead_string, ti_calculate from ..util import get_relativedelta, ti_get_seconds_from_relativedelta from ..util import time_string_to_met_time, seconds_to_met_time +from ..util import parse_var_list from . import ReformatGriddedWrapper '''!@namespace PCPCombineWrapper @@ -56,14 +57,14 @@ def create_c_dict(self): if fcst_run: c_dict = self.set_fcst_or_obs_dict_items('FCST', c_dict) - c_dict['VAR_LIST_FCST'] = util.parse_var_list( + c_dict['VAR_LIST_FCST'] = parse_var_list( self.config, data_type='FCST', met_tool=self.app_name ) if obs_run: c_dict = self.set_fcst_or_obs_dict_items('OBS', c_dict) - c_dict['VAR_LIST_OBS'] = util.parse_var_list( + c_dict['VAR_LIST_OBS'] = parse_var_list( self.config, data_type='OBS', met_tool=self.app_name diff --git a/metplus/wrappers/point_stat_wrapper.py b/metplus/wrappers/point_stat_wrapper.py index a21df7caad..f5a1a64de7 100755 --- a/metplus/wrappers/point_stat_wrapper.py +++ b/metplus/wrappers/point_stat_wrapper.py @@ -142,32 +142,29 @@ def create_c_dict(self): self.handle_obs_window_variables(c_dict) - self.set_met_config_list(self.env_var_dict, - ['POINT_STAT_MASK_GRID', - 'POINT_STAT_GRID'], - 'grid', - 'METPLUS_MASK_GRID', - allow_empty=True) - - self.set_met_config_list(self.env_var_dict, - ['POINT_STAT_MASK_POLY', - 'POINT_STAT_POLY'], - 'poly', - 'METPLUS_MASK_POLY', - allow_empty=True) - - self.set_met_config_list(self.env_var_dict, - ['POINT_STAT_MASK_SID', - 'POINT_STAT_STATION_ID'], - 'sid', - 'METPLUS_MASK_SID', - allow_empty=True) - - - self.set_met_config_list(self.env_var_dict, - 'POINT_STAT_MESSAGE_TYPE', - 'message_type', - 'METPLUS_MESSAGE_TYPE',) + self.add_met_config(name='grid', + data_type='list', + env_var_name='METPLUS_MASK_GRID', + metplus_configs=['POINT_STAT_MASK_GRID', + 'POINT_STAT_GRID'], + extra_args={'allow_empty': True}) + + self.add_met_config(name='poly', + data_type='list', + env_var_name='METPLUS_MASK_POLY', + metplus_configs=['POINT_STAT_MASK_POLY', + 'POINT_STAT_POLY'], + extra_args={'allow_empty': True}) + + self.add_met_config(name='sid', + data_type='list', + env_var_name='METPLUS_MASK_SID', + metplus_configs=['POINT_STAT_MASK_SID', + 'POINT_STAT_STATION_ID'], + extra_args={'allow_empty': True}) + + self.add_met_config(name='message_type', + data_type='list') self.handle_climo_cdf_dict() diff --git a/metplus/wrappers/regrid_data_plane_wrapper.py b/metplus/wrappers/regrid_data_plane_wrapper.py index b3f0711fcc..a3d544a0db 100755 --- a/metplus/wrappers/regrid_data_plane_wrapper.py +++ b/metplus/wrappers/regrid_data_plane_wrapper.py @@ -15,6 +15,8 @@ from ..util import met_util as util from ..util import time_util from ..util import do_string_sub +from ..util import parse_var_list +from ..util import get_process_list from . import ReformatGriddedWrapper # pylint:disable=pointless-string-statement @@ -106,7 +108,7 @@ def create_c_dict(self): self.log_error("FCST_REGRID_DATA_PLANE_OUTPUT_TEMPLATE must be set if " "FCST_REGRID_DATA_PLANE_RUN is True") - c_dict['VAR_LIST_FCST'] = util.parse_var_list( + c_dict['VAR_LIST_FCST'] = parse_var_list( self.config, data_type='FCST', met_tool=self.app_name @@ -129,7 +131,7 @@ def create_c_dict(self): self.log_error("OBS_REGRID_DATA_PLANE_OUTPUT_TEMPLATE must be set if " "OBS_REGRID_DATA_PLANE_RUN is True") - c_dict['VAR_LIST_OBS'] = util.parse_var_list( + c_dict['VAR_LIST_OBS'] = parse_var_list( self.config, data_type='OBS', met_tool=self.app_name @@ -155,7 +157,7 @@ def create_c_dict(self): # only check if VERIFICATION_GRID is set if running the tool from the process list # RegridDataPlane can be called from other tools like CustomIngest, which sets the # verification grid itself - if 'RegridDataPlane' in util.get_process_list(self.config): + if 'RegridDataPlane' in get_process_list(self.config): if not c_dict['VERIFICATION_GRID']: self.log_error("REGRID_DATA_PLANE_VERIF_GRID must be set.") diff --git a/metplus/wrappers/series_analysis_wrapper.py b/metplus/wrappers/series_analysis_wrapper.py index 972fa1c376..429b139c39 100755 --- a/metplus/wrappers/series_analysis_wrapper.py +++ b/metplus/wrappers/series_analysis_wrapper.py @@ -27,6 +27,7 @@ from ..util import get_lead_sequence, get_lead_sequence_groups, set_input_dict from ..util import ti_get_hours_from_lead, ti_get_seconds_from_lead from ..util import ti_get_lead_string +from ..util import parse_var_list from .plot_data_plane_wrapper import PlotDataPlaneWrapper from . import RuntimeFreqWrapper @@ -86,14 +87,13 @@ def create_c_dict(self): c_dict['VERBOSITY']) ) - self.set_met_config_string(self.env_var_dict, - 'MODEL', - 'model', - 'METPLUS_MODEL') - self.set_met_config_string(self.env_var_dict, - 'OBTYPE', - 'obtype', - 'METPLUS_OBTYPE') + self.add_met_config(name='model', + data_type='string', + metplus_configs=['MODEL']) + + self.add_met_config(name='obtype', + data_type='string', + metplus_configs=['OBTYPE']) # handle old format of MODEL and OBTYPE c_dict['MODEL'] = self.config.getstr('config', 'MODEL', 'WRF') @@ -103,22 +103,18 @@ def create_c_dict(self): self.handle_regrid(c_dict) - self.set_met_config_list(self.env_var_dict, - 'SERIES_ANALYSIS_CAT_THRESH', - 'cat_thresh', - 'METPLUS_CAT_THRESH', - remove_quotes=True) + self.add_met_config(name='cat_thresh', + data_type='list', + extra_args={'remove_quotes': True}) - self.set_met_config_float(self.env_var_dict, - 'SERIES_ANALYSIS_VLD_THRESH', - 'vld_thresh', - 'METPLUS_VLD_THRESH') + self.add_met_config(name='vld_thresh', + data_type='float', + metplus_configs=['SERIES_ANALYSIS_VLD_THRESH', + 'SERIES_ANALYSIS_VALID_THRESH',]) - self.set_met_config_string(self.env_var_dict, - 'SERIES_ANALYSIS_BLOCK_SIZE', - 'block_size', - 'METPLUS_BLOCK_SIZE', - remove_quotes=True) + self.add_met_config(name='block_size', + data_type='string', + extra_args={'remove_quotes': True}) # get stat list to loop over c_dict['STAT_LIST'] = util.getlist( @@ -130,16 +126,18 @@ def create_c_dict(self): self.log_error("Must set SERIES_ANALYSIS_STAT_LIST to run.") # set stat list to set output_stats.cnt in MET config file - self.set_met_config_list(self.env_var_dict, - 'SERIES_ANALYSIS_STAT_LIST', - 'cnt', - 'METPLUS_STAT_LIST') + self.add_met_config(name='cnt', + data_type='list', + env_var_name='METPLUS_STAT_LIST', + metplus_configs=['SERIES_ANALYSIS_STAT_LIST', + 'SERIES_ANALYSIS_CNT']) # set cts list to set output_stats.cts in MET config file - self.set_met_config_list(self.env_var_dict, - 'SERIES_ANALYSIS_CTS_LIST', - 'cts', - 'METPLUS_CTS_LIST') + self.add_met_config(name='cts', + data_type='list', + env_var_name='METPLUS_CTS_LIST', + metplus_configs=['SERIES_ANALYSIS_CTS_LIST', + 'SERIES_ANALYSIS_CTS']) c_dict['PAIRED'] = self.config.getbool('config', 'SERIES_ANALYSIS_IS_PAIRED', @@ -233,8 +231,8 @@ def create_c_dict(self): False) ) - c_dict['VAR_LIST_TEMP'] = util.parse_var_list(self.config, - met_tool=self.app_name) + c_dict['VAR_LIST_TEMP'] = parse_var_list(self.config, + met_tool=self.app_name) if not c_dict['VAR_LIST_TEMP']: self.log_error("No fields specified. Please set " "[FCST/OBS]_VAR_[NAME/LEVELS]") @@ -444,7 +442,7 @@ def get_storm_list(self, time_info): # Now that we have the filter filename for the init time, let's # extract all the storm ids in this filter file. - storm_list = util.get_storm_ids(filter_file) + storm_list = util.get_storms(filter_file, id_only=True) if not storm_list: # No storms for this init time, check next init time in list self.logger.debug("No storms found for current runtime") diff --git a/metplus/wrappers/stat_analysis_wrapper.py b/metplus/wrappers/stat_analysis_wrapper.py index 1d8289aaa2..44fabeb66e 100755 --- a/metplus/wrappers/stat_analysis_wrapper.py +++ b/metplus/wrappers/stat_analysis_wrapper.py @@ -19,7 +19,8 @@ import itertools from ..util import met_util as util -from ..util import do_string_sub +from ..util import do_string_sub, find_indices_in_config_section +from ..util import parse_var_list from . import CommandBuilder class StatAnalysisWrapper(CommandBuilder): @@ -215,7 +216,7 @@ def create_c_dict(self): if not self.MakePlotsWrapper.isOK: self.log_error("MakePlotsWrapper was not initialized correctly.") - c_dict['VAR_LIST'] = util.parse_var_list(self.config) + c_dict['VAR_LIST'] = parse_var_list(self.config) c_dict['MODEL_INFO_LIST'] = self.parse_model_info() if not c_dict['MODEL_LIST'] and c_dict['MODEL_INFO_LIST']: @@ -1229,9 +1230,9 @@ def parse_model_info(self): """ model_info_list = [] model_indices = list( - util.find_indices_in_config_section(r'MODEL(\d+)$', - self.config, - index_index=1).keys() + find_indices_in_config_section(r'MODEL(\d+)$', + self.config, + index_index=1).keys() ) for m in model_indices: model_name = self.config.getstr('config', f'MODEL{m}') diff --git a/metplus/wrappers/tc_gen_wrapper.py b/metplus/wrappers/tc_gen_wrapper.py index 94c6a0596e..91168a0e96 100755 --- a/metplus/wrappers/tc_gen_wrapper.py +++ b/metplus/wrappers/tc_gen_wrapper.py @@ -148,16 +148,16 @@ def create_c_dict(self): data_type='int', metplus_configs=['TC_GEN_VALID_FREQUENCY', 'TC_GEN_VALID_FREQ']) - self.handle_met_config_window('fcst_hr_window') + self.add_met_config_window('fcst_hr_window') self.add_met_config(name='min_duration', data_type='int', metplus_configs=['TC_GEN_MIN_DURATION']) - self.handle_met_config_dict('fcst_genesis', { + self.add_met_config_dict('fcst_genesis', { 'vmax_thresh': 'thresh', 'mslp_thresh': 'thresh', }) - self.handle_met_config_dict('best_genesis', { + self.add_met_config_dict('best_genesis', { 'technique': 'string', 'category': 'list', 'vmax_thresh': 'thresh', @@ -220,8 +220,8 @@ def create_c_dict(self): self.add_met_config(name='dev_hit_radius', data_type='int', metplus_configs=['TC_GEN_DEV_HIT_RADIUS']) - self.handle_met_config_window('dev_hit_window') - self.handle_met_config_window('ops_hit_window') + self.add_met_config_window('dev_hit_window') + self.add_met_config_window('ops_hit_window') self.add_met_config(name='discard_init_post_genesis_flag', data_type='bool', metplus_configs=[ @@ -260,7 +260,7 @@ def create_c_dict(self): data_type='bool', metplus_configs=['TC_GEN_GENESIS_MATCH_POINT_TO_TRACK'] ) - self.handle_met_config_window('genesis_match_window') + self.add_met_config_window('genesis_match_window') # get INPUT_TIME_DICT values since wrapper only runs # once (doesn't look over time) diff --git a/metplus/wrappers/tc_pairs_wrapper.py b/metplus/wrappers/tc_pairs_wrapper.py index 2764202f59..f86e814e13 100755 --- a/metplus/wrappers/tc_pairs_wrapper.py +++ b/metplus/wrappers/tc_pairs_wrapper.py @@ -24,6 +24,7 @@ from ..util import met_util as util from ..util import do_string_sub from ..util import get_tags +from ..util.met_config import add_met_config_dict_list from . import CommandBuilder '''!@namespace TCPairsWrapper @@ -315,66 +316,21 @@ def _read_storm_info(self, c_dict): c_dict['BASIN_LIST'] = basin_list def handle_consensus(self): - children = [ - 'NAME', - 'MEMBERS', - 'REQUIRED', - 'MIN_REQ' - ] - regex = r'^TC_PAIRS_CONSENSUS(\d+)_(\w+)$' - indices = util.find_indices_in_config_section(regex, self.config, - index_index=1, - id_index=2) - - consensus_dict = {} - for index, items in indices.items(): - # read all variables for each index - consensus_items = {} - - # check if any variable found doesn't match valid variables - if any([item for item in items if item not in children]): - self.log_error("Invalid variable: " - f"TC_PAIRS_CONSENSUS{index}_{item}") - - self.add_met_config( - name='name', - data_type='string', - metplus_configs=[f'TC_PAIRS_CONSENSUS{index}_NAME'], - output_dict=consensus_items - ) - self.add_met_config( - name='members', - data_type='list', - metplus_configs=[f'TC_PAIRS_CONSENSUS{index}_MEMBERS'], - output_dict=consensus_items - ) - self.add_met_config( - name='required', - data_type='list', - metplus_configs=[f'TC_PAIRS_CONSENSUS{index}_REQUIRED'], - extra_args={'remove_quotes': True}, - output_dict=consensus_items - ) - self.add_met_config( - name='min_req', - data_type='int', - metplus_configs=[f'TC_PAIRS_CONSENSUS{index}_MIN_REQ'], - output_dict=consensus_items - ) - - self.logger.debug(f'Consensus Items: {consensus_items}') - # format dictionary, then add it to consensus_dict - dict_string = self.format_met_config('dict', - consensus_items, - name='') - consensus_dict[index] = dict_string - - # format list of dictionaries - output_string = self.format_met_config('list', - consensus_dict, - 'consensus') - - self.env_var_dict['METPLUS_CONSENSUS_LIST'] = output_string + dict_items = { + 'name': 'string', + 'members': 'list', + 'required': ('list', 'remove_quotes'), + 'min_req': 'int', + } + return_code = add_met_config_dict_list(config=self.config, + app_name=self.app_name, + output_dict=self.env_var_dict, + dict_name='consensus', + dict_items=dict_items) + if not return_code: + self.isOK = False + + return return_code def run_all_times(self): """! Build up the command to invoke the MET tool tc_pairs. diff --git a/metplus/wrappers/tc_stat_wrapper.py b/metplus/wrappers/tc_stat_wrapper.py index c8a6c52a01..bb3c33e62c 100755 --- a/metplus/wrappers/tc_stat_wrapper.py +++ b/metplus/wrappers/tc_stat_wrapper.py @@ -129,15 +129,77 @@ def create_c_dict(self): # get the MET config file path or use default c_dict['CONFIG_FILE'] = self.get_config_file('TCStatConfig_wrapped') + self.set_met_config_for_environment_variables() + + return c_dict + + def set_met_config_for_environment_variables(self): + """! Set c_dict dictionary entries that will be set as environment + variables to be read by the MET config file. + @param c_dict dictionary to add key/value pairs + """ self.handle_description() - self.set_met_config_for_environment_variables() + for config_list in ['amodel', + 'bmodel', + 'storm_id', + 'basin', + 'cyclone', + 'storm_name', + 'init_hour', + 'lead_req', + 'init_mask', + 'valid_mask', + 'valid_hour', + 'lead', + 'track_watch_warn', + 'column_thresh_name', + 'column_thresh_val', + 'column_str_name', + 'column_str_val', + 'init_thresh_name', + 'init_thresh_val', + 'init_str_name', + 'init_str_val', + ]: + self.add_met_config(name=config_list, + data_type='list') + + for iv_list in ['INIT', 'VALID']: + self.add_met_config(name=f'{iv_list.lower()}_inc', + data_type='list', + metplus_configs=[f'TC_STAT_{iv_list}_INC', + f'TC_STAT_{iv_list}_INCLUDE']) + self.add_met_config(name=f'{iv_list.lower()}_exc', + data_type='list', + metplus_configs=[f'TC_STAT_{iv_list}_EXC', + f'TC_STAT_{iv_list}_EXCLUDE']) + + for config_str in ['INIT_BEG', + 'INIT_END', + 'VALID_BEG', + 'VALID_END', + 'LANDFALL_BEG', + 'LANDFALL_END', + ]: + self.add_met_config(name=config_str.lower(), + data_type='string', + metplus_configs=[f'TC_STAT_{config_str}', + config_str]) + + for config_bool in ['water_only', + 'landfall', + 'match_points', + ]: + + self.add_met_config(name=config_bool, + data_type='bool') self.add_met_config(name='column_str_exc_name', data_type='list', metplus_configs=['TC_STAT_COLUMN_STR_EXC_NAME', 'TC_STAT_COLUMN_STR_EXCLUDE_NAME', - ]) + ]) self.add_met_config(name='column_str_exc_val', data_type='list', metplus_configs=['TC_STAT_COLUMN_STR_EXC_VAL', @@ -154,77 +216,6 @@ def create_c_dict(self): 'TC_STAT_INIT_STR_EXCLUDE_VAL', ]) - return c_dict - - def set_met_config_for_environment_variables(self): - """! Set c_dict dictionary entries that will be set as environment - variables to be read by the MET config file. - @param c_dict dictionary to add key/value pairs - """ - app_name_upper = self.app_name.upper() - - for config_list in ['AMODEL', - 'BMODEL', - 'STORM_ID', - 'BASIN', - 'CYCLONE', - 'STORM_NAME', - 'INIT_HOUR', - 'LEAD_REQ', - 'INIT_MASK', - 'VALID_MASK', - 'VALID_HOUR', - 'LEAD', - 'TRACK_WATCH_WARN', - 'COLUMN_THRESH_NAME', - 'COLUMN_THRESH_VAL', - 'COLUMN_STR_NAME', - 'COLUMN_STR_VAL', - 'INIT_THRESH_NAME', - 'INIT_THRESH_VAL', - 'INIT_STR_NAME', - 'INIT_STR_VAL', - ]: - self.set_met_config_list(self.env_var_dict, - f'{app_name_upper}_{config_list}', - config_list.lower(), - f'METPLUS_{config_list}') - - for iv_list in ['INIT', 'VALID',]: - self.set_met_config_list(self.env_var_dict, - f'{app_name_upper}_{iv_list}_INCLUDE', - f'{iv_list.lower()}_inc', - f'METPLUS_{iv_list}_INC' - ) - self.set_met_config_list(self.env_var_dict, - f'{app_name_upper}_{iv_list}_EXCLUDE', - f'{iv_list.lower()}_exc', - f'METPLUS_{iv_list}_EXC' - ) - - for config_str in ['INIT_BEG', - 'INIT_END', - 'VALID_BEG', - 'VALID_END', - 'LANDFALL_BEG', - 'LANDFALL_END', - ]: - self.set_met_config_string(self.env_var_dict, - [f'{app_name_upper}_{config_str}', - f'{config_str}'], - config_str.lower(), - f'METPLUS_{config_str}') - - for config_bool in ['WATER_ONLY', - 'LANDFALL', - 'MATCH_POINTS', - ]: - - self.set_met_config_bool(self.env_var_dict, - f'{app_name_upper}_{config_bool}', - config_bool.lower(), - f'METPLUS_{config_bool}') - def run_at_time(self, input_dict=None): """! Builds the call to the MET tool TC-STAT for all requested initialization times (init or valid). Called from run_metplus diff --git a/metplus/wrappers/tcrmw_wrapper.py b/metplus/wrappers/tcrmw_wrapper.py index 5cf562017f..a5ce97553d 100755 --- a/metplus/wrappers/tcrmw_wrapper.py +++ b/metplus/wrappers/tcrmw_wrapper.py @@ -16,6 +16,7 @@ from ..util import time_util from . import CommandBuilder from ..util import do_string_sub +from ..util import parse_var_list '''!@namespace TCRMWWrapper @brief Wraps the TC-RMW tool @@ -81,92 +82,85 @@ def create_c_dict(self): 'TC_RMW_DECK_TEMPLATE') ) - self.set_met_config_string(self.env_var_dict, - 'TC_RMW_INPUT_DATATYPE', - 'file_type', - 'METPLUS_DATA_FILE_TYPE') + self.add_met_config(name='file_type', + data_type='string', + env_var_name='METPLUS_DATA_FILE_TYPE', + metplus_configs=['TC_RMW_INPUT_DATATYPE', + 'TC_RMW_FILE_TYPE']) - # values used in configuration file - self.set_met_config_string(self.env_var_dict, - 'MODEL', - 'model', - 'METPLUS_MODEL') + self.add_met_config(name='model', + data_type='string', + metplus_configs=['MODEL']) self.handle_regrid(c_dict, set_to_grid=False) - self.set_met_config_int(self.env_var_dict, - 'TC_RMW_N_RANGE', - 'n_range', - 'METPLUS_N_RANGE') - - self.set_met_config_int(self.env_var_dict, - 'TC_RMW_N_AZIMUTH', - 'n_azimuth', - 'METPLUS_N_AZIMUTH') - - self.set_met_config_float(self.env_var_dict, - 'TC_RMW_MAX_RANGE_KM', - 'max_range_km', - 'METPLUS_MAX_RANGE_KM') - - self.set_met_config_float(self.env_var_dict, - 'TC_RMW_DELTA_RANGE_KM', - 'delta_range_km', - 'METPLUS_DELTA_RANGE_KM') - - self.set_met_config_float(self.env_var_dict, - 'TC_RMW_SCALE', - 'rmw_scale', - 'METPLUS_RMW_SCALE') - - self.set_met_config_string(self.env_var_dict, - 'TC_RMW_STORM_ID', - 'storm_id', - 'METPLUS_STORM_ID') - - self.set_met_config_string(self.env_var_dict, - 'TC_RMW_BASIN', - 'basin', - 'METPLUS_BASIN') - - self.set_met_config_string(self.env_var_dict, - 'TC_RMW_CYCLONE', - 'cyclone', - 'METPLUS_CYCLONE') - - self.set_met_config_string(self.env_var_dict, - 'TC_RMW_INIT_INCLUDE', - 'init_inc', - 'METPLUS_INIT_INCLUDE') - - self.set_met_config_string(self.env_var_dict, - 'TC_RMW_VALID_BEG', - 'valid_beg', - 'METPLUS_VALID_BEG') - - self.set_met_config_string(self.env_var_dict, - 'TC_RMW_VALID_END', - 'valid_end', - 'METPLUS_VALID_END') - - self.set_met_config_list(self.env_var_dict, - 'TC_RMW_VALID_INCLUDE_LIST', - 'valid_inc', - 'METPLUS_VALID_INCLUDE_LIST') - - self.set_met_config_list(self.env_var_dict, - 'TC_RMW_VALID_EXCLUDE_LIST', - 'valid_exc', - 'METPLUS_VALID_EXCLUDE_LIST') - - self.set_met_config_list(self.env_var_dict, - 'TC_RMW_VALID_HOUR_LIST', - 'valid_hour', - 'METPLUS_VALID_HOUR_LIST') - - c_dict['VAR_LIST_TEMP'] = util.parse_var_list(self.config, - data_type='FCST', - met_tool=self.app_name) + self.add_met_config(name='n_range', + data_type='int') + + self.add_met_config(name='n_azimuth', + data_type='int') + + self.add_met_config(name='max_range_km', + data_type='float') + + self.add_met_config(name='delta_range_km', + data_type='float') + + self.add_met_config(name='rmw_scale', + data_type='float') + + self.add_met_config(name='storm_id', + data_type='string') + + self.add_met_config(name='basin', + data_type='string') + + self.add_met_config(name='cyclone', + data_type='string') + + self.add_met_config(name='init_inc', + data_type='string', + env_var_name='METPLUS_INIT_INCLUDE', + metplus_configs=['TC_RMW_INIT_INC', + 'TC_RMW_INIT_INCLUDE']) + + self.add_met_config(name='valid_beg', + data_type='string', + metplus_configs=['TC_RMW_VALID_BEG', + 'TC_RMW_VALID_BEGIN']) + + self.add_met_config(name='valid_end', + data_type='string', + metplus_configs=['TC_RMW_VALID_END']) + + self.add_met_config(name='valid_inc', + data_type='list', + env_var_name='METPLUS_VALID_INCLUDE_LIST', + metplus_configs=['TC_RMW_VALID_INCLUDE_LIST', + 'TC_RMW_VALID_INC_LIST', + 'TC_RMW_VALID_INCLUDE', + 'TC_RMW_VALID_INC', + ]) + + self.add_met_config(name='valid_exc', + data_type='list', + env_var_name='METPLUS_VALID_EXCLUDE_LIST', + metplus_configs=['TC_RMW_VALID_EXCLUDE_LIST', + 'TC_RMW_VALID_EXC_LIST', + 'TC_RMW_VALID_EXCLUDE', + 'TC_RMW_VALID_EXC', + ]) + + self.add_met_config(name='valid_hour', + data_type='list', + env_var_name='METPLUS_VALID_HOUR_LIST', + metplus_configs=['TC_RMW_VALID_HOUR_LIST', + 'TC_RMW_VALID_HOUR', + ]) + + c_dict['VAR_LIST_TEMP'] = parse_var_list(self.config, + data_type='FCST', + met_tool=self.app_name) return c_dict