Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feature 986 TCPairs valid time #997

Merged
merged 5 commits into from
Jul 15, 2021
Merged

feature 986 TCPairs valid time #997

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
52dab2b
modified logic to all loop by valid time for TCPairs when loop order …
georgemccabe Jul 7, 2021
6694e7d
modified unit tests to check if LOOP_BY = INIT and VALID both work wh…
georgemccabe Jul 7, 2021
d9748f6
loop over forecast lead list (LEAD_SEQ) if set and run once for each
georgemccabe Jul 7, 2021
1616aa7
Merge branch 'develop' into feature_986_tc_pairs_valid_time
georgemccabe Jul 12, 2021
25bcc56
added option to skip looping over forecast lead list in TCPairs to be…
georgemccabe Jul 12, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions docs/Users_Guide/glossary.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7162,3 +7162,12 @@ METplus Configuration Glossary
variable OR :term:`EXTRACT_TILES_TC_STAT_INPUT_TEMPLATE` but not both.

| *Used by:* ExtractTiles

TC_PAIRS_SKIP_LEAD_SEQ
If True and a forecast lead sequence is set in the configuration, do not
loop over list of leads and process for each. This is used for feature
relative use cases where TCPairs is run for each storm initialization time
and SeriesAnalysis is configured to filter the data by forecast leads.
Default value is False.

| *Used by:* TCPairs
1 change: 1 addition & 0 deletions docs/Users_Guide/wrappers.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5393,6 +5393,7 @@ METplus Configuration
| :term:`TC_PAIRS_CONSENSUS<n>_MEMBERS`
| :term:`TC_PAIRS_CONSENSUS<n>_REQUIRED`
| :term:`TC_PAIRS_CONSENSUS<n>_MIN_REQ`
| :term:`TC_PAIRS_SKIP_LEAD_SEQ`
|

.. warning:: **DEPRECATED:**
Expand Down
250 changes: 135 additions & 115 deletions internal_tests/pytests/tc_pairs/test_tc_pairs_wrapper.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,19 +15,19 @@
time_fmt = '%Y%m%d%H'
run_times = ['2014121318']

def set_minimum_config_settings(config):
def set_minimum_config_settings(config, loop_by='INIT'):
# set config variables to prevent command from running and bypass check
# if input files actually exist
config.set('config', 'DO_NOT_RUN_EXE', True)
config.set('config', 'INPUT_MUST_EXIST', False)

# set process and time config variables
config.set('config', 'PROCESS_LIST', 'TCPairs')
config.set('config', 'LOOP_BY', 'INIT')
config.set('config', 'INIT_TIME_FMT', time_fmt)
config.set('config', 'INIT_BEG', run_times[0])
config.set('config', 'INIT_END', run_times[-1])
config.set('config', 'INIT_INCREMENT', '12H')
config.set('config', 'LOOP_BY', loop_by)
config.set('config', f'{loop_by}_TIME_FMT', time_fmt)
config.set('config', f'{loop_by}_BEG', run_times[0])
config.set('config', f'{loop_by}_END', run_times[-1])
config.set('config', f'{loop_by}_INCREMENT', '12H')
config.set('config', 'TC_PAIRS_CONFIG_FILE',
'{PARM_BASE}/met_config/TCPairsConfig_wrapped')
config.set('config', 'TC_PAIRS_BDECK_TEMPLATE', bdeck_template)
Expand Down Expand Up @@ -264,7 +264,7 @@ def test_tc_pairs_storm_id_lists(metplus_config, config_overrides,
config.set('config', key, value)

wrapper = TCPairsWrapper(config)
assert(wrapper.isOK)
assert wrapper.isOK

all_cmds = wrapper.run_all_times()
print(f"ALL COMMANDS:")
Expand Down Expand Up @@ -356,65 +356,78 @@ def test_tc_pairs_storm_id_lists(metplus_config, config_overrides,
)
def test_tc_pairs_loop_order_processes(metplus_config, config_overrides,
env_var_values):

config = metplus_config()

set_minimum_config_settings(config)

test_data_dir = os.path.join(config.getdir('METPLUS_BASE'),
'internal_tests',
'data',
'tc_pairs')
bdeck_dir = os.path.join(test_data_dir, 'bdeck')
adeck_dir = os.path.join(test_data_dir, 'adeck')

config.set('config', 'TC_PAIRS_BDECK_INPUT_DIR', bdeck_dir)
config.set('config', 'TC_PAIRS_ADECK_INPUT_DIR', adeck_dir)

# LOOP_ORDER processes runs once, times runs once per time
config.set('config', 'LOOP_ORDER', 'processes')

# set config variable overrides
for key, value in config_overrides.items():
config.set('config', key, value)

if 'METPLUS_INIT_BEG' not in env_var_values:
env_var_values['METPLUS_INIT_BEG'] = f'init_beg = "{run_times[0]}";'

if 'METPLUS_INIT_END' not in env_var_values:
env_var_values['METPLUS_INIT_END'] = f'init_end = "{run_times[-1]}";'

wrapper = TCPairsWrapper(config)
assert(wrapper.isOK)

app_path = os.path.join(config.getdir('MET_BIN_DIR'), wrapper.app_name)
verbosity = f"-v {wrapper.c_dict['VERBOSITY']}"
config_file = wrapper.c_dict.get('CONFIG_FILE')
out_dir = wrapper.c_dict.get('OUTPUT_DIR')
expected_cmds = [(f"{app_path} {verbosity} "
f"-bdeck {bdeck_dir}/bmlq2014123118.gfso.0104 "
f"-adeck {adeck_dir}/amlq2014123118.gfso.0104 "
f"-config {config_file} "
f"-out {out_dir}/mlq2014121318.gfso.0104"),
]


all_cmds = wrapper.run_all_times()
print(f"ALL COMMANDS: {all_cmds}")
assert(len(all_cmds) == len(expected_cmds))

for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds):
# ensure commands are generated as expected
assert(cmd == expected_cmd)

# check that environment variables were set properly
for env_var_key in wrapper.WRAPPER_ENV_VAR_KEYS:
match = next((item for item in env_vars if
item.startswith(env_var_key)), None)
assert(match is not None)
print(f'Checking env var: {env_var_key}')
actual_value = match.split('=', 1)[1]
assert(env_var_values.get(env_var_key, '') == actual_value)
# run using init and valid time variables
for loop_by in ['INIT', 'VALID']:
remove_beg = remove_end = False
config = metplus_config()

set_minimum_config_settings(config, loop_by)

test_data_dir = os.path.join(config.getdir('METPLUS_BASE'),
'internal_tests',
'data',
'tc_pairs')
bdeck_dir = os.path.join(test_data_dir, 'bdeck')
adeck_dir = os.path.join(test_data_dir, 'adeck')

config.set('config', 'TC_PAIRS_BDECK_INPUT_DIR', bdeck_dir)
config.set('config', 'TC_PAIRS_ADECK_INPUT_DIR', adeck_dir)

# LOOP_ORDER processes runs once, times runs once per time
config.set('config', 'LOOP_ORDER', 'processes')

# set config variable overrides
for key, value in config_overrides.items():
config.set('config', key, value)

if f'METPLUS_{loop_by}_BEG' not in env_var_values:
env_var_values[f'METPLUS_{loop_by}_BEG'] = (
f'{loop_by.lower()}_beg = "{run_times[0]}";'
)
remove_beg = True

if f'METPLUS_{loop_by}_END' not in env_var_values:
env_var_values[f'METPLUS_{loop_by}_END'] = (
f'{loop_by.lower()}_end = "{run_times[-1]}";'
)
remove_end = True

wrapper = TCPairsWrapper(config)
assert wrapper.isOK

app_path = os.path.join(config.getdir('MET_BIN_DIR'), wrapper.app_name)
verbosity = f"-v {wrapper.c_dict['VERBOSITY']}"
config_file = wrapper.c_dict.get('CONFIG_FILE')
out_dir = wrapper.c_dict.get('OUTPUT_DIR')
expected_cmds = [(f"{app_path} {verbosity} "
f"-bdeck {bdeck_dir}/bmlq2014123118.gfso.0104 "
f"-adeck {adeck_dir}/amlq2014123118.gfso.0104 "
f"-config {config_file} "
f"-out {out_dir}/mlq2014121318.gfso.0104"),
]


all_cmds = wrapper.run_all_times()
print(f"ALL COMMANDS: {all_cmds}")
assert(len(all_cmds) == len(expected_cmds))

for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds):
# ensure commands are generated as expected
assert(cmd == expected_cmd)

# check that environment variables were set properly
for env_var_key in wrapper.WRAPPER_ENV_VAR_KEYS:
match = next((item for item in env_vars if
item.startswith(env_var_key)), None)
assert(match is not None)
print(f'Checking env var: {env_var_key}')
actual_value = match.split('=', 1)[1]
assert(env_var_values.get(env_var_key, '') == actual_value)

if remove_beg:
del env_var_values[f'METPLUS_{loop_by}_BEG']
if remove_end:
del env_var_values[f'METPLUS_{loop_by}_END']

@pytest.mark.parametrize(
'config_overrides, env_var_values', [
Expand All @@ -433,61 +446,68 @@ def test_tc_pairs_loop_order_processes(metplus_config, config_overrides,
)
def test_tc_pairs_read_all_files(metplus_config, config_overrides,
env_var_values):
# run using init and valid time variables
for loop_by in ['INIT', 'VALID']:
config = metplus_config()

config = metplus_config()
set_minimum_config_settings(config, loop_by)

set_minimum_config_settings(config)
test_data_dir = os.path.join(config.getdir('METPLUS_BASE'),
'internal_tests',
'data',
'tc_pairs')
bdeck_dir = os.path.join(test_data_dir, 'bdeck')
adeck_dir = os.path.join(test_data_dir, 'adeck')

test_data_dir = os.path.join(config.getdir('METPLUS_BASE'),
'internal_tests',
'data',
'tc_pairs')
bdeck_dir = os.path.join(test_data_dir, 'bdeck')
adeck_dir = os.path.join(test_data_dir, 'adeck')
config.set('config', 'TC_PAIRS_BDECK_INPUT_DIR', bdeck_dir)
config.set('config', 'TC_PAIRS_ADECK_INPUT_DIR', adeck_dir)

config.set('config', 'TC_PAIRS_BDECK_INPUT_DIR', bdeck_dir)
config.set('config', 'TC_PAIRS_ADECK_INPUT_DIR', adeck_dir)
# LOOP_ORDER processes runs once, times runs once per time
config.set('config', 'LOOP_ORDER', 'processes')

# LOOP_ORDER processes runs once, times runs once per time
config.set('config', 'LOOP_ORDER', 'processes')

config.set('config', 'TC_PAIRS_READ_ALL_FILES', True)
config.set('config', 'TC_PAIRS_OUTPUT_TEMPLATE', '')
config.set('config', 'TC_PAIRS_READ_ALL_FILES', True)
config.set('config', 'TC_PAIRS_OUTPUT_TEMPLATE', '')

# set config variable overrides
for key, value in config_overrides.items():
config.set('config', key, value)
# set config variable overrides
for key, value in config_overrides.items():
config.set('config', key, value)

if 'METPLUS_INIT_BEG' not in env_var_values:
env_var_values['METPLUS_INIT_BEG'] = f'init_beg = "{run_times[0]}";'

if 'METPLUS_INIT_END' not in env_var_values:
env_var_values['METPLUS_INIT_END'] = f'init_end = "{run_times[-1]}";'
env_var_values[f'METPLUS_{loop_by}_BEG'] = (
f'{loop_by.lower()}_beg = "{run_times[0]}";'
)

wrapper = TCPairsWrapper(config)
assert(wrapper.isOK)

app_path = os.path.join(config.getdir('MET_BIN_DIR'), wrapper.app_name)
verbosity = f"-v {wrapper.c_dict['VERBOSITY']}"
config_file = wrapper.c_dict.get('CONFIG_FILE')
out_dir = wrapper.c_dict.get('OUTPUT_DIR')
expected_cmds = [(f"{app_path} {verbosity} "
f"-bdeck {bdeck_dir} "
f"-adeck {adeck_dir} "
f"-config {config_file} "
f"-out {out_dir}/tc_pairs"),
]
env_var_values[f'METPLUS_{loop_by}_END'] = (
f'{loop_by.lower()}_end = "{run_times[-1]}";'
)

all_cmds = wrapper.run_all_times()
print(f"ALL COMMANDS: {all_cmds}")
assert(len(all_cmds) == len(expected_cmds))

for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds):
# check that environment variables were set properly
for env_var_key in wrapper.WRAPPER_ENV_VAR_KEYS:
match = next((item for item in env_vars if
item.startswith(env_var_key)), None)
assert(match is not None)
print(f'Checking env var: {env_var_key}')
actual_value = match.split('=', 1)[1]
assert(env_var_values.get(env_var_key, '') == actual_value)
wrapper = TCPairsWrapper(config)
assert wrapper.isOK

app_path = os.path.join(config.getdir('MET_BIN_DIR'), wrapper.app_name)
verbosity = f"-v {wrapper.c_dict['VERBOSITY']}"
config_file = wrapper.c_dict.get('CONFIG_FILE')
out_dir = wrapper.c_dict.get('OUTPUT_DIR')
expected_cmds = [(f"{app_path} {verbosity} "
f"-bdeck {bdeck_dir} "
f"-adeck {adeck_dir} "
f"-config {config_file} "
f"-out {out_dir}/tc_pairs"),
]

all_cmds = wrapper.run_all_times()
print(f"ALL COMMANDS: {all_cmds}")
assert(len(all_cmds) == len(expected_cmds))

for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds):
# check that environment variables were set properly
for env_var_key in wrapper.WRAPPER_ENV_VAR_KEYS:
match = next((item for item in env_vars if
item.startswith(env_var_key)), None)
assert(match is not None)
print(f'Checking env var: {env_var_key}')
actual_value = match.split('=', 1)[1]
assert(env_var_values.get(env_var_key, '') == actual_value)

# unset begin and end for next loop
del env_var_values[f'METPLUS_{loop_by}_BEG']
del env_var_values[f'METPLUS_{loop_by}_END']
20 changes: 20 additions & 0 deletions metplus/wrappers/command_builder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2283,3 +2283,23 @@ def get_met_config(self, **kwargs):
@returns METConfigInfo object
"""
return met_config(**kwargs)

def get_start_time_input_dict(self):
"""! Get the first run time specified in config. Used if only running
the wrapper once (LOOP_ORDER = processes).

@returns dictionary containing time information for first run time
"""
use_init = util.is_loop_by_init(self.config)
if use_init is None:
self.log_error('Could not read time info')
return None


start_time, _, _ = util.get_start_end_interval_times(self.config)
if start_time is None:
self.log_error("Could not get start time")
return None

input_dict = util.set_input_dict(start_time, self.config, use_init)
return input_dict
Loading