Skip to content

Commit

Permalink
feature 986 TCPairs valid time (#997)
Browse files Browse the repository at this point in the history
  • Loading branch information
@georgemccabe
georgemccabe authored Jul 15, 2021
1 parent 4c7d03b commit 858eca8
Show file tree
Hide file tree
Showing 10 changed files with 211 additions and 147 deletions.
9 changes: 9 additions & 0 deletions docs/Users_Guide/glossary.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7162,3 +7162,12 @@ METplus Configuration Glossary
variable OR :term:`EXTRACT_TILES_TC_STAT_INPUT_TEMPLATE` but not both.

| *Used by:* ExtractTiles
TC_PAIRS_SKIP_LEAD_SEQ
If True and a forecast lead sequence is set in the configuration, do not
loop over list of leads and process for each. This is used for feature
relative use cases where TCPairs is run for each storm initialization time
and SeriesAnalysis is configured to filter the data by forecast leads.
Default value is False.

| *Used by:* TCPairs
1 change: 1 addition & 0 deletions docs/Users_Guide/wrappers.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5393,6 +5393,7 @@ METplus Configuration
| :term:`TC_PAIRS_CONSENSUS<n>_MEMBERS`
| :term:`TC_PAIRS_CONSENSUS<n>_REQUIRED`
| :term:`TC_PAIRS_CONSENSUS<n>_MIN_REQ`
| :term:`TC_PAIRS_SKIP_LEAD_SEQ`
|
.. warning:: **DEPRECATED:**
Expand Down
250 changes: 135 additions & 115 deletions internal_tests/pytests/tc_pairs/test_tc_pairs_wrapper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,19 +15,19 @@
time_fmt = '%Y%m%d%H'
run_times = ['2014121318']

def set_minimum_config_settings(config):
def set_minimum_config_settings(config, loop_by='INIT'):
# set config variables to prevent command from running and bypass check
# if input files actually exist
config.set('config', 'DO_NOT_RUN_EXE', True)
config.set('config', 'INPUT_MUST_EXIST', False)

# set process and time config variables
config.set('config', 'PROCESS_LIST', 'TCPairs')
config.set('config', 'LOOP_BY', 'INIT')
config.set('config', 'INIT_TIME_FMT', time_fmt)
config.set('config', 'INIT_BEG', run_times[0])
config.set('config', 'INIT_END', run_times[-1])
config.set('config', 'INIT_INCREMENT', '12H')
config.set('config', 'LOOP_BY', loop_by)
config.set('config', f'{loop_by}_TIME_FMT', time_fmt)
config.set('config', f'{loop_by}_BEG', run_times[0])
config.set('config', f'{loop_by}_END', run_times[-1])
config.set('config', f'{loop_by}_INCREMENT', '12H')
config.set('config', 'TC_PAIRS_CONFIG_FILE',
'{PARM_BASE}/met_config/TCPairsConfig_wrapped')
config.set('config', 'TC_PAIRS_BDECK_TEMPLATE', bdeck_template)
Expand Down Expand Up @@ -264,7 +264,7 @@ def test_tc_pairs_storm_id_lists(metplus_config, config_overrides,
config.set('config', key, value)

wrapper = TCPairsWrapper(config)
assert(wrapper.isOK)
assert wrapper.isOK

all_cmds = wrapper.run_all_times()
print(f"ALL COMMANDS:")
Expand Down Expand Up @@ -356,65 +356,78 @@ def test_tc_pairs_storm_id_lists(metplus_config, config_overrides,
)
def test_tc_pairs_loop_order_processes(metplus_config, config_overrides,
env_var_values):

config = metplus_config()

set_minimum_config_settings(config)

test_data_dir = os.path.join(config.getdir('METPLUS_BASE'),
'internal_tests',
'data',
'tc_pairs')
bdeck_dir = os.path.join(test_data_dir, 'bdeck')
adeck_dir = os.path.join(test_data_dir, 'adeck')

config.set('config', 'TC_PAIRS_BDECK_INPUT_DIR', bdeck_dir)
config.set('config', 'TC_PAIRS_ADECK_INPUT_DIR', adeck_dir)

# LOOP_ORDER processes runs once, times runs once per time
config.set('config', 'LOOP_ORDER', 'processes')

# set config variable overrides
for key, value in config_overrides.items():
config.set('config', key, value)

if 'METPLUS_INIT_BEG' not in env_var_values:
env_var_values['METPLUS_INIT_BEG'] = f'init_beg = "{run_times[0]}";'

if 'METPLUS_INIT_END' not in env_var_values:
env_var_values['METPLUS_INIT_END'] = f'init_end = "{run_times[-1]}";'

wrapper = TCPairsWrapper(config)
assert(wrapper.isOK)

app_path = os.path.join(config.getdir('MET_BIN_DIR'), wrapper.app_name)
verbosity = f"-v {wrapper.c_dict['VERBOSITY']}"
config_file = wrapper.c_dict.get('CONFIG_FILE')
out_dir = wrapper.c_dict.get('OUTPUT_DIR')
expected_cmds = [(f"{app_path} {verbosity} "
f"-bdeck {bdeck_dir}/bmlq2014123118.gfso.0104 "
f"-adeck {adeck_dir}/amlq2014123118.gfso.0104 "
f"-config {config_file} "
f"-out {out_dir}/mlq2014121318.gfso.0104"),
]


all_cmds = wrapper.run_all_times()
print(f"ALL COMMANDS: {all_cmds}")
assert(len(all_cmds) == len(expected_cmds))

for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds):
# ensure commands are generated as expected
assert(cmd == expected_cmd)

# check that environment variables were set properly
for env_var_key in wrapper.WRAPPER_ENV_VAR_KEYS:
match = next((item for item in env_vars if
item.startswith(env_var_key)), None)
assert(match is not None)
print(f'Checking env var: {env_var_key}')
actual_value = match.split('=', 1)[1]
assert(env_var_values.get(env_var_key, '') == actual_value)
# run using init and valid time variables
for loop_by in ['INIT', 'VALID']:
remove_beg = remove_end = False
config = metplus_config()

set_minimum_config_settings(config, loop_by)

test_data_dir = os.path.join(config.getdir('METPLUS_BASE'),
'internal_tests',
'data',
'tc_pairs')
bdeck_dir = os.path.join(test_data_dir, 'bdeck')
adeck_dir = os.path.join(test_data_dir, 'adeck')

config.set('config', 'TC_PAIRS_BDECK_INPUT_DIR', bdeck_dir)
config.set('config', 'TC_PAIRS_ADECK_INPUT_DIR', adeck_dir)

# LOOP_ORDER processes runs once, times runs once per time
config.set('config', 'LOOP_ORDER', 'processes')

# set config variable overrides
for key, value in config_overrides.items():
config.set('config', key, value)

if f'METPLUS_{loop_by}_BEG' not in env_var_values:
env_var_values[f'METPLUS_{loop_by}_BEG'] = (
f'{loop_by.lower()}_beg = "{run_times[0]}";'
)
remove_beg = True

if f'METPLUS_{loop_by}_END' not in env_var_values:
env_var_values[f'METPLUS_{loop_by}_END'] = (
f'{loop_by.lower()}_end = "{run_times[-1]}";'
)
remove_end = True

wrapper = TCPairsWrapper(config)
assert wrapper.isOK

app_path = os.path.join(config.getdir('MET_BIN_DIR'), wrapper.app_name)
verbosity = f"-v {wrapper.c_dict['VERBOSITY']}"
config_file = wrapper.c_dict.get('CONFIG_FILE')
out_dir = wrapper.c_dict.get('OUTPUT_DIR')
expected_cmds = [(f"{app_path} {verbosity} "
f"-bdeck {bdeck_dir}/bmlq2014123118.gfso.0104 "
f"-adeck {adeck_dir}/amlq2014123118.gfso.0104 "
f"-config {config_file} "
f"-out {out_dir}/mlq2014121318.gfso.0104"),
]


all_cmds = wrapper.run_all_times()
print(f"ALL COMMANDS: {all_cmds}")
assert(len(all_cmds) == len(expected_cmds))

for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds):
# ensure commands are generated as expected
assert(cmd == expected_cmd)

# check that environment variables were set properly
for env_var_key in wrapper.WRAPPER_ENV_VAR_KEYS:
match = next((item for item in env_vars if
item.startswith(env_var_key)), None)
assert(match is not None)
print(f'Checking env var: {env_var_key}')
actual_value = match.split('=', 1)[1]
assert(env_var_values.get(env_var_key, '') == actual_value)

if remove_beg:
del env_var_values[f'METPLUS_{loop_by}_BEG']
if remove_end:
del env_var_values[f'METPLUS_{loop_by}_END']

@pytest.mark.parametrize(
'config_overrides, env_var_values', [
Expand All @@ -433,61 +446,68 @@ def test_tc_pairs_loop_order_processes(metplus_config, config_overrides,
)
def test_tc_pairs_read_all_files(metplus_config, config_overrides,
env_var_values):
# run using init and valid time variables
for loop_by in ['INIT', 'VALID']:
config = metplus_config()

config = metplus_config()
set_minimum_config_settings(config, loop_by)

set_minimum_config_settings(config)
test_data_dir = os.path.join(config.getdir('METPLUS_BASE'),
'internal_tests',
'data',
'tc_pairs')
bdeck_dir = os.path.join(test_data_dir, 'bdeck')
adeck_dir = os.path.join(test_data_dir, 'adeck')

test_data_dir = os.path.join(config.getdir('METPLUS_BASE'),
'internal_tests',
'data',
'tc_pairs')
bdeck_dir = os.path.join(test_data_dir, 'bdeck')
adeck_dir = os.path.join(test_data_dir, 'adeck')
config.set('config', 'TC_PAIRS_BDECK_INPUT_DIR', bdeck_dir)
config.set('config', 'TC_PAIRS_ADECK_INPUT_DIR', adeck_dir)

config.set('config', 'TC_PAIRS_BDECK_INPUT_DIR', bdeck_dir)
config.set('config', 'TC_PAIRS_ADECK_INPUT_DIR', adeck_dir)
# LOOP_ORDER processes runs once, times runs once per time
config.set('config', 'LOOP_ORDER', 'processes')

# LOOP_ORDER processes runs once, times runs once per time
config.set('config', 'LOOP_ORDER', 'processes')

config.set('config', 'TC_PAIRS_READ_ALL_FILES', True)
config.set('config', 'TC_PAIRS_OUTPUT_TEMPLATE', '')
config.set('config', 'TC_PAIRS_READ_ALL_FILES', True)
config.set('config', 'TC_PAIRS_OUTPUT_TEMPLATE', '')

# set config variable overrides
for key, value in config_overrides.items():
config.set('config', key, value)
# set config variable overrides
for key, value in config_overrides.items():
config.set('config', key, value)

if 'METPLUS_INIT_BEG' not in env_var_values:
env_var_values['METPLUS_INIT_BEG'] = f'init_beg = "{run_times[0]}";'

if 'METPLUS_INIT_END' not in env_var_values:
env_var_values['METPLUS_INIT_END'] = f'init_end = "{run_times[-1]}";'
env_var_values[f'METPLUS_{loop_by}_BEG'] = (
f'{loop_by.lower()}_beg = "{run_times[0]}";'
)

wrapper = TCPairsWrapper(config)
assert(wrapper.isOK)

app_path = os.path.join(config.getdir('MET_BIN_DIR'), wrapper.app_name)
verbosity = f"-v {wrapper.c_dict['VERBOSITY']}"
config_file = wrapper.c_dict.get('CONFIG_FILE')
out_dir = wrapper.c_dict.get('OUTPUT_DIR')
expected_cmds = [(f"{app_path} {verbosity} "
f"-bdeck {bdeck_dir} "
f"-adeck {adeck_dir} "
f"-config {config_file} "
f"-out {out_dir}/tc_pairs"),
]
env_var_values[f'METPLUS_{loop_by}_END'] = (
f'{loop_by.lower()}_end = "{run_times[-1]}";'
)

all_cmds = wrapper.run_all_times()
print(f"ALL COMMANDS: {all_cmds}")
assert(len(all_cmds) == len(expected_cmds))

for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds):
# check that environment variables were set properly
for env_var_key in wrapper.WRAPPER_ENV_VAR_KEYS:
match = next((item for item in env_vars if
item.startswith(env_var_key)), None)
assert(match is not None)
print(f'Checking env var: {env_var_key}')
actual_value = match.split('=', 1)[1]
assert(env_var_values.get(env_var_key, '') == actual_value)
wrapper = TCPairsWrapper(config)
assert wrapper.isOK

app_path = os.path.join(config.getdir('MET_BIN_DIR'), wrapper.app_name)
verbosity = f"-v {wrapper.c_dict['VERBOSITY']}"
config_file = wrapper.c_dict.get('CONFIG_FILE')
out_dir = wrapper.c_dict.get('OUTPUT_DIR')
expected_cmds = [(f"{app_path} {verbosity} "
f"-bdeck {bdeck_dir} "
f"-adeck {adeck_dir} "
f"-config {config_file} "
f"-out {out_dir}/tc_pairs"),
]

all_cmds = wrapper.run_all_times()
print(f"ALL COMMANDS: {all_cmds}")
assert(len(all_cmds) == len(expected_cmds))

for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds):
# check that environment variables were set properly
for env_var_key in wrapper.WRAPPER_ENV_VAR_KEYS:
match = next((item for item in env_vars if
item.startswith(env_var_key)), None)
assert(match is not None)
print(f'Checking env var: {env_var_key}')
actual_value = match.split('=', 1)[1]
assert(env_var_values.get(env_var_key, '') == actual_value)

# unset begin and end for next loop
del env_var_values[f'METPLUS_{loop_by}_BEG']
del env_var_values[f'METPLUS_{loop_by}_END']
20 changes: 20 additions & 0 deletions metplus/wrappers/command_builder.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2283,3 +2283,23 @@ def get_met_config(self, **kwargs):
@returns METConfigInfo object
"""
return met_config(**kwargs)

def get_start_time_input_dict(self):
"""! Get the first run time specified in config. Used if only running
the wrapper once (LOOP_ORDER = processes).
@returns dictionary containing time information for first run time
"""
use_init = util.is_loop_by_init(self.config)
if use_init is None:
self.log_error('Could not read time info')
return None


start_time, _, _ = util.get_start_end_interval_times(self.config)
if start_time is None:
self.log_error("Could not get start time")
return None

input_dict = util.set_input_dict(start_time, self.config, use_init)
return input_dict
Loading

0 comments on commit 858eca8

Please sign in to comment.