Skip to content

Commit

Permalink
Feature 2253 py embed test (#2379)
Browse files Browse the repository at this point in the history
  • Loading branch information
@reza-armuei
reza-armuei authored Oct 18, 2023
1 parent 7619276 commit 4e42ab4
Showing 1 changed file with 200 additions and 0 deletions.
200 changes: 200 additions & 0 deletions internal/tests/pytests/wrappers/py_embed_ingest/test_py_embed_ingest.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,200 @@
import os
import datetime
import pytest
from unittest import mock
from metplus.wrappers.py_embed_ingest_wrapper import PyEmbedIngestWrapper
from metplus.wrappers.command_builder import CommandBuilder


time_fmt = "%Y%m%d%H"
run_times = ["2023100600", "2023100612", "2023100700"]
time_info = {
"loop_by": "init",
"init": datetime.datetime(2023, 10, 6, 0, 0),
"now": datetime.datetime(2023, 10, 6, 0, 0),
"today": "20231010",
"instance": "",
"valid": "*",
"lead": "*",
}

# Helper class for string matching
class MatchSubstring(str):
def __eq__(self, other):
return self in other

def mock_get_ingest_items(item_type, index, ingest_script_addons):
# mock `get_ingest_items` to create a test case for config error log where number of
# output fields does not the same as number of script.
if item_type=="OUTPUT_FIELD_NAME":
return ["field_1", "field_2"]
else:
return ["field_1"]

def set_minimum_config_settings(config):
# set config variables to prevent command from running and bypass check
# if input files actually exist
config.set("config", "DO_NOT_RUN_EXE", True)
config.set("config", "INPUT_MUST_EXIST", False)

# set process and time config variables
config.set("config", "PROCESS_LIST", "PyEmbedIngest")
config.set("config", "LOOP_BY", "INIT")
config.set("config", "INIT_TIME_FMT", time_fmt)
config.set("config", "INIT_BEG", run_times[0])
config.set("config", "INIT_END", run_times[-1])
config.set("config", "INIT_INCREMENT", "12H")
config.set("config", "LEAD_SEQ", "12H")


@pytest.mark.parametrize(
'config_overrides, mock_file_check, expected_values, expected_error_numbers', [
# 0 a case with one index while find_and_check_output_file returns True
(
{
'PY_EMBED_INGEST_1_OUTPUT_DIR': "outdir/",
"PY_EMBED_INGEST_1_OUTPUT_TEMPLATE": "output_file.nc",
"PY_EMBED_INGEST_1_SCRIPT": "fake_script",
"PY_EMBED_INGEST_1_TYPE": "NUMPY",
"PY_EMBED_INGEST_1_OUTPUT_GRID": "fake_grid",
"PY_EMBED_INGEST_1_OUTPUT_FIELD_NAME": "fake_filed"
},
True,
[
{
"output_dir": "outdir/",
"output_template": "output_file.nc",
"output_field_names": ["fake_filed"],
"scripts": ["fake_script"],
"input_type": "NUMPY",
"output_grid": "fake_grid",
"index": "1",
}
],
1,
),
# 1 a case with one index while find_and_check_output_file returns False
(
{
'PY_EMBED_INGEST_1_OUTPUT_DIR': "outdir/",
"PY_EMBED_INGEST_1_OUTPUT_TEMPLATE": "output_file.nc",
"PY_EMBED_INGEST_1_SCRIPT": "fake_script",
"PY_EMBED_INGEST_1_TYPE": "NUMPY",
"PY_EMBED_INGEST_1_OUTPUT_GRID": "fake_grid",
"PY_EMBED_INGEST_1_OUTPUT_FIELD_NAME": "fake_filed"
},
False,
[{
"output_dir": "outdir/",
"output_template": "output_file.nc",
"output_field_names": ["fake_filed"],
"scripts": ["fake_script"],
"input_type": "NUMPY",
"output_grid": "fake_grid",
"index": "1",
}],
0,
),
# 2 a case with two indices while find_and_check_output_file returns True
(
{
'PY_EMBED_INGEST_1_OUTPUT_DIR': "outdir/",
"PY_EMBED_INGEST_1_OUTPUT_TEMPLATE": "first_output_file.nc",
"PY_EMBED_INGEST_1_SCRIPT": "first_fake_script",
"PY_EMBED_INGEST_1_TYPE": "NUMPY",
"PY_EMBED_INGEST_1_OUTPUT_GRID": "first_fake_grid",
"PY_EMBED_INGEST_1_OUTPUT_FIELD_NAME": "first_fake_filed",
'PY_EMBED_INGEST_2_OUTPUT_DIR': "outdir/",
"PY_EMBED_INGEST_2_OUTPUT_TEMPLATE": "second_output_file.nc",
"PY_EMBED_INGEST_2_SCRIPT": "second_fake_script",
"PY_EMBED_INGEST_2_TYPE": "NUMPY",
"PY_EMBED_INGEST_2_OUTPUT_GRID": "second_fake_grid",
"PY_EMBED_INGEST_2_OUTPUT_FIELD_NAME": "second_fake_filed",
},
True,
[
{
"output_dir": "outdir/",
"output_template": "first_output_file.nc",
"output_field_names": ["first_fake_filed"],
"scripts": ["first_fake_script"],
"input_type": "NUMPY",
"output_grid": "first_fake_grid",
"index": "1",
},
{
"output_dir": "outdir/",
"output_template": "second_output_file.nc",
"output_field_names": ["second_fake_filed"],
"scripts": ["second_fake_script"],
"input_type": "NUMPY",
"output_grid": "second_fake_grid",
"index": "2",
},
],
2,
),
]
)
@pytest.mark.wrapper_b
def test_required_job_template(
metplus_config,
mock_file_check,
config_overrides,
expected_values,
expected_error_numbers
):
config = metplus_config

set_minimum_config_settings(config)
with mock.patch.object(CommandBuilder, "build", return_value=False):
with mock.patch.object(
CommandBuilder, "find_and_check_output_file", return_value=mock_file_check
):
# set config variable overrides
for key, value in config_overrides.items():
config.set('config', key, value)

wrapper = PyEmbedIngestWrapper(config)
assert wrapper.isOK
assert wrapper.c_dict["INGESTERS"] == expected_values
assert wrapper.run_at_time_once(time_info) == True
assert wrapper.errors == expected_error_numbers

@pytest.mark.wrapper
def test_conf_error_log(metplus_config, monkeypatch):
'''Check correct error messages are logged'''
config = metplus_config
set_minimum_config_settings(config)

config_overrides = {
'PY_EMBED_INGEST_1_OUTPUT_DIR': "outdir/",
"PY_EMBED_INGEST_1_OUTPUT_TEMPLATE": "output_file.nc",
"PY_EMBED_INGEST_1_SCRIPT": "fake_script xv mv",
"PY_EMBED_INGEST_1_TYPE": "PANDAS",
"PY_EMBED_INGEST_1_OUTPUT_GRID": "fake_grid",
"PY_EMBED_INGEST_1_OUTPUT_FIELD_NAME": "fake_filed",
'PY_EMBED_INGEST_2_OUTPUT_DIR': "outdir/",
"PY_EMBED_INGEST_2_OUTPUT_TEMPLATE": "output_file.nc",
"PY_EMBED_INGEST_2_SCRIPT": "fake_script",
"PY_EMBED_INGEST_2_TYPE": "FAKE",
"PY_EMBED_INGEST_2_OUTPUT_GRID": "",
"PY_EMBED_INGEST_2_OUTPUT_FIELD_NAME": "fake_filed",
"PY_EMBED_INGEST_2_OUTPUT_FIELD_NAME_2": "fake_filed2"
}
# set config variable overrides
for key, value in config_overrides.items():
config.set('config', key, value)
wrapper = PyEmbedIngestWrapper(config)
expected_error = [
'Running PyEmbedIngester on pandas data not yet implemented',
'PY_EMBED_INGEST_2_TYPE (FAKE) not valid. Valid types are NUMPY, XARRAY, PANDAS',
'Must set PY_EMBED_INGEST_2_OUTPUT_GRID',
"If using PY_EMBED_INGEST_1_OUTPUT_FIELD_NAME*, the number of output names must"
" match the number of PY_EMBED_INGEST_1_SCRIPT* values"
]
with mock.patch.object(wrapper, "get_ingest_items", mock_get_ingest_items):
wrapper.create_c_dict()

for msg in expected_error:
wrapper.logger.error.assert_any_call(MatchSubstring(msg))

0 comments on commit 4e42ab4

Please sign in to comment.