From 510c379068a5dc1d60675c23d3441826294e31d8 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Tue, 24 May 2022 09:21:48 -0600 Subject: [PATCH 1/3] Per #1608, added support for loading files that match MODE*.txt and MTD*.txt. Improved logic to check for directories that contain files that can be loaded into the database by adding functions to check file names. Added unit tests for new functions --- .../pytests/met_db_load/test_met_db_load.py | 40 +++++++++++++ metplus/wrappers/met_db_load_wrapper.py | 58 +++++++++++-------- 2 files changed, 74 insertions(+), 24 deletions(-) create mode 100644 internal_tests/pytests/met_db_load/test_met_db_load.py diff --git a/internal_tests/pytests/met_db_load/test_met_db_load.py b/internal_tests/pytests/met_db_load/test_met_db_load.py new file mode 100644 index 0000000000..4f536c1c86 --- /dev/null +++ b/internal_tests/pytests/met_db_load/test_met_db_load.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 + +import pytest + +from metplus.wrappers.met_db_load_wrapper import METDbLoadWrapper + +@pytest.mark.parametrize( + 'filename, expected_result', [ + ('myfile.png', False), + ('anotherfile.txt', False), + ('goodfile.stat', True), + ('goodfile.tcst', True), + ('MODE_goodfile.txt', True), + ('MTD_goodfile.txt', True), + ('MONSTER_badfile.txt', False), + ] +) +def test_is_loadable_file(filename, expected_result): + assert METDbLoadWrapper._is_loadable_file(filename) == expected_result + +@pytest.mark.parametrize( + 'filenames, expected_result', [ + (['myfile.png', + 'anotherfile.txt'], False), + (['myfile.png', + 'goodfile.stat'], True), + (['myfile.png', + 'goodfile.tcst', + 'anotherfile.txt'], True), + (['myfile.png', + 'MODE_goodfile.txt'], True), + (['myfile.png', + 'MTD_goodfile.txt'], True), + (['myfile.png', + 'MONSTER_badfile.txt'], False), + ([], False), + ] +) +def test_has_loadable_file(filenames, expected_result): + assert METDbLoadWrapper._has_loadable_file(filenames) == expected_result diff --git a/metplus/wrappers/met_db_load_wrapper.py b/metplus/wrappers/met_db_load_wrapper.py index 65f075792b..63910ba9e9 100755 --- a/metplus/wrappers/met_db_load_wrapper.py +++ b/metplus/wrappers/met_db_load_wrapper.py @@ -28,21 +28,22 @@ class METDbLoadWrapper(RuntimeFreqWrapper): and all c_dict values are prepended with MV_. The name is the key and string specifying the type is the value. """ - CONFIG_NAMES = {'HOST': 'string', - 'DATABASE': 'string', - 'USER': 'string', - 'PASSWORD': 'string', - 'VERBOSE': 'bool', - 'INSERT_SIZE': 'int', - 'MODE_HEADER_DB_CHECK': 'bool', - 'DROP_INDEXES': 'bool', - 'APPLY_INDEXES': 'bool', - 'GROUP': 'string', - 'LOAD_STAT': 'bool', - 'LOAD_MODE': 'bool', - 'LOAD_MTD': 'bool', - 'LOAD_MPR': 'bool', - } + CONFIG_NAMES = { + 'HOST': 'string', + 'DATABASE': 'string', + 'USER': 'string', + 'PASSWORD': 'string', + 'VERBOSE': 'bool', + 'INSERT_SIZE': 'int', + 'MODE_HEADER_DB_CHECK': 'bool', + 'DROP_INDEXES': 'bool', + 'APPLY_INDEXES': 'bool', + 'GROUP': 'string', + 'LOAD_STAT': 'bool', + 'LOAD_MODE': 'bool', + 'LOAD_MTD': 'bool', + 'LOAD_MPR': 'bool', + } def __init__(self, config, instance=None): met_data_db_dir = config.getdir('MET_DATA_DB_DIR') @@ -155,7 +156,7 @@ def get_all_files(self, custom=None): def get_stat_directories(self, input_paths): """! Traverse through files under input path and find all directories - that contain .stat or .tcst files. + that contain .stat, .tcst, MODE*.txt, and MTD*.txt files. @param input_path top level directory to search @returns list of unique directories that contain stat files @@ -165,13 +166,8 @@ def get_stat_directories(self, input_paths): self.logger.debug("Finding directories with stat files " f"under {input_path}") for root, _, files in os.walk(input_path): - for filename in files: - if (not filename.endswith('.stat') and - not filename.endswith('.tcst')): - continue - filepath = os.path.join(root, filename) - stat_dir = os.path.dirname(filepath) - stat_dirs.add(stat_dir) + if self._has_loadable_file(files): + stat_dirs.add(root) stat_dirs = list(stat_dirs) for stat_dir in stat_dirs: @@ -179,7 +175,21 @@ def get_stat_directories(self, input_paths): return stat_dirs - def format_stat_dirs(self, stat_dirs): + @staticmethod + def _has_loadable_file(files): + return any([filename for filename in files + if METDbLoadWrapper._is_loadable_file(filename)]) + + @staticmethod + def _is_loadable_file(filename): + return (filename.endswith('.stat') or + filename.endswith('.tcst') or + (filename.endswith('.txt') and + (filename.startswith('MODE') or + filename.startswith('MTD')))) + + @staticmethod + def format_stat_dirs(stat_dirs): """! Format list of stat directories to substitute into XML file. tags wil be added around each value. From 3cbbf58b05093a48f9accb57cb0200c5243a7069 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Tue, 24 May 2022 09:21:58 -0600 Subject: [PATCH 2/3] removed unused imports --- .../pytests/grid_stat/test_grid_stat_wrapper.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/internal_tests/pytests/grid_stat/test_grid_stat_wrapper.py b/internal_tests/pytests/grid_stat/test_grid_stat_wrapper.py index 76f25e6d0c..ebfc84a737 100644 --- a/internal_tests/pytests/grid_stat/test_grid_stat_wrapper.py +++ b/internal_tests/pytests/grid_stat/test_grid_stat_wrapper.py @@ -1,18 +1,10 @@ #!/usr/bin/env python3 -import os -import sys -import re -import logging -from collections import namedtuple import pytest +import os from datetime import datetime -import produtil - from metplus.wrappers.grid_stat_wrapper import GridStatWrapper -from metplus.util import met_util as util -from metplus.util import time_util fcst_dir = '/some/path/fcst' obs_dir = '/some/path/obs' From 6425e7a761c0cb6a5545c9787bea6780aaf62571 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Tue, 24 May 2022 16:27:27 -0600 Subject: [PATCH 3/3] Per #1608, look for files that start with lowercase mode or mtd, not uppercase --- .../pytests/met_db_load/test_met_db_load.py | 12 ++++++------ metplus/wrappers/met_db_load_wrapper.py | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/internal_tests/pytests/met_db_load/test_met_db_load.py b/internal_tests/pytests/met_db_load/test_met_db_load.py index 4f536c1c86..7e881d6563 100644 --- a/internal_tests/pytests/met_db_load/test_met_db_load.py +++ b/internal_tests/pytests/met_db_load/test_met_db_load.py @@ -10,9 +10,9 @@ ('anotherfile.txt', False), ('goodfile.stat', True), ('goodfile.tcst', True), - ('MODE_goodfile.txt', True), - ('MTD_goodfile.txt', True), - ('MONSTER_badfile.txt', False), + ('mode_goodfile.txt', True), + ('mtd_goodfile.txt', True), + ('monster_badfile.txt', False), ] ) def test_is_loadable_file(filename, expected_result): @@ -28,11 +28,11 @@ def test_is_loadable_file(filename, expected_result): 'goodfile.tcst', 'anotherfile.txt'], True), (['myfile.png', - 'MODE_goodfile.txt'], True), + 'mode_goodfile.txt'], True), (['myfile.png', - 'MTD_goodfile.txt'], True), + 'mtd_goodfile.txt'], True), (['myfile.png', - 'MONSTER_badfile.txt'], False), + 'monster_badfile.txt'], False), ([], False), ] ) diff --git a/metplus/wrappers/met_db_load_wrapper.py b/metplus/wrappers/met_db_load_wrapper.py index 63910ba9e9..63fe189714 100755 --- a/metplus/wrappers/met_db_load_wrapper.py +++ b/metplus/wrappers/met_db_load_wrapper.py @@ -156,7 +156,7 @@ def get_all_files(self, custom=None): def get_stat_directories(self, input_paths): """! Traverse through files under input path and find all directories - that contain .stat, .tcst, MODE*.txt, and MTD*.txt files. + that contain .stat, .tcst, mode*.txt, and mtd*.txt files. @param input_path top level directory to search @returns list of unique directories that contain stat files @@ -185,8 +185,8 @@ def _is_loadable_file(filename): return (filename.endswith('.stat') or filename.endswith('.tcst') or (filename.endswith('.txt') and - (filename.startswith('MODE') or - filename.startswith('MTD')))) + (filename.startswith('mode') or + filename.startswith('mtd')))) @staticmethod def format_stat_dirs(stat_dirs):