Bugfix #2705 develop - SeriesAnalysis fix time substitution into fiel…

…d info (#2709)

* fix SonarQube complaints

* Per #2705, set correct time information in field info by using the time info used to find the input files (self.c_dict['ALL_FILES']) instead of relying on parsing the time info from the other input files (fcst vs. obs)

* clean up unused code after bugfix for #2705

internal/tests/pytests/util/string_template_substitution/test_string_template_substitution.py

@@ -354,15 +354,15 @@ def test_filename_does_not_match_template():
     template = "{init?fmt=%Y%m%d%H}_dog_A{lead?fmt=%HH}h"
     filepath = "1987020103_cat_A03h"
     out = parse_template(template, filepath)
-    assert out == None
+    assert out is None
 
 
 @pytest.mark.util
 def test_filename_does_not_match_template_end():
     template = "{init?fmt=%Y%m%d%H}_dog_A{lead?fmt=%HH}h"
     filepath = "1987020103_dog_A03d"
     out = parse_template(template, filepath)
-    assert out == None
+    assert out is None
 
 
 @pytest.mark.util
@@ -379,7 +379,7 @@ def test_get_tags():
 # value is the formatted output string, like 01
 # ttype is the unit to check, i.e. 'H', 'M', 'S', 'd', 's'
 @pytest.mark.parametrize(
-    'format, key, value, ttype', [
+    'fmt, key, value, ttype', [
         ('H', 1, '01', 'H'),
         ('1H', 1, '1', 'H'),
         ('2H', 1, '01', 'H'),
@@ -418,15 +418,15 @@ def test_get_tags():
     ]
 )
 @pytest.mark.util
-def test_format_one_time_item(format, key ,value, ttype):
-    assert format_one_time_item(format, key, ttype) == value
+def test_format_one_time_item(fmt, key ,value, ttype):
+    assert format_one_time_item(fmt, key, ttype) == value
 
 
 # format is the time format to use, like, %M or %H%M
 # seconds is the integer number of seconds of the offset to use, i.e. 3601
 # value is the formatted output string, like 010001
 @pytest.mark.parametrize(
-    'format, seconds, value', [
+    'fmt, seconds, value', [
         ('%H', 1, '00'),
         ('%M', 1, '00'),
         ('%S', 1, '01'),
@@ -448,9 +448,9 @@ def test_format_one_time_item(format, key ,value, ttype):
     ]
 )
 @pytest.mark.util
-def test_format_hms(format, seconds, value):
+def test_format_hms(fmt, seconds, value):
     # format should be something like %M or %H%M
-    assert format_hms(format, seconds == value)
+    assert format_hms(fmt, seconds == value)
 
 
 @pytest.mark.util
@@ -539,7 +539,6 @@ def test_populate_match_dict(template, filepath, expected_match_dict, expected_v
         elif match_dict is None:
             # if expected is not None, fail if actual is None
             assert False
-            return
 
         num_keys = len(match_dict.keys())
         expected_num_keys = len(expected_match_dict.keys())
@@ -562,8 +561,6 @@ def test_populate_match_dict(template, filepath, expected_match_dict, expected_v
             print(f"Incorrect valid shift. Actual {valid_shift}, Expected: {expected_valid_shift}")
             assert False
 
-        assert True
-
     except TypeError:
         assert expected_match_dict is None and expected_valid_shift is None
 
@@ -591,8 +588,6 @@ def test_get_fmt_info(fmt, filepath, identifier, expected_fmt_len, expected_matc
         print(f"Expected Dictionary: {expected_match_dict}")
         assert False
 
-    assert True
-
 
 @pytest.mark.parametrize(
     'templ, expected_filename', [

metplus/wrappers/series_analysis_wrapper.py

@@ -784,9 +784,7 @@ def build_and_run_series_request(self, time_info, fcst_path, obs_path):
             add_field_info_to_time_info(time_info, var_info)
 
             # get formatted field dictionary to pass into the MET config file
-            fcst_field, obs_field = (
-                self.get_formatted_fields(var_info, time_info, fcst_path, obs_path)
-            )
+            fcst_field, obs_field = self.get_formatted_fields(var_info, time_info)
             if fcst_field is None:
                 continue
 
@@ -1035,22 +1033,20 @@ def _get_netcdf_min_max(filepath, variable_name):
         except (FileNotFoundError, KeyError):
             return None, None
 
-    def get_formatted_fields(self, var_info, time_info, fcst_path, obs_path):
+    def get_formatted_fields(self, var_info, time_info):
         """! Get forecast and observation field information for var_info and
             format it so it can be passed into the MET config file
 
             @param var_info dictionary containing info to format
             @param time_info dictionary containing time information
-            @param fcst_path path to file list file for forecast data
-            @param obs_path path to file list file for observation data
             @returns tuple containing strings of the formatted forecast and
             observation information or (None, None) if something went wrong
         """
         fcst_field_list = (
-            self._get_field_list('fcst', var_info, time_info, obs_path)
+            self._get_field_list('fcst', var_info, time_info)
         )
         obs_field_list = (
-            self._get_field_list('obs', var_info, time_info, fcst_path)
+            self._get_field_list('obs', var_info, time_info)
         )
 
         if not fcst_field_list or not obs_field_list:
@@ -1062,7 +1058,7 @@ def get_formatted_fields(self, var_info, time_info, fcst_path, obs_path):
 
         return fcst_fields, obs_fields
 
-    def _get_field_list(self, data_type, var_info, time_info, file_list_path):
+    def _get_field_list(self, data_type, var_info, time_info):
         """!Get formatted field information in a list.
         If no time (init/valid/lead) filename template tags were found in the
         level value or if the time info contains all init/valid/lead values
@@ -1075,10 +1071,8 @@ def _get_field_list(self, data_type, var_info, time_info, file_list_path):
         @param data_type type of data to process, e.g. fcst or obs
         @param var_info dictionary containing info to format
         @param time_info dictionary containing time information
-        @param file_list_path path to file list file to parse
         @returns list containing formatted field info to pass to MET config
         """
-        other = 'OBS' if data_type == 'fcst' else 'FCST'
         # if there are no time tags (init/valid/lead) in the field level
         # or if init, valid, and lead have values in time_info,
         # get field info for a single field to pass to the MET config file
@@ -1088,15 +1082,10 @@ def _get_field_list(self, data_type, var_info, time_info, file_list_path):
 
         field_list = []
 
-        # handle multiple templates
-        templates = []
-        for template in self.c_dict[f'{other}_INPUT_TEMPLATE'].split(','):
-            templates.append(os.path.join(self.c_dict[f'{other}_INPUT_DIR'], template.strip()))
-
-        # loop through fcst/obs files to extract time info
+        # loop through fcst/obs files to read time info
         # for each file apply time info to field info and add to list
-        for file_time_info in self._get_times_from_file_list(file_list_path,
-                                                             templates):
+        for file_dict in self.c_dict['ALL_FILES']:
+            file_time_info = file_dict['time_info']
             field = self._get_field_sub_level(data_type, var_info, file_time_info)
             if field:
                 field_list.extend(field)