feature 1445 normalize in GenEnsProd (#1478)

docs/Users_Guide/glossary.rst

@@ -9152,6 +9152,11 @@ METplus Configuration Glossary
 
      | *Used by:* SeriesAnalysis
 
+   GEN_ENS_PROD_NORMALIZE
+     Specify the value for 'normalize' in the MET configuration file for GenEnsProd.
+
+     | *Used by:* GenEnsProd
+
    ENSEMBLE_STAT_PROB_CAT_THRESH
      Specify the value for 'prob_cat_thresh' in the MET configuration file for EnsembleStat.
 

docs/Users_Guide/wrappers.rst

@@ -1102,6 +1102,7 @@ METplus Configuration
 | :term:`GEN_ENS_PROD_CENSOR_THRESH`
 | :term:`GEN_ENS_PROD_CENSOR_VAL`
 | :term:`GEN_ENS_PROD_CAT_THRESH`
+| :term:`GEN_ENS_PROD_NORMALIZE`
 | :term:`GEN_ENS_PROD_NC_VAR_STR`
 | :term:`GEN_ENS_PROD_ENS_THRESH`
 | :term:`GEN_ENS_PROD_ENS_VLD_THRESH`
@@ -1239,6 +1240,17 @@ see :ref:`How METplus controls MET config file settings<metplus-control-met>`.
    * - :term:`GEN_ENS_PROD_CENSOR_VAL`
      - censor_val
 
+**${METPLUS_NORMALIZE}**
+
+.. list-table::
+   :widths: 5 5
+   :header-rows: 0
+
+   * - METplus Config(s)
+     - MET Config File
+   * - :term:`GEN_ENS_PROD_NORMALIZE`
+     - normalize
+
 **${METPLUS_CAT_THRESH}**
 
 .. list-table::

internal_tests/pytests/ensemble_stat/test_ensemble_stat_wrapper.py

@@ -735,3 +735,39 @@ def test_get_config_file(metplus_config):
     config.set('config', 'ENSEMBLE_STAT_CONFIG_FILE', fake_config_name)
     wrapper = EnsembleStatWrapper(config)
     assert wrapper.c_dict['CONFIG_FILE'] == fake_config_name
+
+@pytest.mark.parametrize(
+    'config_overrides, expected_num_files', [
+        ({}, 4),
+        ({'ENSEMBLE_STAT_ENS_MEMBER_IDS': '1'}, 1),
+    ]
+)
+def test_ensemble_stat_fill_missing(metplus_config, config_overrides,
+                                    expected_num_files):
+    config = metplus_config()
+
+    set_minimum_config_settings(config)
+
+    # change some config values for this test
+    config.set('config', 'INIT_END', run_times[0])
+    config.set('config', 'ENSEMBLE_STAT_N_MEMBERS', 4)
+
+    # set config variable overrides
+    for key, value in config_overrides.items():
+        config.set('config', key, value)
+
+    wrapper = EnsembleStatWrapper(config)
+
+    file_list_file = os.path.join(wrapper.config.getdir('STAGING_DIR'),
+                                  'file_lists',
+                                  '20050807000000_12_ensemble_stat.txt')
+    if os.path.exists(file_list_file):
+        os.remove(file_list_file)
+
+    all_cmds = wrapper.run_all_times()
+    assert len(all_cmds) == 1
+
+    with open(file_list_file, 'r') as file_handle:
+        actual_num_files = len(file_handle.read().splitlines()) - 1
+
+    assert actual_num_files == expected_num_files

internal_tests/pytests/gen_ens_prod/test_gen_ens_prod_wrapper.py

@@ -51,6 +51,15 @@ def set_minimum_config_settings(config):
     config.set('config', 'ENS_VAR1_NAME', ens_name)
     config.set('config', 'ENS_VAR1_LEVELS', ens_level)
 
+def handle_input_dir(config):
+    test_data_dir = os.path.join(config.getdir('METPLUS_BASE'),
+                                 'internal_tests',
+                                 'data')
+    input_dir = os.path.join(test_data_dir, 'ens')
+    config.set('config', 'GEN_ENS_PROD_INPUT_DIR', input_dir)
+    config.set('config', 'GEN_ENS_PROD_CTRL_INPUT_DIR', input_dir)
+    return input_dir
+
 @pytest.mark.parametrize(
     'config_overrides, env_var_values', [
         # 0
@@ -342,12 +351,16 @@ def set_minimum_config_settings(config):
                      'type = [{method = GAUSSIAN;width = 1;}];}'
              )
          }),
-
+        # 59
         ({'GEN_ENS_PROD_ENS_MEMBER_IDS': '1,2,3,4', },
          {'METPLUS_ENS_MEMBER_IDS': 'ens_member_ids = ["1", "2", "3", "4"];'}),
-
+        # 60
         ({'GEN_ENS_PROD_CONTROL_ID': '0', },
          {'METPLUS_CONTROL_ID': 'control_id = "0";'}),
+        # 61
+        ({'GEN_ENS_PROD_NORMALIZE': 'CLIMO_STD_ANOM', },
+         {'METPLUS_NORMALIZE': 'normalize = CLIMO_STD_ANOM;'}),
+
     ]
 )
 def test_gen_ens_prod_single_field(metplus_config, config_overrides,
@@ -361,12 +374,7 @@ def test_gen_ens_prod_single_field(metplus_config, config_overrides,
     for key, value in config_overrides.items():
         config.set('config', key, value)
 
-    test_data_dir = os.path.join(config.getdir('METPLUS_BASE'),
-                                 'internal_tests',
-                                 'data')
-    input_dir = os.path.join(test_data_dir, 'ens')
-    config.set('config', 'GEN_ENS_PROD_INPUT_DIR', input_dir)
-    config.set('config', 'GEN_ENS_PROD_CTRL_INPUT_DIR', input_dir)
+    input_dir = handle_input_dir(config)
 
     wrapper = GenEnsProdWrapper(config)
     assert wrapper.isOK
@@ -414,17 +422,59 @@ def test_gen_ens_prod_single_field(metplus_config, config_overrides,
             else:
                 assert(env_var_values.get(env_var_key, '') == actual_value)
 
-def test_get_config_file(metplus_config):
-    fake_config_name = '/my/config/file'
-
+@pytest.mark.parametrize(
+    'use_default_config_file', [
+        True,
+        False,
+    ]
+)
+def test_get_config_file(metplus_config, use_default_config_file):
     config = metplus_config()
-    default_config_file = os.path.join(config.getdir('PARM_BASE'),
-                                       'met_config',
-                                       'GenEnsProdConfig_wrapped')
+
+    if use_default_config_file:
+        config_file = os.path.join(config.getdir('PARM_BASE'),
+                                   'met_config',
+                                   'GenEnsProdConfig_wrapped')
+    else:
+        config_file = '/my/config/file'
+        config.set('config', 'GEN_ENS_PROD_CONFIG_FILE', config_file)
 
     wrapper = GenEnsProdWrapper(config)
-    assert wrapper.c_dict['CONFIG_FILE'] == default_config_file
+    assert wrapper.c_dict['CONFIG_FILE'] == config_file
+
+@pytest.mark.parametrize(
+    'config_overrides, expected_num_files', [
+        ({}, 10),
+        ({'GEN_ENS_PROD_ENS_MEMBER_IDS': '1'}, 5),
+    ]
+)
+def test_gen_ens_prod_fill_missing(metplus_config, config_overrides,
+                                   expected_num_files):
+    config = metplus_config()
+
+    set_minimum_config_settings(config)
+    handle_input_dir(config)
+
+    # change some config values for this test
+    config.set('config', 'INIT_END', run_times[0])
+    config.set('config', 'GEN_ENS_PROD_N_MEMBERS', 10)
+
+    # set config variable overrides
+    for key, value in config_overrides.items():
+        config.set('config', key, value)
 
-    config.set('config', 'GEN_ENS_PROD_CONFIG_FILE', fake_config_name)
     wrapper = GenEnsProdWrapper(config)
-    assert wrapper.c_dict['CONFIG_FILE'] == fake_config_name
+
+    file_list_file = os.path.join(wrapper.config.getdir('STAGING_DIR'),
+                                  'file_lists',
+                                  '20091231120000_24_gen_ens_prod.txt')
+    if os.path.exists(file_list_file):
+        os.remove(file_list_file)
+
+    all_cmds = wrapper.run_all_times()
+    assert len(all_cmds) == 1
+
+    with open(file_list_file, 'r') as file_handle:
+        actual_num_files = len(file_handle.read().splitlines()) - 1
+
+    assert actual_num_files == expected_num_files

metplus/wrappers/command_builder.py

@@ -822,13 +822,16 @@ def find_file_in_window(self, level, data_type, time_info, mandatory=True,
 
         return out
 
-    def find_input_files_ensemble(self, time_info):
+    def find_input_files_ensemble(self, time_info, fill_missing=True):
         """! Get a list of all input files and optional control file.
         Warn and remove control file if found in ensemble list. Ensure that
         if defined, the number of ensemble members (N_MEMBERS) corresponds to
         the file list that was found.
 
             @param time_info dictionary containing timing information
+            @param fill_missing If True, fill list of files with MISSING so
+            that number of files matches number of expected members. Defaults
+            to True.
             @returns True on success
         """
 
@@ -872,7 +875,9 @@ def find_input_files_ensemble(self, time_info):
                 input_files.remove(ctrl_file)
 
         # compare number of files found to expected number of members
-        if not self._check_expected_ensembles(input_files):
+        if not fill_missing:
+            self.logger.debug('Skipping logic to fill file list with MISSING')
+        elif not self._check_expected_ensembles(input_files):
             return False
 
         # write file that contains list of ensemble files

metplus/wrappers/ensemble_stat_wrapper.py

@@ -416,7 +416,10 @@ def run_at_time_all_fields(self, time_info):
                 @param time_info dictionary containing timing information
         """
         # get ensemble model files
-        if not self.find_input_files_ensemble(time_info):
+        # do not fill file list with missing if ens_member_ids is used
+        fill_missing = not self.env_var_dict.get('METPLUS_ENS_MEMBER_IDS')
+        if not self.find_input_files_ensemble(time_info,
+                                              fill_missing=fill_missing):
             return
 
         # parse var list for ENS fields

metplus/wrappers/gen_ens_prod_wrapper.py

@@ -32,6 +32,7 @@ class GenEnsProdWrapper(LoopTimesWrapper):
         'METPLUS_ENSEMBLE_FLAG_DICT',
         'METPLUS_ENS_MEMBER_IDS',
         'METPLUS_CONTROL_ID',
+        'METPLUS_NORMALIZE',
     ]
 
     ENSEMBLE_FLAGS = [
@@ -167,6 +168,11 @@ def create_c_dict(self):
                                              'GEN_ENS_PROD_ENS_VLD_THRESH'],
                             )
 
+        self.add_met_config(name='normalize',
+                            data_type='string',
+                            extra_args={'remove_quotes': True},
+                            )
+
         # parse var list for ENS fields
         c_dict['ENS_VAR_LIST_TEMP'] = parse_var_list(
             self.config,
@@ -178,7 +184,8 @@ def create_c_dict(self):
                             data_type='string',
                             env_var_name='ENS_FILE_TYPE',
                             metplus_configs=['GEN_ENS_PROD_ENS_FILE_TYPE',
-                                             'GEN_ENS_PROD_FILE_TYPE'],
+                                             'GEN_ENS_PROD_FILE_TYPE',
+                                             'ENS_FILE_TYPE'],
                             extra_args={'remove_quotes': True,
                                         'uppercase': True})
 
@@ -227,7 +234,10 @@ def run_at_time_once(self, time_info):
         if not self.find_field_info(time_info):
             return False
 
-        if not self.find_input_files_ensemble(time_info):
+        # do not fill file list with missing if ens_member_ids is used
+        fill_missing = not self.env_var_dict.get('METPLUS_ENS_MEMBER_IDS')
+        if not self.find_input_files_ensemble(time_info,
+                                              fill_missing=fill_missing):
             return False
 
         if not self.find_and_check_output_file(time_info):

parm/met_config/GenEnsProdConfig_wrapped

@@ -39,6 +39,9 @@ ${METPLUS_CENSOR_THRESH}
 //censor_val    =
 ${METPLUS_CENSOR_VAL}
 
+//normalize =
+${METPLUS_NORMALIZE}
+
 //cat_thresh    =
 ${METPLUS_CAT_THRESH}