feature 1586 added wrapper-specific version of probabilistic config variables

docs/Users_Guide/glossary.rst

@@ -1250,14 +1250,92 @@ METplus Configuration Glossary
      .. warning:: **DEPRECATED:**
 
    FCST_IS_PROB
-     Specify whether the forecast data are probabilistic or not. Acceptable values: true/false
+     Boolean to specify whether the forecast data are probabilistic or not.
 
-     | *Used by:*  EnsembleStat, GridStat, MODE, MTD, PointStat
+     | *Used by:*  EnsembleStat, GridStat, MODE, MTD, PointStat, SeriesAnalysis
 
    FCST_PROB_IN_GRIB_PDS
-     Specify whether the probabilistic forecast data is stored in the GRIB Product Definition Section or not.Acceptable values: true/false. Only used when FCST_IS_PROB is True. This does not need to be set if the FCST_<APP_NAME>_INPUT_DATATYPE is set to NetCDF.
+     Boolean to specify whether the probabilistic forecast data is stored in
+     the GRIB Product Definition Section or not.
+     Only used when :term:`FCST_IS_PROB` is True.
 
-     | *Used by:*  EnsembleStat, GridStat, MODE, MTD, PointStat
+     | *Used by:*  EnsembleStat, GridStat, MODE, MTD, PointStat, SeriesAnalysis
+
+   OBS_IS_PROB
+     Specify whether the observation data are probabilistic or not.
+     Used when setting OBS_* variables to process probabilistic forecast data.
+     See :term:`FCST_IS_PROB`
+
+     | *Used by:*  EnsembleStat, GridStat, MODE, MTD, PointStat, SeriesAnalysis
+
+   OBS_PROB_IN_GRIB_PDS
+     Boolean to specify whether the probabilistic forecast data is stored in
+     the GRIB Product Definition Section or not.
+     Used when setting OBS_* variables to process probabilistic forecast data.
+     Only used when :term:`OBS_IS_PROB` is True.
+     See :term:`FCST_PROB_IN_GRIB_PDS` and :term:`FCST_IS_PROB`.
+
+     | *Used by:*  EnsembleStat, GridStat, MODE, MTD, PointStat, SeriesAnalysis
+
+   FCST_GRID_STAT_IS_PROB
+      Wrapper-specific version of :term:`FCST_IS_PROB`.
+
+     | *Used by:*  GridStat
+
+   FCST_GRID_STAT_PROB_IN_GRIB_PDS
+      Wrapper-specific version of :term:`FCST_PROB_IN_GRIB_PDS`.
+
+     | *Used by:*  GridStat
+
+   FCST_ENSEMBLE_STAT_IS_PROB
+      Wrapper-specific version of :term:`FCST_IS_PROB`.
+
+     | *Used by:*  EnsembleStat
+
+   FCST_ENSEMBLE_STAT_PROB_IN_GRIB_PDS
+      Wrapper-specific version of :term:`FCST_PROB_IN_GRIB_PDS`.
+
+     | *Used by:*  EnsembleStat
+
+   FCST_MODE_IS_PROB
+      Wrapper-specific version of :term:`FCST_IS_PROB`.
+
+     | *Used by:*  MODE
+
+   FCST_MODE_PROB_IN_GRIB_PDS
+      Wrapper-specific version of :term:`FCST_PROB_IN_GRIB_PDS`.
+
+     | *Used by:*  MODE
+
+   FCST_MTD_IS_PROB
+      Wrapper-specific version of :term:`FCST_IS_PROB`.
+
+     | *Used by:*  MTD
+
+   FCST_MTD_PROB_IN_GRIB_PDS
+      Wrapper-specific version of :term:`FCST_PROB_IN_GRIB_PDS`.
+
+     | *Used by:*  MTD
+
+   FCST_POINT_STAT_IS_PROB
+      Wrapper-specific version of :term:`FCST_IS_PROB`.
+
+     | *Used by:*  PointStat
+
+   FCST_POINT_STAT_PROB_IN_GRIB_PDS
+      Wrapper-specific version of :term:`FCST_PROB_IN_GRIB_PDS`.
+
+     | *Used by:*  PointStat
+
+   FCST_SERIES_ANALYSIS_IS_PROB
+      Wrapper-specific version of :term:`FCST_IS_PROB`.
+
+     | *Used by:*  SeriesAnalysis
+
+   FCST_SERIES_ANALYSIS_PROB_IN_GRIB_PDS
+      Wrapper-specific version of :term:`FCST_PROB_IN_GRIB_PDS`.
+
+     | *Used by:*  SeriesAnalysis
 
    FCST_LEAD
      .. warning:: **DEPRECATED:** Please use :term:`FCST_LEAD_LIST` instead.
@@ -2796,16 +2874,6 @@ METplus Configuration Glossary
    OBS_IS_DAILY_FILE
      .. warning:: **DEPRECATED:**
 
-   OBS_IS_PROB
-     Used when setting OBS_* variables to process forecast data for comparisons with mtd. Specify whether the observation data are probabilistic or not. See :term:`FCST_IS_PROB` .Acceptable values: true/false
-
-     | *Used by:*  EnsembleStat, GridStat, MODE, MTD, PointStat
-
-   OBS_PROB_IN_GRIB_PDS
-     Specify whether the probabilistic observation data is stored in the GRIB Product Definition Section or not.Acceptable values: true/false. Only used when :term:`OBS_IS_PROB` is True. This does not need to be set if the OBS_<APP_NAME>_INPUT_DATATYPE is set to NetCDF.
-
-     | *Used by:*  EnsembleStat, GridStat, MODE, MTD, PointStat
-
    OBS_LEVEL
      .. warning:: **DEPRECATED:** Please use :term:`OBS_PCP_COMBINE_INPUT_LEVEL` instead.
 

docs/Users_Guide/wrappers.rst

@@ -296,6 +296,8 @@ METplus Configuration
 | :term:`ENSEMBLE_STAT_PROB_CAT_THRESH`
 | :term:`ENSEMBLE_STAT_PROB_PCT_THRESH`
 | :term:`ENSEMBLE_STAT_ECLV_POINTS`
+| :term:`FCST_ENSEMBLE_STAT_IS_PROB`
+| :term:`FCST_ENSEMBLE_STAT_PROB_IN_GRIB_PDS`
 | :term:`ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE` (optional)
 | :term:`ENS_VAR<n>_NAME` (optional)
 | :term:`ENS_VAR<n>_LEVELS` (optional)
@@ -2948,6 +2950,8 @@ METplus Configuration
 | :term:`GRID_STAT_FOURIER_WAVE_1D_END`
 | :term:`GRID_STAT_CENSOR_THRESH`
 | :term:`GRID_STAT_CENSOR_VAL`
+| :term:`FCST_GRID_STAT_IS_PROB`
+| :term:`FCST_GRID_STAT_PROB_IN_GRIB_PDS`
 | :term:`GRID_STAT_MASK_GRID` (optional)
 | :term:`GRID_STAT_MASK_POLY` (optional)
 | :term:`GRID_STAT_MET_CONFIG_OVERRIDES`
@@ -4114,6 +4118,8 @@ METplus Configuration
 | :term:`MODE_INTEREST_FUNCTION_CONVEX_HULL_DIST`
 | :term:`MODE_PS_PLOT_FLAG`
 | :term:`MODE_CT_STATS_FLAG`
+| :term:`FCST_MODE_IS_PROB`
+| :term:`FCST_MODE_PROB_IN_GRIB_PDS`
 | :term:`FCST_MODE_VAR<n>_NAME` (optional)
 | :term:`FCST_MODE_VAR<n>_LEVELS` (optional)
 | :term:`FCST_MODE_VAR<n>_THRESH` (optional)
@@ -4716,6 +4722,8 @@ METplus Configuration
 | :term:`MTD_REGRID_VLD_THRESH`
 | :term:`MTD_REGRID_SHAPE`
 | :term:`MTD_MET_CONFIG_OVERRIDES`
+| :term:`FCST_MTD_IS_PROB`
+| :term:`FCST_MTD_PROB_IN_GRIB_PDS`
 | :term:`FCST_MTD_VAR<n>_NAME` (optional)
 | :term:`FCST_MTD_VAR<n>_LEVELS` (optional)
 | :term:`FCST_MTD_VAR<n>_THRESH` (optional)
@@ -5209,8 +5217,6 @@ METplus Configuration
 | :term:`OBS_PCP_COMBINE_INPUT_TEMPLATE`
 | :term:`OBS_PCP_COMBINE_OUTPUT_TEMPLATE`
 | :term:`LOG_PCP_COMBINE_VERBOSITY`
-| :term:`FCST_IS_PROB`
-| :term:`OBS_IS_PROB`
 | :term:`FCST_PCP_COMBINE_INPUT_ACCUMS`
 | :term:`FCST_PCP_COMBINE_INPUT_NAMES`
 | :term:`FCST_PCP_COMBINE_INPUT_LEVELS`
@@ -5457,6 +5463,8 @@ Configuration
 | :term:`POINT_STAT_HIRA_SHAPE`
 | :term:`POINT_STAT_HIRA_PROB_CAT_THRESH`
 | :term:`POINT_STAT_MESSAGE_TYPE_GROUP_MAP`
+| :term:`FCST_POINT_STAT_IS_PROB`
+| :term:`FCST_POINT_STAT_PROB_IN_GRIB_PDS`
 | :term:`FCST_POINT_STAT_WINDOW_BEGIN` (optional)
 | :term:`FCST_POINT_STAT_WINDOW_END` (optional)
 | :term:`OBS_POINT_STAT_WINDOW_BEGIN` (optional)
@@ -6104,6 +6112,8 @@ METplus Configuration
 | :term:`SERIES_ANALYSIS_OUTPUT_STATS_PRC`
 | :term:`FCST_SERIES_ANALYSIS_CAT_THRESH`
 | :term:`OBS_SERIES_ANALYSIS_CAT_THRESH`
+| :term:`FCST_SERIES_ANALYSIS_IS_PROB`
+| :term:`FCST_SERIES_ANALYSIS_PROB_IN_GRIB_PDS`
 |
 
 .. warning:: **DEPRECATED:**

internal_tests/pytests/compare_gridded/test_compare_gridded.py

@@ -274,12 +274,12 @@ def test_handle_window_once(metplus_config, win, app_win, file_win, app_file_win
                   'FCST_FILE_WINDOW_BEGIN',
                   'FILE_WINDOW_BEGIN',
                  ]
-    fcst_file_window_begin = cgw.handle_window_once(input_list, 0)
+    fcst_file_window_begin = cgw._handle_window_once(input_list, 0)
 
     input_list = ['OBS_APP_NAME_WINDOW_BEGIN',
                   'OBS_WINDOW_BEGIN',
                  ]
-    obs_window_begin = cgw.handle_window_once(input_list, -5400)
+    obs_window_begin = cgw._handle_window_once(input_list, -5400)
 
     assert(obs_window_begin == win_value and
            fcst_file_window_begin == file_win_value)

internal_tests/pytests/grid_stat/test_grid_stat_wrapper.py

@@ -51,6 +51,52 @@ def set_minimum_config_settings(config):
     config.set('config', 'OBS_VAR1_NAME', obs_name)
     config.set('config', 'OBS_VAR1_LEVELS', obs_level)
 
+@pytest.mark.parametrize(
+    'config_overrides, expected_values', [
+        # 0 generic FCST is prob
+        ({'FCST_IS_PROB': True},
+         {'FCST_IS_PROB': True, 'OBS_IS_PROB': False}),
+        # 1 generic OBS is prob
+        ({'OBS_IS_PROB': True},
+         {'FCST_IS_PROB': False, 'OBS_IS_PROB': True}),
+        # 2 generic FCST and OBS is prob
+        ({'FCST_IS_PROB': True, 'OBS_IS_PROB': True},
+         {'FCST_IS_PROB': True, 'OBS_IS_PROB': True}),
+        # 3 generic FCST true, wrapper FCST false
+        ({'FCST_IS_PROB': True, 'FCST_GRID_STAT_IS_PROB': False},
+         {'FCST_IS_PROB': False, 'OBS_IS_PROB': False}),
+        # 4 generic OBS true, wrapper OBS false
+        ({'OBS_IS_PROB': True, 'OBS_GRID_STAT_IS_PROB': False},
+         {'FCST_IS_PROB': False, 'OBS_IS_PROB': False}),
+        # 5 generic FCST unset, wrapper FCST true
+        ({'FCST_GRID_STAT_IS_PROB': True},
+         {'FCST_IS_PROB': True, 'OBS_IS_PROB': False}),
+        # 6 generic OBS unset, wrapper OBS true
+        ({'OBS_GRID_STAT_IS_PROB': True},
+         {'FCST_IS_PROB': False, 'OBS_IS_PROB': True}),
+        # 7 generic FCST false, wrapper FCST true
+        ({'FCST_IS_PROB': False, 'FCST_GRID_STAT_IS_PROB': True},
+         {'FCST_IS_PROB': True, 'OBS_IS_PROB': False}),
+        # 8 generic FCST true, wrapper FCST false
+        ({'FCST_IS_PROB': True, 'FCST_GRID_STAT_IS_PROB': False},
+         {'FCST_IS_PROB': False, 'OBS_IS_PROB': False}),
+    ]
+)
+def test_grid_stat_is_prob(metplus_config, config_overrides, expected_values):
+
+    config = metplus_config()
+
+    set_minimum_config_settings(config)
+
+    # set config variable overrides
+    for key, value in config_overrides.items():
+        config.set('config', key, value)
+
+    wrapper = GridStatWrapper(config)
+    assert wrapper.isOK
+    for key, expected_value in expected_values.items():
+        assert expected_value == wrapper.c_dict[key]
+
 @pytest.mark.parametrize(
     'config_overrides, env_var_values', [
         # 0 no climo settings

metplus/util/__init__.py

@@ -8,3 +8,4 @@
 from .string_template_substitution import *
 from .met_config import *
 from .time_looping import *
+from .field_util import *

metplus/util/field_util.py

@@ -0,0 +1,147 @@
+"""
+Program Name: field_util.py
+Contact(s): George McCabe <[email protected]>
+Description: METplus utility to handle MET config dictionaries with field info
+"""
+
+from . import get_threshold_via_regex, is_python_script, remove_quotes
+
+def field_read_prob_info(config, c_dict, data_types, app_name):
+    """! Read probabilistic variables for each field data type from the config
+         object and sets values in the c_dict as appropriate.
+
+        @param config METplusConfig object to read
+        @param c_dict dictionary to set values
+        @param data_types list of field types to check, i.e. FCST, OBS
+        @param app_name name of tool used to read wrapper-specific configs
+    """
+    for data_type in data_types:
+        # check both wrapper-specific variable and generic variable
+        config_names = [
+            f'{data_type}_{app_name.upper()}_IS_PROB',
+            f'{data_type}_IS_PROB',
+        ]
+        name = config.get_mp_config_name(config_names)
+
+        is_prob = config.getbool('config', name) if name else False
+        c_dict[f'{data_type}_IS_PROB'] = is_prob
+
+        # if field type is probabilistic, check if prob info is in GRIB PDS
+        if not is_prob:
+            continue
+
+        config_names = [
+            f'{data_type}_{app_name.upper()}_PROB_IN_GRIB_PDS',
+            f'{data_type}_PROB_IN_GRIB_PDS',
+        ]
+        name = config.get_mp_config_name(config_names)
+        prob_in_pds = config.getbool('config', name) if name else False
+        c_dict[f'{data_type}_PROB_IN_GRIB_PDS'] = prob_in_pds
+
+
+def get_field_info(c_dict, data_type='', v_name='', v_level='', v_thresh=None,
+                   v_extra='', add_curly_braces=True):
+    """! Format field information into format expected by MET config file
+
+         @param c_dict config dictionary to read values
+         @param v_level level of data to extract
+         @param v_thresh threshold value to use in comparison
+         @param v_name name of field to process
+         @param v_extra additional field information to add if available
+         @param data_type type of data to find i.e. FCST or OBS
+         @param add_curly_braces if True, add curly braces around each
+          field info string. If False, add single quotes around each
+          field info string (defaults to True)
+         @rtype string
+         @return Returns a list of formatted field information or a string
+          containing an error message if something went wrong
+    """
+    # if thresholds are set
+    if v_thresh:
+        # if neither fcst or obs are probabilistic,
+        # pass in all thresholds as a comma-separated list for 1 field info
+        if (not c_dict.get('FCST_IS_PROB', False) and
+                not c_dict.get('OBS_IS_PROB', False)):
+            thresholds = [','.join(v_thresh)]
+        else:
+            thresholds = v_thresh
+    # if no thresholds are specified, fail if prob field is in grib PDS
+    elif (c_dict.get(f'{data_type}_IS_PROB', False) and
+          c_dict.get(f'{data_type}_PROB_IN_GRIB_PDS', False) and
+          not is_python_script(v_name)):
+        return 'No threshold was specified for probabilistic GRIB data'
+    else:
+        thresholds = [None]
+
+    # list to hold field information
+    fields = []
+
+    for thresh in thresholds:
+        if (c_dict.get(f'{data_type}_PROB_IN_GRIB_PDS', False) and
+                not is_python_script(v_name)):
+            field = _handle_grib_pds_field_info(v_name, v_level, thresh)
+        else:
+            # add field name
+            field = f'name="{v_name}";'
+
+            if v_level:
+                field += f' level="{remove_quotes(v_level)}";'
+
+            if c_dict.get(f'{data_type}_IS_PROB', False):
+                field += " prob=TRUE;"
+
+        # handle cat_thresh
+        if c_dict.get(f'{data_type}_IS_PROB', False):
+            # add probabilistic cat thresh if different from default ==0.1
+            cat_thresh = c_dict.get(f'{data_type}_PROB_THRESH')
+        else:
+            cat_thresh = thresh
+
+        if cat_thresh:
+            field += f" cat_thresh=[ {cat_thresh} ];"
+
+        # handle extra options if set
+        if v_extra:
+            extra = v_extra.strip()
+            # if trailing semi-colon is not found, add it
+            if not extra.endswith(';'):
+                extra = f"{extra};"
+            field += f' {extra}'
+
+        # add curly braces around field info if requested
+        # otherwise add single quotes around field info
+        field = f'{{ {field} }}' if add_curly_braces else f"'{field}'"
+
+        # add field info string to list of fields
+        fields.append(field)
+
+    # return list of strings in field dictionary format
+    return fields
+
+
+def _handle_grib_pds_field_info(v_name, v_level, thresh):
+    """! Format field string to read probabilistic data from the PDS of a GRIB
+     file. Thresholds are formatted using thresh_lo and thresh_hi syntax.
+
+        @param v_name name of field to read
+        @param v_level level of field to read
+        @param thresh threshold value to format if set
+        @returns formatted field string
+    """
+
+    field = f'name="PROB"; level="{v_level}"; prob={{ name="{v_name}";'
+
+    if thresh:
+        thresh_tuple_list = get_threshold_via_regex(thresh)
+        for comparison, number in thresh_tuple_list:
+            # skip adding thresh_lo or thresh_hi if comparison is NA
+            if comparison == 'NA':
+                continue
+
+            if comparison in ["gt", "ge", ">", ">=", "==", "eq"]:
+                field = f"{field} thresh_lo={number};"
+            if comparison in ["lt", "le", "<", "<=", "==", "eq"]:
+                field = f"{field} thresh_hi={number};"
+
+    # add closing curly brace for prob=
+    return f'{field} }}'