Feature #2433 Ugrid config variables in GridStat/PointStat

docs/Users_Guide/glossary.rst

@@ -11522,6 +11522,48 @@ METplus Configuration Glossary
 
      | *Used by:* WaveletStat
 
+   GRID_STAT_UGRID_DATASET
+     Specify the value for 'ugrid_dataset' in the MET configuration file for GridStat.
+
+     | *Used by:* GridStat
+
+   GRID_STAT_UGRID_MAX_DISTANCE_KM
+     Specify the value for 'ugrid_max_distance_km' in the MET configuration file for GridStat.
+
+     | *Used by:* GridStat
+
+   GRID_STAT_UGRID_COORDINATES_FILE
+     Specify the value for 'ugrid_coordinates_file' in the MET configuration file for GridStat.
+
+     | *Used by:* GridStat
+
+   GRID_STAT_UGRID_CONFIG_FILE
+     Specify the path to the optional ugrid configuration file for GridStat.
+     This is added to the command using the -config argument.
+
+     | *Used by:* GridStat
+
+   POINT_STAT_UGRID_DATASET
+     Specify the value for 'ugrid_dataset' in the MET configuration file for PointStat.
+
+     | *Used by:* PointStat
+
+   POINT_STAT_UGRID_MAX_DISTANCE_KM
+     Specify the value for 'ugrid_max_distance_km' in the MET configuration file for PointStat.
+
+     | *Used by:* PointStat
+
+   POINT_STAT_UGRID_COORDINATES_FILE
+     Specify the value for 'ugrid_coordinates_file' in the MET configuration file for PointStat.
+
+     | *Used by:* PointStat
+
+   POINT_STAT_UGRID_CONFIG_FILE
+     Specify the path to the optional ugrid configuration file for PointStat.
+     This is added to the command using the -config argument.
+
+     | *Used by:* PointStat
+
    POINT_STAT_LAND_MASK_FLAG
      Specify the value for 'land_mask.flag' in the MET configuration file for PointStat.
 

docs/Users_Guide/wrappers.rst

@@ -3158,6 +3158,10 @@ METplus Configuration
 | :term:`GRID_STAT_INC_VALID_TIMES`
 | :term:`GRID_STAT_SKIP_INIT_TIMES`
 | :term:`GRID_STAT_INC_INIT_TIMES`
+| :term:`GRID_STAT_UGRID_DATASET`
+| :term:`GRID_STAT_UGRID_MAX_DISTANCE_KM`
+| :term:`GRID_STAT_UGRID_COORDINATES_FILE`
+| :term:`GRID_STAT_UGRID_CONFIG_FILE`
 |
 
 .. warning:: **DEPRECATED**
@@ -3571,6 +3575,18 @@ ${METPLUS_DISTANCE_MAP_DICT}
    * - :term:`GRID_STAT_DISTANCE_MAP_BETA_VALUE_N`
      - distance_map.beta_value(n)
 
+${METPLUS_SEEPS_P1_THRESH}
+""""""""""""""""""""""""""
+
+.. list-table::
+   :widths: 5 5
+   :header-rows: 1
+
+   * - METplus Config(s)
+     - MET Config File
+   * - :term:`GRID_STAT_SEEPS_P1_THRESH`
+     - seeps_p1_thresh
+
 ${METPLUS_OUTPUT_FLAG_DICT}
 """""""""""""""""""""""""""
 
@@ -3659,17 +3675,41 @@ ${METPLUS_NC_PAIRS_FLAG_DICT}
    * - :term:`GRID_STAT_NC_PAIRS_FLAG_SEEPS`
      - nc_pairs_flag.seeps
 
-${METPLUS_SEEPS_P1_THRESH}
-""""""""""""""""""""""""""
+${METPLUS_UGRID_DATASET}
+^^^^^^^^^^^^^^^^^^^^^^^^
 
 .. list-table::
    :widths: 5 5
    :header-rows: 1
 
    * - METplus Config(s)
      - MET Config File
-   * - :term:`GRID_STAT_SEEPS_P1_THRESH`
-     - seeps_p1_thresh
+   * - :term:`GRID_STAT_UGRID_DATASET`
+     - ugrid_dataset
+
+${METPLUS_UGRID_MAX_DISTANCE_KM}
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. list-table::
+   :widths: 5 5
+   :header-rows: 1
+
+   * - METplus Config(s)
+     - MET Config File
+   * - :term:`GRID_STAT_UGRID_MAX_DISTANCE_KM`
+     - ugrid_max_distance_km
+
+${METPLUS_UGRID_COORDINATES_FILE}
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. list-table::
+   :widths: 5 5
+   :header-rows: 1
+
+   * - METplus Config(s)
+     - MET Config File
+   * - :term:`GRID_STAT_UGRID_COORDINATES_FILE`
+     - ugrid_coordinates_file
 
 ${METPLUS_GRID_WEIGHT_FLAG}
 """""""""""""""""""""""""""
@@ -6393,6 +6433,10 @@ Configuration
 | :term:`POINT_STAT_INC_VALID_TIMES`
 | :term:`POINT_STAT_SKIP_INIT_TIMES`
 | :term:`POINT_STAT_INC_INIT_TIMES`
+| :term:`POINT_STAT_UGRID_DATASET`
+| :term:`POINT_STAT_UGRID_MAX_DISTANCE_KM`
+| :term:`POINT_STAT_UGRID_COORDINATES_FILE`
+| :term:`POINT_STAT_UGRID_CONFIG_FILE`
 |
 
 .. warning:: **DEPRECATED:**
@@ -6855,6 +6899,18 @@ ${METPLUS_HIRA_DICT}
    * - :term:`POINT_STAT_HIRA_PROB_CAT_THRESH`
      - hira.prob_cat_thresh
 
+${METPLUS_SEEPS_P1_THRESH}
+""""""""""""""""""""""""""
+
+.. list-table::
+   :widths: 5 5
+   :header-rows: 1
+
+   * - METplus Config(s)
+     - MET Config File
+   * - :term:`POINT_STAT_SEEPS_P1_THRESH`
+     - seeps_p1_thresh
+
 ${METPLUS_OUTPUT_FLAG_DICT}
 """""""""""""""""""""""""""
 
@@ -6909,18 +6965,42 @@ ${METPLUS_OUTPUT_FLAG_DICT}
    * - :term:`POINT_STAT_OUTPUT_FLAG_SEEPS_MPR`
      - output_flag.seeps_mpr
 
-${METPLUS_SEEPS_P1_THRESH}
-""""""""""""""""""""""""""
+${METPLUS_UGRID_DATASET}
+^^^^^^^^^^^^^^^^^^^^^^^^
 
 .. list-table::
    :widths: 5 5
    :header-rows: 1
 
    * - METplus Config(s)
      - MET Config File
-   * - :term:`POINT_STAT_SEEPS_P1_THRESH`
-     - seeps_p1_thresh
-
+   * - :term:`POINT_STAT_UGRID_DATASET`
+     - ugrid_dataset
+
+${METPLUS_UGRID_MAX_DISTANCE_KM}
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. list-table::
+   :widths: 5 5
+   :header-rows: 1
+
+   * - METplus Config(s)
+     - MET Config File
+   * - :term:`POINT_STAT_UGRID_MAX_DISTANCE_KM`
+     - ugrid_max_distance_km
+
+${METPLUS_UGRID_COORDINATES_FILE}
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. list-table::
+   :widths: 5 5
+   :header-rows: 1
+
+   * - METplus Config(s)
+     - MET Config File
+   * - :term:`POINT_STAT_UGRID_COORDINATES_FILE`
+     - ugrid_coordinates_file
+
 ${METPLUS_OUTPUT_PREFIX}
 """"""""""""""""""""""""
 

docs/Verification_Datasets/datasets/gdas_prepbufr.rst

@@ -11,6 +11,8 @@ Description
 
   Note: GDAS prepbufr data is also available via other sources (e.g., NOMADS, NOAA HPSS, and NCAR/Research Data Archive); however, NOAA/NCEI does not require an account and provides a long archive period.
 
+  .. note:: Surface winds are by default encoded with a quality mark of 9, to avoid being used in data assimilation. To use surface winds from GDAS prepbufr, be sure to include quality mark values of 9 when running PB2NC, which can be controlled via the PB2NC configuration file.
+
 Recommended use
   GDAS prepbufr data are commonly used for verification of standard meteorological variables, such as temperatures, dew point temperature, wind speed, visibility, and precipitation, and precipitation type.
 

internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py

@@ -21,6 +21,8 @@
 time_fmt = '%Y%m%d%H'
 run_times = ['2005080700', '2005080712']
 
+ugrid_config_file = '/some/path/UgridConfig_fake'
+
 
 def set_minimum_config_settings(config):
     # set config variables to prevent command from running and bypass check
@@ -708,6 +710,13 @@ def test_grid_stat_is_prob(metplus_config, config_overrides, expected_values):
          {'METPLUS_SEEPS_P1_THRESH': 'seeps_p1_thresh = ge0.1&&le0.85;'}),
         ({'GRID_STAT_CAT_THRESH': 'NA', },
          {'METPLUS_CAT_THRESH': 'cat_thresh = [NA];'}),
+        ({'GRID_STAT_UGRID_DATASET': 'mpas', },
+         {'METPLUS_UGRID_DATASET': 'ugrid_dataset = "mpas";'}),
+        ({'GRID_STAT_UGRID_MAX_DISTANCE_KM': '30', },
+         {'METPLUS_UGRID_MAX_DISTANCE_KM': 'ugrid_max_distance_km = 30;'}),
+        ({'GRID_STAT_UGRID_COORDINATES_FILE': '/met/test/input/ugrid_data/mpas/static.40962_reduced.nc', },
+         {'METPLUS_UGRID_COORDINATES_FILE': 'ugrid_coordinates_file = "/met/test/input/ugrid_data/mpas/static.40962_reduced.nc";'}),
+        ({'GRID_STAT_UGRID_CONFIG_FILE': ugrid_config_file, }, {}),
 
     ]
 )
@@ -726,18 +735,26 @@ def test_grid_stat_single_field(metplus_config, config_overrides,
     wrapper = GridStatWrapper(config)
     assert wrapper.isOK
 
+    # add extra command line arguments
+    extra_args = [' '] * len(run_times)
+
+    if 'GRID_STAT_UGRID_CONFIG_FILE' in config_overrides:
+        for index in range(0, len(run_times)):
+            extra_args[index] += f'-ugrid_config {ugrid_config_file} '
+
+
     app_path = os.path.join(config.getdir('MET_BIN_DIR'), wrapper.app_name)
     verbosity = f"-v {wrapper.c_dict['VERBOSITY']}"
     config_file = wrapper.c_dict.get('CONFIG_FILE')
     out_dir = wrapper.c_dict.get('OUTPUT_DIR')
     expected_cmds = [(f"{app_path} {verbosity} "
                       f"{fcst_dir}/2005080700/fcst_file_F012 "
                       f"{obs_dir}/2005080712/obs_file "
-                      f"{config_file} -outdir {out_dir}/2005080712"),
+                      f"{config_file}{extra_args[0]}-outdir {out_dir}/2005080712"),
                      (f"{app_path} {verbosity} "
                       f"{fcst_dir}/2005080712/fcst_file_F012 "
                       f"{obs_dir}/2005080800/obs_file "
-                      f"{config_file} -outdir {out_dir}/2005080800"),
+                      f"{config_file}{extra_args[1]}-outdir {out_dir}/2005080800"),
                      ]
 
     all_cmds = wrapper.run_all_times()

internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py

@@ -25,6 +25,7 @@
     valid = valid.strftime(time_fmt)
     valids.append(valid)
 
+ugrid_config_file = '/some/path/UgridConfig_fake'
 
 def set_minimum_config_settings(config):
     # set config variables to prevent command from running and bypass check
@@ -602,6 +603,12 @@ def test_met_dictionary_in_var_options(metplus_config):
                  'sid = ["one", "two"];'
                  'llpnt = [{ name = "LAT30TO40"; lat_thresh = >=30&&<=40; lon_thresh = NA; }, { name = "BOX"; lat_thresh = >=20&&<=40; lon_thresh = >=-110&&<=-90; }];}'
          )}),
+        ({'POINT_STAT_UGRID_DATASET': 'mpas', },
+         {'METPLUS_UGRID_DATASET': 'ugrid_dataset = "mpas";'}),
+        ({'POINT_STAT_UGRID_MAX_DISTANCE_KM': '30', },
+         {'METPLUS_UGRID_MAX_DISTANCE_KM': 'ugrid_max_distance_km = 30;'}),
+        ({'POINT_STAT_UGRID_COORDINATES_FILE': '/met/test/input/ugrid_data/mpas/static.40962_reduced.nc', },
+         {'METPLUS_UGRID_COORDINATES_FILE': 'ugrid_coordinates_file = "/met/test/input/ugrid_data/mpas/static.40962_reduced.nc";'}),
         # land_mask dictionary
         ({'POINT_STAT_LAND_MASK_FLAG': 'false', },
          {'METPLUS_LAND_MASK_DICT': 'land_mask = {flag = FALSE;}'}),
@@ -659,6 +666,7 @@ def test_met_dictionary_in_var_options(metplus_config):
          {'METPLUS_OBS_SUMMARY': 'obs_summary = NONE;'}),
         ({'POINT_STAT_OBS_PERC_VALUE': '50', },
          {'METPLUS_OBS_PERC_VALUE': 'obs_perc_value = 50;'}),
+        ({'POINT_STAT_UGRID_CONFIG_FILE': ugrid_config_file, }, {}),
 
     ]
 )
@@ -733,7 +741,14 @@ def test_point_stat_all_fields(metplus_config, config_overrides,
     verbosity = f"-v {wrapper.c_dict['VERBOSITY']}"
     config_file = wrapper.c_dict.get('CONFIG_FILE')
     out_dir = wrapper.c_dict.get('OUTPUT_DIR')
+
+    # add extra command line arguments
     extra_args = [' '] * len(inits)
+
+    if 'POINT_STAT_UGRID_CONFIG_FILE' in config_overrides:
+        for index in range(0, len(inits)):
+            extra_args[index] += f'-ugrid_config {ugrid_config_file} '
+
     for beg_end in ('BEG', 'END'):
         if f'POINT_STAT_OBS_VALID_{beg_end}' in config_overrides:
             for index in range(0, len(inits)):
@@ -748,10 +763,10 @@ def test_point_stat_all_fields(metplus_config, config_overrides,
     expected_cmds = []
     for index in range(0, len(inits)):
         expected_cmds.append(
-            f"{app_path} {verbosity}{extra_args[index]}"
+            f"{app_path} {verbosity} "
             f"{fcst_dir}/{inits[index]}/fcst_file_F{lead_hour_str} "
             f"{obs_dir}/{valids[index]}/obs_file "
-            f"{config_file} -outdir {out_dir}/{valids[index]}"
+            f"{config_file}{extra_args[index]}-outdir {out_dir}/{valids[index]}"
         )
 
     all_cmds = wrapper.run_all_times()

metplus/wrappers/compare_gridded_wrapper.py

@@ -267,7 +267,11 @@ def set_command_line_arguments(self, time_info):
 
         @param time_info dictionary with time information
         """
-        return None
+        # add ugrid config file if requested
+        if self.c_dict.get('UGRID_CONFIG_FILE'):
+            ugrid_config = self.c_dict['UGRID_CONFIG_FILE']
+            ugrid_config = do_string_sub(ugrid_config, **time_info)
+            self.args.append(f'-ugrid_config {ugrid_config}')
 
     def get_command(self):
         """! Builds the command to run the MET application
@@ -279,8 +283,6 @@ def get_command(self):
             return None
 
         cmd = '{} -v {} '.format(self.app_path, self.c_dict['VERBOSITY'])
-        for arg in self.args:
-            cmd += arg + " "
 
         if len(self.infiles) == 0:
             self.log_error("No input filenames specified")
@@ -304,6 +306,9 @@ def get_command(self):
 
         cmd += self.param + ' '
 
+        for arg in self.args:
+            cmd += arg + " "
+
         if self.outdir == "":
             self.log_error("No output directory specified")
             return None

metplus/wrappers/grid_stat_wrapper.py

@@ -56,6 +56,9 @@ class GridStatWrapper(CompareGriddedWrapper):
         'METPLUS_CENSOR_VAL',
         'METPLUS_SEEPS_P1_THRESH',
         'METPLUS_CAT_THRESH',
+        'METPLUS_UGRID_DATASET',
+        'METPLUS_UGRID_MAX_DISTANCE_KM',
+        'METPLUS_UGRID_COORDINATES_FILE',
     ]
 
     # deprecated env vars that are no longer supported in the wrapped MET conf
@@ -131,6 +134,11 @@ def create_c_dict(self):
         # get the MET config file path or use default
         c_dict['CONFIG_FILE'] = self.get_config_file('GridStatConfig_wrapped')
 
+        # get optional ugrid config file if requested
+        c_dict['UGRID_CONFIG_FILE'] = (
+            self.config.getraw('config', 'GRID_STAT_UGRID_CONFIG_FILE')
+        )
+
         self.get_input_templates(c_dict, {
             'FCST': {'prefix': 'FCST_GRID_STAT', 'required': True},
             'OBS': {'prefix': 'OBS_GRID_STAT', 'required': True},
@@ -212,6 +220,10 @@ def create_c_dict(self):
         self.add_met_config(name='nc_pairs_var_name', data_type='string',
                             metplus_configs=['GRID_STAT_NC_PAIRS_VAR_NAME'])
 
+        self.add_met_config(name='ugrid_dataset', data_type='string')
+        self.add_met_config(name='ugrid_max_distance_km', data_type='int')
+        self.add_met_config(name='ugrid_coordinates_file', data_type='string')
+
         self.add_met_config(name='grid_weight_flag', data_type='string',
                             metplus_configs=['GRID_STAT_GRID_WEIGHT_FLAG'],
                             extra_args={'remove_quotes': True,