Feature 779 pb2nc configs

docs/Users_Guide/glossary.rst

@@ -6664,3 +6664,28 @@ METplus Configuration Glossary
      Specify the value for 'climo_stdev.time_interp_method' in the MET configuration file for PointStat.
 
      | *Used by:* PointStat
+
+   PB2NC_PB_REPORT_TYPE
+     Specify the value for 'pb_report_type' in the MET configuration file for PB2NC.
+
+     | *Used by:* PB2NC
+
+   PB2NC_LEVEL_RANGE_BEG
+     Specify the value for 'level_range.beg' in the MET configuration file for PB2NC.
+
+     | *Used by:* PB2NC
+
+   PB2NC_LEVEL_RANGE_END
+     Specify the value for 'level_range.end' in the MET configuration file for PB2NC.
+
+     | *Used by:* PB2NC
+
+   PB2NC_LEVEL_CATEGORY
+     Specify the value for 'level_category' in the MET configuration file for PB2NC.
+
+     | *Used by:* PB2NC
+
+   PB2NC_QUALITY_MARK_THRESH
+     Specify the value for 'quality_mark_thresh' in the MET configuration file for PB2NC.
+
+     | *Used by:* PB2NC

docs/Users_Guide/wrappers.rst

@@ -2656,7 +2656,11 @@ Configuration
 | :term:`PB2NC_VALID_END`
 | :term:`PB2NC_CUSTOM_LOOP_LIST`
 | :term:`PB2NC_MET_CONFIG_OVERRIDES`
-|
+| :term:`PB2NC_PB_REPORT_TYPE`
+| :term:`PB2NC_LEVEL_RANGE_BEG`
+| :term:`PB2NC_LEVEL_RANGE_END`
+| :term:`PB2NC_LEVEL_CATEGORY`
+| :term:`PB2NC_QUALITY_MARK_THRESH`
 
 .. warning:: **DEPRECATED:**
 
@@ -2792,6 +2796,53 @@ Below the file contents are descriptions of each environment variable referenced
    * - :term:`PB2NC_MET_CONFIG_OVERRIDES`
      - n/a
 
+**${METPLUS_PB_REPORT_TYPE}**
+
+.. list-table::
+   :widths: 5 5
+   :header-rows: 0
+
+   * - METplus Config(s)
+     - MET Config File
+   * - :term:`PB2NC_PB_REPORT_TYPE`
+     - pb_report_type
+
+**${METPLUS_LEVEL_RANGE_DICT}**
+
+.. list-table::
+   :widths: 5 5
+   :header-rows: 0
+
+   * - METplus Config(s)
+     - MET Config File
+   * - :term:`PB2NC_LEVEL_RANGE_BEG`
+     - level_range.beg
+   * - :term:`PB2NC_LEVEL_RANGE_END`
+     - level_range.end
+
+**${METPLUS_LEVEL_CATEGORY}**
+
+.. list-table::
+   :widths: 5 5
+   :header-rows: 0
+
+   * - METplus Config(s)
+     - MET Config File
+   * - :term:`PB2NC_LEVEL_CATEGORY`
+     - level_category
+
+**${METPLUS_QUALITY_MARK_THRESH}**
+
+.. list-table::
+   :widths: 5 5
+   :header-rows: 0
+
+   * - METplus Config(s)
+     - MET Config File
+   * - :term:`PB2NC_QUALITY_MARK_THRESH`
+     - quality_mark_thresh
+
+
 .. _pcp_combine_wrapper:
 
 PCPCombine

internal_tests/pytests/pb2nc/test_pb2nc_wrapper.py

@@ -169,3 +169,181 @@ def test_find_input_files(metplus_config, offsets, offset_to_find):
         assert result is None
     else:
         assert result['offset_hours'] == offset_to_find
+
+@pytest.mark.parametrize(
+    'config_overrides, env_var_values', [
+        ({'PB2NC_MESSAGE_TYPE': 'ADPUPA, ADPSFC'},
+         {'METPLUS_MESSAGE_TYPE': 'message_type = ["ADPUPA", "ADPSFC"];'}),
+
+        ({'PB2NC_STATION_ID': 'station1, station2'},
+         {'METPLUS_STATION_ID': 'station_id = ["station1", "station2"];'}),
+
+        ({'OBS_PB2NC_WINDOW_BEGIN': '-1800', },
+         {'METPLUS_OBS_WINDOW_DICT': 'obs_window = {beg = -1800;}'}),
+
+        ({'OBS_PB2NC_WINDOW_END': '1800', },
+         {'METPLUS_OBS_WINDOW_DICT': 'obs_window = {end = 1800;}'}),
+
+        ({
+             'OBS_PB2NC_WINDOW_BEGIN': '-1800',
+             'OBS_PB2NC_WINDOW_END': '1800',
+         },
+         {'METPLUS_OBS_WINDOW_DICT': 'obs_window = {beg = -1800;end = 1800;}'}),
+
+        ({'PB2NC_MASK_GRID': 'FULL', },
+         {'METPLUS_MASK_DICT': 'mask = {grid = "FULL";}'}),
+
+        ({'PB2NC_MASK_POLY': 'SAO.poly', },
+         {'METPLUS_MASK_DICT': 'mask = {poly = "SAO.poly";}'}),
+
+        ({'PB2NC_MASK_GRID': 'FULL',
+          'PB2NC_MASK_POLY': 'SAO.poly'},
+         {'METPLUS_MASK_DICT': 'mask = {grid = "FULL";poly = "SAO.poly";}'}),
+
+        ({'PB2NC_PB_REPORT_TYPE': '1, 2, 3, 4, 5', },
+         {'METPLUS_PB_REPORT_TYPE': 'pb_report_type = [1, 2, 3, 4, 5];'}),
+
+        ({'PB2NC_LEVEL_RANGE_BEGIN': '2', },
+         {'METPLUS_LEVEL_RANGE_DICT': 'level_range = {beg = 2;}'}),
+
+        ({'PB2NC_LEVEL_RANGE_END': '512', },
+         {'METPLUS_LEVEL_RANGE_DICT': 'level_range = {end = 512;}'}),
+
+        ({'PB2NC_LEVEL_RANGE_BEGIN': '2',
+          'PB2NC_LEVEL_RANGE_END': '512',
+         },
+         {'METPLUS_LEVEL_RANGE_DICT': 'level_range = {beg = 2;end = 512;}'}),
+
+        ({'PB2NC_LEVEL_CATEGORY': '0, 1, 4, 5, 6', },
+         {'METPLUS_LEVEL_CATEGORY': 'level_category = [0, 1, 4, 5, 6];'}),
+
+        ({'PB2NC_OBS_BUFR_VAR_LIST': 'QOB, TOB', },
+         {'METPLUS_OBS_BUFR_VAR': 'obs_bufr_var = ["QOB", "TOB"];'}),
+
+        ({'PB2NC_QUALITY_MARK_THRESH': '3', },
+         {'METPLUS_QUALITY_MARK_THRESH': 'quality_mark_thresh = 3;'}),
+
+        ({'PB2NC_TIME_SUMMARY_FLAG': 'True', },
+         {'METPLUS_TIME_SUMMARY_DICT': 'time_summary = {flag = TRUE;}'}),
+
+        ({'PB2NC_TIME_SUMMARY_RAW_DATA': 'True', },
+         {'METPLUS_TIME_SUMMARY_DICT': 'time_summary = {raw_data = TRUE;}'}),
+
+        ({'PB2NC_TIME_SUMMARY_BEG': '012345', },
+         {'METPLUS_TIME_SUMMARY_DICT': 'time_summary = {beg = "012345";}'}),
+
+        ({'PB2NC_TIME_SUMMARY_END': '234559', },
+         {'METPLUS_TIME_SUMMARY_DICT': 'time_summary = {end = "234559";}'}),
+
+        ({'PB2NC_TIME_SUMMARY_STEP': '200', },
+         {'METPLUS_TIME_SUMMARY_DICT': 'time_summary = {step = 200;}'}),
+
+        ({'PB2NC_TIME_SUMMARY_WIDTH': '500', },
+         {'METPLUS_TIME_SUMMARY_DICT': 'time_summary = {width = 500;}'}),
+
+        ({'PB2NC_TIME_SUMMARY_GRIB_CODES': '11, 204, 211', },
+         {'METPLUS_TIME_SUMMARY_DICT': 'time_summary = {grib_code = [11, 204, 211];}'}),
+
+        ({'PB2NC_TIME_SUMMARY_VAR_NAMES': 'PMO,TOB,TDO', },
+         {'METPLUS_TIME_SUMMARY_DICT': 'time_summary = {obs_var = ["PMO", "TOB", "TDO"];}'}),
+
+        ({'PB2NC_TIME_SUMMARY_TYPE': 'min, max, range', },
+         {'METPLUS_TIME_SUMMARY_DICT': 'time_summary = {type = ["min", "max", "range"];}'}),
+
+        ({'PB2NC_TIME_SUMMARY_VLD_FREQ': '1', },
+         {'METPLUS_TIME_SUMMARY_DICT': 'time_summary = {vld_freq = 1;}'}),
+
+        ({'PB2NC_TIME_SUMMARY_VLD_THRESH': '0.1', },
+         {
+             'METPLUS_TIME_SUMMARY_DICT': 'time_summary = {vld_thresh = 0.1;}'}),
+
+        ({
+             'PB2NC_TIME_SUMMARY_FLAG': 'TRUE',
+             'PB2NC_TIME_SUMMARY_RAW_DATA': 'TRUE',
+             'PB2NC_TIME_SUMMARY_BEG': '012345',
+             'PB2NC_TIME_SUMMARY_END': '234559',
+             'PB2NC_TIME_SUMMARY_STEP': '200',
+             'PB2NC_TIME_SUMMARY_WIDTH': '500',
+             'PB2NC_TIME_SUMMARY_GRIB_CODES': '11, 204, 211',
+             'PB2NC_TIME_SUMMARY_VAR_NAMES': 'PMO,TOB,TDO',
+             'PB2NC_TIME_SUMMARY_TYPE': 'min, max, range',
+             'PB2NC_TIME_SUMMARY_VLD_FREQ': '1',
+             'PB2NC_TIME_SUMMARY_VLD_THRESH': '0.1',
+         },
+         {
+             'METPLUS_TIME_SUMMARY_DICT': ('time_summary = {flag = TRUE;'
+                                           'raw_data = TRUE;beg = "012345";'
+                                           'end = "234559";step = 200;'
+                                           'width = 500;grib_code = [11, 204, 211];'
+                                           'obs_var = ["PMO", "TOB", "TDO"];'
+                                           'type = ["min", "max", "range"];'
+                                           'vld_freq = 1;'
+                                           'vld_thresh = 0.1;}')}),
+
+    ]
+)
+def test_pb2nc_all_fields(metplus_config, config_overrides,
+                               env_var_values):
+    input_dir = '/some/input/dir'
+    config = metplus_config()
+
+    # set config variables to prevent command from running and bypass check
+    # if input files actually exist
+    config.set('config', 'DO_NOT_RUN_EXE', True)
+    config.set('config', 'INPUT_MUST_EXIST', False)
+
+    # set process and time config variables
+    config.set('config', 'PROCESS_LIST', 'PB2NC')
+    config.set('config', 'LOOP_BY', 'VALID')
+    config.set('config', 'VALID_TIME_FMT', '%Y%m%d%H')
+    config.set('config', 'VALID_BEG', '2007033112')
+    config.set('config', 'VALID_END', '2007040100')
+    config.set('config', 'VALID_INCREMENT', '12H')
+    config.set('config', 'LEAD_SEQ', '0')
+    config.set('config', 'PB2NC_OFFSETS', '12')
+    config.set('config', 'LOOP_ORDER', 'processes')
+
+    config.set('config', 'PB2NC_CONFIG_FILE',
+               '{PARM_BASE}/met_config/PB2NCConfig_wrapped')
+    config.set('config', 'PB2NC_INPUT_DIR', input_dir)
+    config.set('config', 'PB2NC_INPUT_TEMPLATE',
+               'ndas.t{da_init?fmt=%2H}z.prepbufr.tm{offset?fmt=%2H}.{da_init?fmt=%Y%m%d}.nr')
+    config.set('config', 'PB2NC_OUTPUT_DIR',
+               '{OUTPUT_BASE}/PB2NC/output')
+    config.set('config', 'PB2NC_OUTPUT_TEMPLATE', '{valid?fmt=%Y%m%d%H}.nc')
+
+    # set config variable overrides
+    for key, value in config_overrides.items():
+        config.set('config', key, value)
+
+    wrapper = PB2NCWrapper(config)
+    assert(wrapper.isOK)
+
+    app_path = os.path.join(config.getdir('MET_BIN_DIR'), wrapper.app_name)
+    verbosity = f"-v {wrapper.c_dict['VERBOSITY']}"
+    config_file = wrapper.c_dict.get('CONFIG_FILE')
+    out_dir = wrapper.c_dict.get('OUTPUT_DIR')
+    expected_cmds = [(f"{app_path} {verbosity} "
+                      f"{input_dir}/ndas.t00z.prepbufr.tm12.20070401.nr "
+                      f"{out_dir}/2007033112.nc "
+                      f"{config_file}"),
+                     (f"{app_path} {verbosity} "
+                      f"{input_dir}/ndas.t12z.prepbufr.tm12.20070401.nr "
+                      f"{out_dir}/2007040100.nc "
+                      f"{config_file}"),
+                     ]
+
+    all_cmds = wrapper.run_all_times()
+    print(f"ALL COMMANDS: {all_cmds}")
+
+    for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds):
+        # ensure commands are generated as expected
+        assert(cmd == expected_cmd)
+
+        # check that environment variables were set properly
+        for env_var_key in wrapper.WRAPPER_ENV_VAR_KEYS:
+            match = next((item for item in env_vars if
+                          item.startswith(env_var_key)), None)
+            assert(match is not None)
+            value = match.split('=', 1)[1]
+            assert(env_var_values.get(env_var_key, '') == value)

metplus/wrappers/command_builder.py

@@ -330,6 +330,7 @@ def handle_obs_window_variables(self, c_dict):
         tmp_dict = {}
         for edge, default_val in edges:
             input_list = [f'OBS_{app}_WINDOW_{edge}',
+                          f'{app}_OBS_WINDOW_{edge}',
                           f'OBS_WINDOW_{edge}',
                          ]
             output_key = f'OBS_WINDOW_{edge}'

metplus/wrappers/pb2nc_wrapper.py

@@ -30,6 +30,10 @@ class PB2NCWrapper(CommandBuilder):
         'METPLUS_MASK_DICT',
         'METPLUS_OBS_BUFR_VAR',
         'METPLUS_TIME_SUMMARY_DICT',
+        'METPLUS_PB_REPORT_TYPE',
+        'METPLUS_LEVEL_RANGE_DICT',
+        'METPLUS_LEVEL_CATEGORY',
+        'METPLUS_QUALITY_MARK_THRESH',
     ]
 
     def __init__(self, config, instance=None, config_overrides={}):
@@ -136,13 +140,46 @@ def create_c_dict(self):
             station_id = '[]'
         c_dict['STATION_ID'] = station_id
 
-        c_dict['GRID'] = self.config.getstr('config', 'PB2NC_GRID')
-        c_dict['POLY'] = self.config.getstr('config', 'PB2NC_POLY')
+        c_dict['GRID'] = self.config.getstr('config', 'PB2NC_GRID', '')
+        c_dict['POLY'] = self.config.getstr('config', 'PB2NC_POLY', '')
 
         c_dict['BUFR_VAR_LIST'] = (
             self.get_env_var_value('METPLUS_OBS_BUFR_VAR')
         )
 
+        self.add_met_config(name='pb_report_type',
+                            data_type='list',
+                            metplus_configs=['PB2NC_PB_REPORT_TYPE'],
+                            extra_args={'remove_quotes': True})
+
+        # get level_range beg and end
+        level_range_items = []
+        level_range_items.append(
+            self.get_met_config(name='beg',
+                                data_type='int',
+                                metplus_configs=['PB2NC_LEVEL_RANGE_BEG',
+                                                 'PB2NC_LEVEL_RANGE_BEGIN'])
+        )
+        level_range_items.append(
+            self.get_met_config(name='end',
+                                data_type='int',
+                                metplus_configs=['PB2NC_LEVEL_RANGE_END'])
+        )
+
+        self.add_met_config(name='level_range',
+                            data_type='dict',
+                            children=level_range_items)
+
+        self.add_met_config(name='level_category',
+                            data_type='list',
+                            metplus_configs=['PB2NC_LEVEL_CATEGORY'],
+                            extra_args={'remove_quotes': True})
+
+        self.add_met_config(name='quality_mark_thresh',
+                            data_type='int',
+                            metplus_configs=['PB2NC_QUALITY_MARK_THRESH'])
+
+
         return c_dict
 
     def set_environment_variables(self, time_info):
@@ -153,26 +190,28 @@ def set_environment_variables(self, time_info):
             @param time_info dictionary containing timing info from current run
         """
         # set old method of setting env vars needed for MET config file
-        self.add_env_var("PB2NC_MESSAGE_TYPE", self.c_dict['MESSAGE_TYPE'])
-        self.add_env_var("PB2NC_STATION_ID", self.c_dict['STATION_ID'])
+        self.add_env_var("PB2NC_MESSAGE_TYPE", self.c_dict.get('MESSAGE_TYPE', ''))
+        self.add_env_var("PB2NC_STATION_ID", self.c_dict.get('STATION_ID', ''))
         self.add_env_var("OBS_WINDOW_BEGIN",
-                         str(self.c_dict['OBS_WINDOW_BEGIN']))
-        self.add_env_var("OBS_WINDOW_END", str(self.c_dict['OBS_WINDOW_END']))
-        self.add_env_var("PB2NC_GRID", self.c_dict['GRID'])
-        self.add_env_var("PB2NC_POLY", self.c_dict['POLY'])
+                         str(self.c_dict.get('OBS_WINDOW_BEGIN', '')))
+        self.add_env_var("OBS_WINDOW_END",
+                         str(self.c_dict.get('OBS_WINDOW_END', '')))
+        self.add_env_var("PB2NC_GRID", self.c_dict.get('GRID', ''))
+        self.add_env_var("PB2NC_POLY", self.c_dict.get('POLY', ''))
 
-        self.add_env_var("OBS_BUFR_VAR_LIST", self.c_dict['BUFR_VAR_LIST'])
+        self.add_env_var("OBS_BUFR_VAR_LIST", self.c_dict.get('BUFR_VAR_LIST',
+                                                              ''))
 
         self.add_env_var('TIME_SUMMARY_FLAG',
-                         self.c_dict['TIME_SUMMARY_FLAG'])
+                         self.c_dict.get('TIME_SUMMARY_FLAG', ''))
         self.add_env_var('TIME_SUMMARY_BEG',
-                         self.c_dict['TIME_SUMMARY_BEG'])
+                         self.c_dict.get('TIME_SUMMARY_BEG', ''))
         self.add_env_var('TIME_SUMMARY_END',
-                         self.c_dict['TIME_SUMMARY_END'])
+                         self.c_dict.get('TIME_SUMMARY_END', ''))
         self.add_env_var('TIME_SUMMARY_VAR_NAMES',
-                         self.c_dict['TIME_SUMMARY_VAR_NAMES'])
+                         self.c_dict.get('TIME_SUMMARY_VAR_NAMES', ''))
         self.add_env_var('TIME_SUMMARY_TYPES',
-                         self.c_dict['TIME_SUMMARY_TYPES'])
+                         self.c_dict.get('TIME_SUMMARY_TYPES', ''))
 
         super().set_environment_variables(time_info)