Feature 1062 pcp combine level template (#1118)

.github/jobs/set_job_controls.sh

@@ -73,11 +73,15 @@ else
     run_all_use_cases=false
   fi
 
-  if grep -q "ci-run-diff" <<< "$commit_msg"; then
+  if grep -q "ci-run-all-diff" <<< "$commit_msg"; then
     run_all_use_cases=true
     run_diff=true
   fi
 
+  if grep -q "ci-run-diff" <<< "$commit_msg"; then
+    run_diff=true
+  fi
+
   if grep -q "ci-run-all-cases" <<< "$commit_msg"; then
     run_use_cases=true
     run_all_use_cases=true

docs/Contributors_Guide/continuous_integration.rst

@@ -120,7 +120,10 @@ Here is a list of the currently supported keywords and what they control:
 * **ci-skip-all**: Don't run anything - skip all automation jobs
 * **ci-skip-use-cases**: Don't run any use cases
 * **ci-run-all-cases**: Run all use cases
-* **ci-run-diff**: Obtain truth data and run diffing logic
+* **ci-run-diff**: Obtain truth data and run diffing logic for
+  use cases that are marked to run
+* **ci-run-all-diff**: Obtain truth data and run diffing logic for
+  all use cases
 * **ci-only-docs**: Only run build documentation job - skip the rest
 
 Force MET Version Used for Tests

docs/Users_Guide/glossary.rst

@@ -1247,7 +1247,7 @@ METplus Configuration Glossary
      .. warning:: **DEPRECATED:** Please use :term:`FCST_POINT_STAT_INPUT_TEMPLATE` instead.
 
    FCST_IS_DAILY_FILE
-     .. warning:: **DEPRECATED:** Please use :term:`FCST_PCP_COMBINE_IS_DAILY_FILE` instead.
+     .. warning:: **DEPRECATED:**
 
    FCST_IS_PROB
      Specify whether the forecast data are probabilistic or not. Acceptable values: true/false
@@ -1361,12 +1361,31 @@ METplus Configuration Glossary
      .. warning:: **DEPRECATED:** Please use :term:`FCST_PCP_COMBINE_INPUT_NAMES` instead.
 
    FCST_PCP_COMBINE_DATA_INTERVAL
-     Specify the accumulation interval of the forecast dataset used by the MET pcp_combine tool when processing daily input files. A corresponding variable exists for observation data called :term:`OBS_PCP_COMBINE_DATA_INTERVAL`.
+     .. warning:: **DEPRECATED:**
+
+   FCST_PCP_COMBINE_DERIVE_LOOKBACK
+     .. warning:: **DEPRECATED:** Please use :term:`FCST_PCP_COMBINE_LOOKBACK` instead.
+
+   FCST_PCP_COMBINE_LOOKBACK
+     Specify how far to look back in time to find files for building
+     commands to run the pcp_combine tool.
+     If processing precipitation accumulation data, this is equivalent to the
+     desired output accumulation to compute.
+     Units are assumed to be hours unless a time identifier such as
+     Y, m, d, H, M, S is specified at the end of the value, i.e. 30M or 1m.
+     If unset, :term:`FCST_PCP_COMBINE_OUTPUT_ACCUM` will be used.
+     If that is unset, then :term:`FCST_PCP_COMBINE_DERIVE_LOOKBACK` will be
+     used.
+     If none of the variables are set or set to 0, data will be obtained by
+     using the input template with the current runtime instead of looking
+     backwards in time.
+     A corresponding variable exists for observation data called
+     :term:`OBS_PCP_COMBINE_LOOKBACK`.
 
      | *Used by:*  PCPCombine
 
-   FCST_PCP_COMBINE_DERIVE_LOOKBACK
-     Specify how far to look back in time in hours to find files for running the MET pcp_combine tool in derive mode. If set to 0 or unset, data will be obtained by using the input template with the current runtime instead of looking backwards in time. A corresponding variable exists for observation data called :term:`OBS_PCP_COMBINE_DERIVE_LOOKBACK`.
+   OBS_PCP_COMBINE_LOOKBACK
+     See :term:`FCST_PCP_COMBINE_LOOKBACK`.
 
      | *Used by:*  PCPCombine
 
@@ -1389,9 +1408,7 @@ METplus Configuration Glossary
      | *Used by:*  PCPCombine
 
    FCST_PCP_COMBINE_IS_DAILY_FILE
-     Specify whether the forecast file is a daily file or not. A corresponding variable exists for observation data called :term:`OBS_PCP_COMBINE_IS_DAILY_FILE`.Acceptable values: true/false
-
-     | *Used by:*  PCPCombine
+     .. warning:: **DEPRECATED:**
 
    FCST_PCP_COMBINE_METHOD
      Specify the method to be used with the MET pcp_combine tool processing forecast data.Valid options are ADD, SUM, SUBTRACT, DERIVE, and USER_DEFINED. A corresponding variable exists for observation data called :term:`OBS_PCP_COMBINE_METHOD`.
@@ -1438,9 +1455,7 @@ METplus Configuration Glossary
      | *Used by:*  PCPCombine
 
    FCST_PCP_COMBINE_TIMES_PER_FILE
-     Specify the number of accumulation intervals of the forecast dataset used by the MET pcp_combine tool when processing daily input files. A corresponding variable exists for observation data called :term:`OBS_PCP_COMBINE_TIMES_PER_FILE`.
-
-     | *Used by:*  PCPCombine
+     .. warning:: **DEPRECATED:**
 
    FCST_POINT_STAT_FILE_WINDOW_BEGIN
      See :term:`OBS_POINT_STAT_FILE_WINDOW_BEGIN`
@@ -2624,10 +2639,10 @@ METplus Configuration Glossary
      .. warning:: **DEPRECATED:** Please use :term:`PB2NC_OBS_BUFR_VAR_LIST` instead.
 
    OBS_DATA_INTERVAL
-     .. warning:: **DEPRECATED:** Use :term:`OBS_PCP_COMBINE_DATA_INTERVAL` instead.
+     .. warning:: **DEPRECATED:**
 
    FCST_DATA_INTERVAL
-     .. warning:: **DEPRECATED:** Use :term:`FCST_PCP_COMBINE_DATA_INTERVAL` instead.
+     .. warning:: **DEPRECATED:**
 
    FCST_ENSEMBLE_STAT_INPUT_DATATYPE
      Specify the data type of the input directory for forecast files used with the MET ensemble_stat tool. Currently valid options are NETCDF, GRIB, and GEMPAK. If set to GEMPAK, data will automatically be converted to NetCDF via GempakToCF. Similar variables exists for observation grid and point data called :term:`OBS_ENSEMBLE_STAT_INPUT_GRID_DATATYPE` and :term:`OBS_ENSEMBLE_STAT_INPUT_POINT_DATATYPE`.
@@ -2759,7 +2774,7 @@ METplus Configuration Glossary
      .. warning:: **DEPRECATED:** Please use :term:`OBS_POINT_STAT_INPUT_TEMPLATE` instead.
 
    OBS_IS_DAILY_FILE
-     .. warning:: **DEPRECATED:** Please use :term:`OBS_PCP_COMBINE_IS_DAILY_FILE` instead.
+     .. warning:: **DEPRECATED:**
 
    OBS_IS_PROB
      Used when setting OBS_* variables to process forecast data for comparisons with mtd. Specify whether the observation data are probabilistic or not. See :term:`FCST_IS_PROB` .Acceptable values: true/false
@@ -2861,20 +2876,18 @@ METplus Configuration Glossary
      .. warning:: **DEPRECATED:** Please use :term:`OBS_PCP_COMBINE_INPUT_DATATYPE` instead.
 
    OBS_TIMES_PER_FILE
-     .. warning:: **DEPRECATED:** Please use :term:`OBS_PCP_COMBINE_TIMES_PER_FILE` instead.
+     .. warning:: **DEPRECATED:**
 
    FCST_TIMES_PER_FILE
-     .. warning:: **DEPRECATED:** Please use :term:`FCST_PCP_COMBINE_TIMES_PER_FILE` instead.
+     .. warning:: **DEPRECATED:**
 
    OBS_PCP_COMBINE_<n>_FIELD_NAME
      See :term:`FCST_PCP_COMBINE_<n>_FIELD_NAME`.
 
      | *Used by:*  PCPCombine
 
    OBS_PCP_COMBINE_DATA_INTERVAL
-     See :term:`FCST_PCP_COMBINE_DATA_INTERVAL`.
-
-     | *Used by:*  PCPCombine
+     .. warning:: **DEPRECATED:**
 
    OBS_PCP_COMBINE_DERIVE_LOOKBACK
      See :term:`FCST_PCP_COMBINE_DERIVE_LOOKBACK`.
@@ -2902,9 +2915,7 @@ METplus Configuration Glossary
      | *Used by:*  PCPCombine
 
    OBS_PCP_COMBINE_IS_DAILY_FILE
-     See :term:`FCST_PCP_COMBINE_IS_DAILY_FILE`. Acceptable values: true/false
-
-     | *Used by:*  PCPCombine
+     .. warning:: **DEPRECATED:**
 
    OBS_PCP_COMBINE_METHOD
      See :term:`FCST_PCP_COMBINE_METHOD`.
@@ -2942,9 +2953,7 @@ METplus Configuration Glossary
      | *Used by:*  PCPCombine
 
    OBS_PCP_COMBINE_TIMES_PER_FILE
-     See :term:`FCST_PCP_COMBINE_TIMES_PER_FILE`.
-
-     | *Used by:*  PCPCombine
+     .. warning:: **DEPRECATED:**
 
    OBS_POINT_STAT_FILE_WINDOW_BEGIN
      Used to control the lower bound of the window around the valid time to determine if a file should be used for processing by PointStat. See :ref:`Directory_and_Filename_Template_Info` subsection called 'Using Windows to Find Valid Files.' Units are seconds. If :term:`OBS_POINT_STAT_FILE_WINDOW_BEGIN` is not set in the config file, the value of :term:`OBS_FILE_WINDOW_BEGIN` will be used instead. If both file window begin and window end values are set to 0, then METplus will require an input file with an exact time match to process.
@@ -4357,27 +4366,26 @@ METplus Configuration Glossary
      | *Used by:*  PCPCombine
 
    FCST_PCP_COMBINE_OUTPUT_ACCUM
-     Specify desired accumulation to be built from the forecast data. Units are assumed to be hours unless a time identifier such as Y, m, d, H, M, S is specifed at the end of the value, i.e. 30M or 1m. If this variable is not set, then FCST_VAR<n>_LEVELS is used.
+     Specify desired accumulation to be built from the forecast data.
+     Synonym for :term:`FCST_PCP_COMBINE_LOOKBACK`.
 
      A corresponding variable exists for observation data called :term:`OBS_PCP_COMBINE_OUTPUT_ACCUM`.
 
-     Examples:
-
-     15H
-
-     This will attempt to build a 15 hour accumulation.
-
      | *Used by:*  PCPCombine
 
    FCST_PCP_COMBINE_OUTPUT_NAME
-     Specify the output field name from processing forecast data. If this variable is not set, then :term:`FCST_VAR<n>_NAME` is used.
+     Specify the output field name from processing forecast data.
+     If this variable is not set, then :term:`FCST_VAR<n>_NAME` is used.
 
-     A corresponding variable exists for observation data called :term:`OBS_PCP_COMBINE_OUTPUT_NAME`.
+     A corresponding variable exists for observation data called
+     :term:`OBS_PCP_COMBINE_OUTPUT_NAME`.
 
      Example: APCP
 
+     | *Used by:*  PCPCombine
+
    OBS_PCP_COMBINE_OUTPUT_ACCUM
-     See :term:`FCST_PCP_COMBINE_OUTPUT_NAME`.
+     See :term:`FCST_PCP_COMBINE_LOOKBACK`.
 
      | *Used by:*  PCPCombine
 

docs/Users_Guide/wrappers.rst

@@ -4288,18 +4288,12 @@ METplus Configuration
 | :term:`OBS_PCP_COMBINE_CONSTANT_INIT`
 | :term:`FCST_PCP_COMBINE_STAT_LIST`
 | :term:`OBS_PCP_COMBINE_STAT_LIST`
-| :term:`FCST_PCP_COMBINE_DERIVE_LOOKBACK`
-| :term:`OBS_PCP_COMBINE_DERIVE_LOOKBACK`
 | :term:`PCP_COMBINE_SKIP_IF_OUTPUT_EXISTS`
-| :term:`FCST_PCP_COMBINE_DATA_INTERVAL`
-| :term:`OBS_PCP_COMBINE_DATA_INTERVAL`
-| :term:`FCST_PCP_COMBINE_TIMES_PER_FILE`
-| :term:`OBS_PCP_COMBINE_TIMES_PER_FILE`
-| :term:`FCST_PCP_COMBINE_IS_DAILY_FILE`
-| :term:`OBS_PCP_COMBINE_IS_DAILY_FILE`
 | :term:`FCST_PCP_COMBINE_COMMAND`
 | :term:`OBS_PCP_COMBINE_COMMAND`
 | :term:`PCP_COMBINE_CUSTOM_LOOP_LIST`
+| :term:`FCST_PCP_COMBINE_LOOKBACK`
+| :term:`OBS_PCP_COMBINE_LOOKBACK`
 | :term:`FCST_PCP_COMBINE_EXTRA_NAMES` (optional)
 | :term:`FCST_PCP_COMBINE_EXTRA_LEVELS` (optional)
 | :term:`FCST_PCP_COMBINE_EXTRA_OUTPUT_NAMES` (optional)
@@ -4331,6 +4325,14 @@ METplus Configuration
    | :term:`OBS_PCP_COMBINE_INPUT_LEVEL`
    | :term:`FCST_PCP_COMBINE_<n>_FIELD_NAME`
    | :term:`OBS_PCP_COMBINE_<n>_FIELD_NAME`
+   | :term:`FCST_PCP_COMBINE_DATA_INTERVAL`
+   | :term:`OBS_PCP_COMBINE_DATA_INTERVAL`
+   | :term:`FCST_PCP_COMBINE_TIMES_PER_FILE`
+   | :term:`OBS_PCP_COMBINE_TIMES_PER_FILE`
+   | :term:`FCST_PCP_COMBINE_IS_DAILY_FILE`
+   | :term:`OBS_PCP_COMBINE_IS_DAILY_FILE`
+   | :term:`FCST_PCP_COMBINE_DERIVE_LOOKBACK`
+   | :term:`OBS_PCP_COMBINE_DERIVE_LOOKBACK`
    |
 
 .. _plot_data_plane_wrapper:

internal_tests/pytests/compare_gridded/test_compare_gridded.py

@@ -121,20 +121,20 @@ def test_get_field_info_no_prob(metplus_config, key, value):
     'key, value', [
         # forecast grib name level thresh
         (['NAME', 'L0', ['gt3', '<=5'], '', 'FCST'],
-         ['{ name=\"PROB\"; level=\"L0\"; prob={ name=\"NAME\"; thresh_lo=3.0; } cat_thresh=[==0.1]; }',
-          '{ name=\"PROB\"; level=\"L0\"; prob={ name=\"NAME\"; thresh_hi=5.0; } cat_thresh=[==0.1]; }']),
+         ['{ name=\"PROB\"; level=\"L0\"; prob={ name=\"NAME\"; thresh_lo=3.0; } cat_thresh=[ ==0.1 ]; }',
+          '{ name=\"PROB\"; level=\"L0\"; prob={ name=\"NAME\"; thresh_hi=5.0; } cat_thresh=[ ==0.1 ]; }']),
 
         # obs grib name level thresh
         (['NAME', 'L0', ['gt3', '<=5'], '', 'OBS'],
          ['{ name=\"NAME\"; level=\"L0\"; cat_thresh=[ gt3 ]; }',
           '{ name=\"NAME\"; level=\"L0\"; cat_thresh=[ <=5 ]; }']),
 
         (['NAME', 'L0', ['gt3&&lt5'], '', 'FCST'],
-         ['{ name=\"PROB\"; level=\"L0\"; prob={ name=\"NAME\"; thresh_lo=3.0; thresh_hi=5.0; } cat_thresh=[==0.1]; }']),
+         ['{ name=\"PROB\"; level=\"L0\"; prob={ name=\"NAME\"; thresh_lo=3.0; thresh_hi=5.0; } cat_thresh=[ ==0.1 ]; }']),
 
         # fcst grib name py script
         (['/some/script/name.py args /path/of/infile.txt', '', [], '', 'FCST'],
-         ['{ name=\"/some/script/name.py args /path/of/infile.txt\"; prob=TRUE; cat_thresh=[==0.1]; }']),
+         ['{ name=\"/some/script/name.py args /path/of/infile.txt\"; prob=TRUE; cat_thresh=[ ==0.1 ]; }']),
 
         # obs name py script
         (['/some/script/name.py args /path/of/infile.txt', '', [], '', 'OBS'],
@@ -150,11 +150,11 @@ def test_get_field_info_fcst_prob_grib_pds(metplus_config, key, value):
     w.c_dict['FCST_PROB_IN_GRIB_PDS'] = True
     w.c_dict['FCST_PROB_THRESH'] = '==0.1'
 
-    field_dict = {'v_name' : key[0],
-                  'v_level' : key[1],
-                  'v_thresh' : key[2],
-                  'v_extra' : key[3],
-                  'd_type' : key[4],
+    field_dict = {'v_name': key[0],
+                  'v_level': key[1],
+                  'v_thresh': key[2],
+                  'v_extra': key[3],
+                  'd_type': key[4],
                   }
 
     fields = w.get_field_info(**field_dict)
@@ -170,20 +170,20 @@ def test_get_field_info_fcst_prob_grib_pds(metplus_config, key, value):
     'key, value', [
         # forecast grib name level thresh
         (['NAME', 'L0', ['gt3', '<=5'], '', 'FCST'],
-         ['{ name=\"NAME\"; level=\"L0\"; prob=TRUE; cat_thresh=[==0.1]; }',
-          '{ name=\"NAME\"; level=\"L0\"; prob=TRUE; cat_thresh=[==0.1]; }']),
+         ['{ name=\"NAME\"; level=\"L0\"; prob=TRUE; cat_thresh=[ ==0.1 ]; }',
+          '{ name=\"NAME\"; level=\"L0\"; prob=TRUE; cat_thresh=[ ==0.1 ]; }']),
 
         # obs grib name level thresh
         (['NAME', 'L0', ['gt3', '<=5'], '', 'OBS'],
          ['{ name=\"NAME\"; level=\"L0\"; cat_thresh=[ gt3 ]; }',
           '{ name=\"NAME\"; level=\"L0\"; cat_thresh=[ <=5 ]; }']),
 
         (['NAME', 'L0', ['gt3&&lt5'], '', 'FCST'],
-         ['{ name=\"NAME\"; level=\"L0\"; prob=TRUE; cat_thresh=[==0.1]; }']),
+         ['{ name=\"NAME\"; level=\"L0\"; prob=TRUE; cat_thresh=[ ==0.1 ]; }']),
 
         # fcst grib name py script
         (['/some/script/name.py args /path/of/infile.txt', '', [], '', 'FCST'],
-         ['{ name=\"/some/script/name.py args /path/of/infile.txt\"; prob=TRUE; cat_thresh=[==0.1]; }']),
+         ['{ name=\"/some/script/name.py args /path/of/infile.txt\"; prob=TRUE; cat_thresh=[ ==0.1 ]; }']),
 
         # obs name py script
         (['/some/script/name.py args /path/of/infile.txt', '', [], '', 'OBS'],

internal_tests/pytests/pcp_combine/test1.conf

@@ -1,6 +1,7 @@
 [config]
 FCST_PCP_COMBINE_INPUT_ACCUMS = 6
 FCST_PCP_COMBINE_INPUT_NAMES = P06M_NONE
+FCST_PCP_COMBINE_INPUT_LEVELS = "(*,*)"
 
 OBS_PCP_COMBINE_INPUT_ACCUMS = 1
 OBS_PCP_COMBINE_INPUT_NAMES = P01M_NONE