From af49567473a0986cfebf6fd9785ec29515374aca Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 2 Aug 2023 14:29:07 -0600 Subject: [PATCH] Update develop-ref after #2280 (#2286) Co-authored-by: cristianastan2 Co-authored-by: John Halley Gotway Co-authored-by: bikegeek Co-authored-by: Lisa Goodrich Co-authored-by: Julie Prestopnik Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> Co-authored-by: Hank Fisher Co-authored-by: Dan Adriaansen Co-authored-by: johnhg Co-authored-by: jprestop Co-authored-by: Tracy Hertneky Co-authored-by: Giovanni Rosa Co-authored-by: j-opatz <59586397+j-opatz@users.noreply.github.com> Co-authored-by: Mrinal Biswas Co-authored-by: j-opatz Co-authored-by: Daniel Adriaansen Co-authored-by: Jonathan Vigh Co-authored-by: root Co-authored-by: bikegeek <3753118+bikegeek@users.noreply.github.com> Co-authored-by: Will Mayfield <59745143+willmayfield@users.noreply.github.com> Co-authored-by: lisagoodrich <33230218+lisagoodrich@users.noreply.github.com> Co-authored-by: metplus-bot <97135045+metplus-bot@users.noreply.github.com> Co-authored-by: Tracy Hertneky <39317287+hertneky@users.noreply.github.com> Co-authored-by: Giovanni Rosa Co-authored-by: mrinalbiswas Co-authored-by: Christina Kalb Co-authored-by: jason-english <73247785+jason-english@users.noreply.github.com> Co-authored-by: John Sharples <41682323+John-Sharples@users.noreply.github.com> fix release (#1790) fix GitHub Actions warnings (#1864) fix #1884 develop PCPCombine {custom} in subtract method (#1887) fix #1939 develop - failure reading obs when zipped file also exists (#1941) Closes https://github.com/dtcenter/METplus/issues/1986 fix develop Fix broken documentation links (#2004) fix #2026 develop StatAnalysis looping (#2028) fix priority of obs_window config variables so that wrapper-specific version is preferred over generic OBS_WINDOW_BEGIN/END (#2062) fix #2070 var list numeric order (#2072) fix #2087 develop docs_pdf (#2091) fix #2096/#2098 develop - fix skip if output exists and do not error if no commands were run (#2099) Fix for Dockerfile smell DL4000 (#2112) fix #2082 develop regrid.convert/censor_thresh/censor_val (#2140) fix #2082 main_v5.0 regrid.convert/censor_thresh/censor_val (#2101) fix #2137 develop PointStat -obs_valid_beg/end (#2141) fix failured introduced by urllib3 (see https://github.com/urllib3/urllib3/issues/2168) fix #2161 develop PCPCombine additional field arguments in -subtract mode (#2162) fix #2168 develop - StatAnalysis time shift (#2169) fix releases. (#2183) fix #2189 develop - spaces in complex thresholds (#2191) fix #2179 develop TCPairs fix -diag argument (#2187) fixes (#2200) fix diff tests (#2217) fix automated tests (#2237) fix #2235 rename multivar_itensity to multivar_intensity_flag (#2236) fix #2241 Create directory containing -out_stat file (#2242) fix #2245 use unique run ID to name logger instance (#2247) fix #2244 develop fix diff tests (#2254) fixture to set pytest tmpdir (#2261) fix #1853 develop - PointStat don't require mask variables to be set (#2262) fix #2279 develop - buoy station file from 2022 (#2280) --- .github/jobs/docker_setup.sh | 11 +- .github/parm/Externals_metdataio_stable.cfg | 2 +- .../parm/Externals_metplotcalcpy_stable.cfg | 4 +- .github/pull_request_template.md | 2 +- .github/workflows/testing.yml | 2 +- build_components/Externals_stable.cfg | 10 +- docs/Contributors_Guide/add_use_case.rst | 35 +- docs/Contributors_Guide/coding_standards.rst | 108 +--- .../continuous_integration.rst | 176 ++++++- docs/Contributors_Guide/github_workflow.rst | 102 ++-- docs/Contributors_Guide/testing.rst | 58 ++- docs/Users_Guide/quicksearch.rst | 4 + docs/Users_Guide/release-notes.rst | 52 +- docs/Users_Guide/wrappers.rst | 30 +- docs/conf.py | 2 +- .../met_tool_wrapper/TCDiag/TCDiag.py | 4 +- .../UserScript_fcstGEFS_Difficulty_Index.py | 2 + .../s2s/TCGen_fcstGFSO_obsBDECKS_GDF_TDF.py | 3 +- ...UserScript_fcstS2S_obsERAI_CrossSpectra.py | 2 + .../UserScript_obsERA_obsOnly_Stratosphere.py | 1 + ...UserScript_obsPrecip_obsOnly_Hovmoeller.py | 1 + .../UserScript_obsERA_obsOnly_Blocking.py | 2 + ...UserScript_obsERA_obsOnly_WeatherRegime.py | 2 + .../s2s_mjo/UserScript_fcstGFS_obsERA_OMI.py | 2 + .../UserScript_obsCFSR_obsOnly_MJO_ENSO.py | 2 + .../UserScript_obsERA_obsOnly_PhaseDiagram.py | 1 + .../s2s_mjo/UserScript_obsERA_obsOnly_RMM.py | 2 + ...stFV3_fcstOnly_PhysicsTendency_Planview.py | 1 + ...ly_PhysicsTendency_VerticalCrossSection.py | 1 + ...cstOnly_PhysicsTendency_VerticalProfile.py | 1 + internal/scripts/aws/EC2_README | 48 ++ .../metplus_components_v5.1_py3.10.sh | 36 ++ .../installation/modulefiles/5.1.0.lua_wcoss2 | 26 + .../installation/modulefiles/5.1.0_acorn | 22 + .../installation/modulefiles/5.1.0_cheyenne | 17 + .../installation/modulefiles/5.1.0_orion | 25 + internal/tests/pytests/conftest.py | 87 +++- .../pytests/util/diff_util/test_diff_util.py | 465 +++++++++++++++++- .../pytests/wrappers/point_stat/__init__.py | 0 .../point_stat/point_stat_test_conus_sfc.conf | 138 ------ .../point_stat/point_stat_test_upper_air.conf | 128 ----- .../refactor_point_stat_test_conus_sfc.conf | 111 ----- .../point_stat/test_point_stat_wrapper.py | 113 ++++- .../series_analysis/test_series_analysis.py | 96 +++- metplus/RELEASE_DATE | 2 +- metplus/VERSION | 2 +- metplus/util/diff_util.py | 10 +- metplus/wrappers/command_builder.py | 7 +- metplus/wrappers/point_stat_wrapper.py | 13 +- parm/met_config/PointStatConfig_wrapped | 9 +- .../PointStat_fcstGFS_obsNDBC_WaveHeight.conf | 2 + requirements.txt | 2 +- 52 files changed, 1258 insertions(+), 726 deletions(-) create mode 100644 internal/scripts/aws/EC2_README create mode 100644 internal/scripts/installation/metplus_components_v5.1_py3.10.sh create mode 100644 internal/scripts/installation/modulefiles/5.1.0.lua_wcoss2 create mode 100644 internal/scripts/installation/modulefiles/5.1.0_acorn create mode 100644 internal/scripts/installation/modulefiles/5.1.0_cheyenne create mode 100644 internal/scripts/installation/modulefiles/5.1.0_orion delete mode 100644 internal/tests/pytests/wrappers/point_stat/__init__.py delete mode 100644 internal/tests/pytests/wrappers/point_stat/point_stat_test_conus_sfc.conf delete mode 100644 internal/tests/pytests/wrappers/point_stat/point_stat_test_upper_air.conf delete mode 100644 internal/tests/pytests/wrappers/point_stat/refactor_point_stat_test_conus_sfc.conf diff --git a/.github/jobs/docker_setup.sh b/.github/jobs/docker_setup.sh index 47116c0232..8cab9ad44d 100755 --- a/.github/jobs/docker_setup.sh +++ b/.github/jobs/docker_setup.sh @@ -43,10 +43,13 @@ elif [ "${EXTERNAL_TRIGGER}" == "true" ]; then MET_TAG=${MET_TAG}-lite fi -# if MET_FORCE_TAG variable is set and not empty, use that version instead -if [ ! -z "$MET_FORCE_TAG" ]; then - MET_TAG=$MET_FORCE_TAG - MET_DOCKER_REPO=met +# if SET_MET_IMAGE variable is set and not empty, use that version instead +# format is : where the repo used is dtcenter/ and +# the tag used is , e.g. met:11.1.0 uses dtcenter/met:11.1.0 +if [ ! -z "$SET_MET_IMAGE" ]; then + force_info=(${SET_MET_IMAGE//:/ }) + MET_DOCKER_REPO=${force_info[0]} + MET_TAG=${force_info[1]} fi echo Using MET_DOCKER_REPO=$MET_DOCKER_REPO diff --git a/.github/parm/Externals_metdataio_stable.cfg b/.github/parm/Externals_metdataio_stable.cfg index 3a6b51b4b2..af45402334 100644 --- a/.github/parm/Externals_metdataio_stable.cfg +++ b/.github/parm/Externals_metdataio_stable.cfg @@ -3,7 +3,7 @@ local_path = ../METdataio protocol = git required = True repo_url = https://github.com/dtcenter/METdataio -branch = main_v2.0 +branch = main_v2.1 [externals_description] schema_version = 1.0.0 diff --git a/.github/parm/Externals_metplotcalcpy_stable.cfg b/.github/parm/Externals_metplotcalcpy_stable.cfg index b0486d0814..5aed1cb69b 100644 --- a/.github/parm/Externals_metplotcalcpy_stable.cfg +++ b/.github/parm/Externals_metplotcalcpy_stable.cfg @@ -3,14 +3,14 @@ local_path = ../METcalcpy protocol = git required = True repo_url = https://github.com/dtcenter/METcalcpy -branch = main_v2.0 +branch = main_v2.1 [METplotpy] local_path = ../METplotpy protocol = git required = True repo_url = https://github.com/dtcenter/METplotpy -branch = main_v2.0 +branch = main_v2.1 [externals_description] schema_version = 1.0.0 diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index a8d9a04af0..48c6621f2a 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -23,6 +23,6 @@ See the [METplus Workflow](https://metplus.readthedocs.io/en/latest/Contributors Select: **Reviewer(s)** Select: **Organization** level software support **Project** or **Repository** level development cycle **Project** Select: **Milestone** as the version that will include these changes -- [ ] After submitting the PR, select **Development** issue with the original issue number. +- [ ] fter submitting the PR, select the :gear: icon in the **Development** section of the right hand sidebar. Search for the issue that this PR will close and select it, if it is not already selected. - [ ] After the PR is approved, merge your changes. If permissions do not allow this, request that the reviewer do the merge. - [ ] Close the linked issue and delete your feature or bugfix branch from GitHub. diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 556efb70c6..aff2d9a496 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -99,7 +99,7 @@ jobs: DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} EXTERNAL_TRIGGER: ${{ needs.job_control.outputs.external_trigger }} - #MET_FORCE_TAG: 11.0.0 + #SET_MET_IMAGE: met:11.0.0 update_data_volumes: name: Docker Setup - Update Data Volumes diff --git a/build_components/Externals_stable.cfg b/build_components/Externals_stable.cfg index 28380c3ba3..4fbdba1094 100644 --- a/build_components/Externals_stable.cfg +++ b/build_components/Externals_stable.cfg @@ -3,35 +3,35 @@ local_path = ../MET protocol = git required = True repo_url = https://github.com/dtcenter/MET -tag = v11.0.0 +tag = v11.1.0 [METviewer] local_path = ../METviewer protocol = git required = True repo_url = https://github.com/dtcenter/METviewer -tag = v5.0.0 +tag = v5.1.0 [METplotpy] local_path = ../METplotpy protocol = git required = True repo_url = https://github.com/dtcenter/METplotpy -tag = v2.0.0 +tag = v2.1.0 [METcalcpy] local_path = ../METcalcpy protocol = git required = True repo_url = https://github.com/dtcenter/METcalcpy -tag = v2.0.0 +tag = v2.1.0 [METdataio] local_path = ../METdataio protocol = git required = True repo_url = https://github.com/dtcenter/METdataio -tag = v2.0.0 +tag = v2.1.0 [externals_description] diff --git a/docs/Contributors_Guide/add_use_case.rst b/docs/Contributors_Guide/add_use_case.rst index b4ca716bb1..e6445f5944 100644 --- a/docs/Contributors_Guide/add_use_case.rst +++ b/docs/Contributors_Guide/add_use_case.rst @@ -630,14 +630,19 @@ Add volume_mount_directories file ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Copy the volume_mount_directories file from the develop directory into the -branch directory. Update the entry for the new tarball if the mounting point -has changed (unlikely) or add a new entry if adding a new sample data -tarfile. The format of this file generally follows -:model_applications/, i.e. -climate:model_applications/climate:: +branch directory:: cp ${METPLUS_DATA_TARFILE_DIR}/develop/volume_mount_directories ${METPLUS_DATA_TARFILE_DIR}/${METPLUS_FEATURE_BRANCH} +**IF YOU ARE ADDING A NEW USE CASE TO AN EXISTING CATEGORY, SKIP TO THE NEXT STEP.** + +If you are adding a new use case category, add a new entry to the volume mount +directories file for the new category. +Add the new entry in alphabetical order so it is easier for others to review. +The format of this file follows +****:model_applications/****, e.g. +**climate**:model_applications/**climate**. + Log out of DTC Web Server ^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -983,7 +988,11 @@ copy the feature file into the upcoming METplus version directory and the develo Copy data from the feature directory into the next version directory ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Copy files +"""""""""" + **Make sure the paths are correct before copying.** +Paths may need to be adjusted. **ONLY RUN THE COMMAND THAT IS APPROPRIATE TO THE USE CASE. READ CAREFULLY!** @@ -1013,6 +1022,9 @@ After verifying the directories are correct, copy the files:: cp -r $from_directory $to_directory/ +Handle existing tarfile in vX.Y +""""""""""""""""""""""""""""""" + List the tarfile for the use case category in the next release version directory:: cd ${METPLUS_DATA_TARFILE_DIR}/v${METPLUS_VERSION} @@ -1033,14 +1045,21 @@ another METplus version**, then simply remove the tarfile link:: unlink sample_data-${METPLUS_USE_CASE_CATEGORY}.tgz - -Remove old data (if applicable). + +**OR** + +**CONDITION 3: IF the sample data tarfile for the category does not exist** +(because it is a new use case category), continue to the next step. + +Remove old data (if applicable) +""""""""""""""""""""""""""""""" If the pull request notes mention an old directory path that should be removed, please remove that directory. Be careful not to remove any files that are still needed. -Create the new sample data tarfile. +Create the new sample data tarfile +"""""""""""""""""""""""""""""""""" **ONLY RUN THE COMMAND THAT IS APPROPRIATE TO THE USE CASE. READ CAREFULLY!** diff --git a/docs/Contributors_Guide/coding_standards.rst b/docs/Contributors_Guide/coding_standards.rst index 99bac04427..67cee26da1 100644 --- a/docs/Contributors_Guide/coding_standards.rst +++ b/docs/Contributors_Guide/coding_standards.rst @@ -4,116 +4,10 @@ Coding Standards **************** - - * Python code style outlined in `PEP8 `_ -* Python section of the `NCEP Coding Standards `_ -* `NCO WCOSS Implementation Standards `_ for - directory structure and script naming conventions * `Doxygen `_, `Python docstrings `_, and `Sphinx `_ for documentation * **NOTE: Please do not use f-strings in the run_metplus.py file so that the Python version check can notify the user of the incorrect version. Using Python 3.5 or earlier will output the SyntaxError from the f-string instead of the useful error message.** -Python Code Analysis Tools -========================== - -Static Tools ------------- - -pylint -^^^^^^ - -`pylint `_ is a tool that checks -for errors in Python code, tries to enforce a coding standard and looks for code -smells. - -To `install pylint `_ -the following can be run: - -.. code-block:: ini - - pip install pylint - -or - -.. code-block:: ini - - conda install pylint - - -To check for errors as well as PEP-8 style code, run: - -.. code-block:: ini - - pylint pep8 - -replacing with the name of the file to analyze. - - -pyflakes -^^^^^^^^ - -`pyflakes `_ is a simple program which -checks Python source files for errors. Pyflakes analyzes programs and -detects various errors. It works by parsing the source file, not importing -it, so it is safe to use on modules with side effects. It’s also much faster. - - -To install pyflakes the following can be run: - -.. code-block:: ini - - pip install pyflakes - -or - -.. code-block:: ini - - conda install pyflakes - - -`flake8 `_ is wrapper -to pyflakes, performs PEP-8 style checking in addition to error checking. - -vulture -^^^^^^^ - -`vulture `_ finds unused code in Python -programs and is useful for cleaning up and finding errors in large code bases. -It checks for unused imports, variables, methods, and classes. - -To install vulture the following can be run: - -.. code-block:: ini - - pip install vulture - -or - -.. code-block:: ini - - conda install vulture - - -Dynamic (run-time) Tools ------------------------- - -Code Coverage Analysis -^^^^^^^^^^^^^^^^^^^^^^ - -Code coverage analysis tools are useful when running unit tests to determine -whether tests are executing all possible branches, loops, etc. - -**Examples:** - -`Coverage.py `_: A free tool for -monitoring the coverage of your Python apps, monitoring every bit of your code -to find what was executed and what was not. - -`pytest-cov `_: A free language plug-in -to produce a coverage report of your app. - -`figleaf `_: A code coverage analysis -tool intended to be to be a minimal replacement of 'coverage.py' that supports -more configurable coverage gathering and reporting. +Python code analysis tools like **pylint** can be used to check for errors and violations of PEP8 standards. diff --git a/docs/Contributors_Guide/continuous_integration.rst b/docs/Contributors_Guide/continuous_integration.rst index 996e93c98d..0fbe5fff72 100644 --- a/docs/Contributors_Guide/continuous_integration.rst +++ b/docs/Contributors_Guide/continuous_integration.rst @@ -96,6 +96,8 @@ was saved as a GitHub Secret. Testing Workflow ================ +The testing workflow file is found in **.github/workflows/testing.yml**. + Name ---- @@ -470,7 +472,7 @@ Create/Update METplus Docker Image env: DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - #MET_FORCE_TAG: 10.0.0 + #SET_MET_IMAGE: met:10.0.0 This job calls the **docker_setup.sh** script. This script builds a METplus Docker image and pushes it to DockerHub. @@ -496,15 +498,35 @@ development testing. If testing is done on a stable release, then the corresponding MET stable release will be used. However, there may be an instance where a change in MET breaks something in another METplus component, i.e. METplotpy or METviewer, until a corresponding change is made to that -component. If this occurs then some of the METplus use cases may break. To -allow the tests to run successfully in the meantime, an option was added to -force the version of the MET tag that is used to build the METplus Docker image -that is used for testing. In the **testing.yml** workflow file, -there is a commented variable called -MET_FORCE_TAG that can be uncommented and set to force the version of MET to -use. This variable is found in the **get_image** job under the **env** section +component. If this occurs then some of the METplus use cases may break. + +Another situation that may require a different MET Docker image is if there +are changes in a MET feature or bugfix branch that are needed to test changes +in METplus. + +To allow the tests to run successfully in these cases, an option was added to +force a specific MET Docker image to be used to build the METplus Docker image +that is used for testing. + +In the **testing.yml** workflow file, there is a commented variable called +SET_MET_IMAGE that can be uncommented and set the MET Docker image to use. +This variable is found in the **get_image** job under the **env** section for the step named "Get METplus Image." +The format of the value is : +where the DockerHub repo used is dtcenter/ and the tag used is . + +Stable releases of MET are found in the +`dtcenter/met DockerHub repo `_ +and are named using the X.Y.Z version of the release, +so setting **SET_MET_IMAGE=met:11.1.0** will use dtcenter/met:11.1.0. + +Development versions of MET are found in the +`dtcenter/met-dev DockerHub repo `_ +and are named using the branch name, +so setting **SET_MET_IMAGE=met-dev:feature_XYZ_info** will use +dtcenter/met-dev:feature_XYZ_info. + .. _cg-ci-update-data-volumes: @@ -686,28 +708,140 @@ on DockerHub in *dtcenter/metplus-envs* and are named with a tag that corresponds to the keyword without the **_env** suffix. The environments were created using Docker commands via scripts that are found in *internal/scripts/docker_env*. -Existing keywords that set up Conda environments used for use cases are: - -* cfgrib_env -* h5py_env -* icecover_env -* metdatadb_env -* metplotpy_env -* netcdf4_env -* pygrib_env -* spacetime_env -* weatherregime_env -* xesmf_env +Existing keywords that set up Conda environments used for use cases, +with the versions of Python packages they contain are: + +**py_embed_base_env** + +* Python 3.10.4 +* xarray 2022.3.0 +* netcdf4 1.5.8 + +Note: Adding the py_embed_base_env keyword is not necessary if the *py_embed* +keyword is used (see Other Keywords). A Python Embedding use case that only +requires the minimum packages needed to run Python Embedding can use the +version of Python that MET was installed with. +The list of packages is only included here for reference, as other +environments use this environment as a base. + +**cfgrib_env** + +* Python 3.10.4 +* metpy 1.4.0 +* netcdf4 1.5.8 +* cfgrib 0.9.10.1 +* pygrib 2.1.4 + +**cycloneplotter_env** + +* Python 3.10.4 +* cartopy 0.20.3 +* matplotlib 3.5.2 +* pandas 1.4.3 + +**geovista_env** + +* geovista +* xarray 2022.11.0 +* iris 3.3.1 + +**h5py_env** + +* All packages in py_embed_base_env +* h5py 3.6.0 + +**icecover_env** + +* All packages in py_embed_base_env +* xarray 2022.3.0 +* pyresample 1.24.1 +* scikit-learn 1.1.1 +* pyproj 3.3.1 + +**metdataio_env** + +* Python 3.10.4 +* lxml 4.9.1 +* pymysql 1.0.2 +* pandas 1.5.1 + +**metplotpy_env** + +* Python 3.10.4 +* matplotlib 3.6.3 +* scipy 1.9.3 +* plotly 5.13.0 +* xarray 2023.1.0 +* netcdf4 1.6.2 +* pyyaml 6.0 +* python-kaleido 0.2.1 +* imageio 2.25.0 +* imutils 0.5.4 +* scikit-image +* pint 0.20.1 +* metpy +* cartopy 0.21.1 + +**netcdf4_env** + +* Python 3.10.4 +* netcdf4 1.5.8 + +**pandac_env** + +* All packages in metplotpy_env +* pygrib 2.1.4 + +**pygrib_env** + +* All packages in py_embed_base_env +* pygrib 2.1.4 +* metpy 1.3.0 + +**spacetime_env** + +* Python 3.10.4 +* netCDF4 1.5.8 +* xarray 2022.3.0 +* scipy 1.8.1 +* matplotlib 3.5.2 +* pyngl 1.6.1 +* pyyaml 6.0 + +**swpc_metpy_env** + +* All packages in py_embed_base_env +* metpy 1.4 + +**weatherregime_env** + +* All packages in py_embed_base_env +* scikit-learn 1.1.1 +* eofs 1.4.0 +* cmocean 2.0 + +**xesmf_env** + +* Python 3.10.4 +* netcdf4 1.5.8 +* xarray 2022.3.0 +* xesmf 0.3.0 + Example:: spacetime_env The above example uses the Conda environment -in *dtcenter/metplus-envs*:**spacetime** to run a user script. +in *dtcenter/metplus-envs*:**spacetime**.vX.Y to run a user script +where X.Y is the version of METplus when the environment was lasted updated, +e.g. 5.1. Note that only one dependency that contains the **_env** suffix can be supplied to a given use case. +If a new use case requires packages that are not included in these environments, +create a new discussion on the METplus Discussions board. + Other Environments """""""""""""""""" diff --git a/docs/Contributors_Guide/github_workflow.rst b/docs/Contributors_Guide/github_workflow.rst index 161e1c44d9..fd1451778d 100644 --- a/docs/Contributors_Guide/github_workflow.rst +++ b/docs/Contributors_Guide/github_workflow.rst @@ -482,6 +482,7 @@ Create a feature branch .. code-block:: ini + git fetch upstream develop git checkout develop * Verify the current development branch is active by running: @@ -674,70 +675,75 @@ Open a pull request then merge the files into the repository's develop branch. The develop branch will be used to create a future METplus release. -* In the browser, navigate to https://github.com//METplus - replacing - with the user's GitHub username and no angle brackets <>. +* If working from a fork, navigate to + https://github.com//METplus where ** is + the user's GitHub username. If working from a branch in the dtcenter + organization, navigate to https://github.com/dtcenter/METplus -* Click on the green 'Compare & pull request' button. +* Click the 'Pull Requests' tab and click the green "New pull request" button. - * A web page with four grey buttons should appear: +* If working from a fork, a web page with four grey buttons should appear: - * On the left-most button (for setting the base repository), - make sure the - 'base repository:dtcenter/METplus' is selected. + * On the left-most button (for setting the base repository), + make sure the 'base repository:dtcenter/METplus' is selected. - * For the base button, make sure to select 'base:develop'. + * For the base button, make sure to select 'base:develop'. - * For the head repository button, make sure to select - 'head repository:/METplus' - with the appropriate replacement for - . + * For the head repository button, make sure to select + 'head repository:/METplus' + with the appropriate replacement for + . - * For the compare button, make sure to select - 'compare:' - where corresponds to the feature branch - where the changes have been made (e.g. feature_777_wrapper_xyz). + * For the compare button, make sure to select + 'compare:' + where corresponds to the feature branch + where the changes have been made (e.g. feature_777_wrapper_xyz). - * In the 'write' window, follow the directions and fill in the template. - Add any additional comments/details. When filling in the template, - be sure to "Define the PR metadata, as permissions allow. - Select: **Reviewer(s), Project(s)**, and **Milestone**". When selecting a - reviewer, internal contributors submitting pull requests should select - the appropriate reviewer(s) and let the reviewer know that the pull - request has been assigned to them. If external contributors are unsure - who to assign as a reviewer, create a post in the - `METplus GitHub Discussions Forum `_ - asking for help with the assignment of a reviewer. +* If working from a branch in the dtcenter organization, there should be + two grey buttons. + + * For the **base** button, select *develop*. + * For the **compare** button, select the feature or bugfix branch. + +* In the 'write' window, follow the directions and fill in the template. + Add any additional comments/details. When filling in the template, + be sure to "Define the PR metadata, as permissions allow. + Select: **Reviewer(s), Project(s)**, and **Milestone**". When selecting a + reviewer, internal contributors submitting pull requests should select + the appropriate reviewer(s) and let the reviewer know that the pull + request has been assigned to them. If external contributors are unsure + who to assign as a reviewer, create a post in the + `METplus GitHub Discussions Forum `_ + asking for help with the assignment of a reviewer. - * When everything looks satisfactory, click on the green 'Create pull - request' button. +* When everything looks satisfactory, click on the green 'Create pull + request' button. - * Before a pull request is created, a "Project" can be selected, but there - is no option to select a cycle. +* Before a pull request is created, a "Project" can be selected, but there + is no option to select a cycle. - .. figure:: figure/1PR-before-created.png + .. figure:: figure/1PR-before-created.png - After the pull request is created, more options appear under the - "Project" section. + After the pull request is created, more options appear under the "Project" section. - .. figure:: figure/2PR-after-created.png + .. figure:: figure/2PR-after-created.png - Click the "Status" drop down and select "Review". + Click the "Status" drop down and select "Review". - .. figure:: figure/3PR-set-status.png + .. figure:: figure/3PR-set-status.png - Click on "+1 more" then under "Cycle", click "Choose an iteration" and - select the current development cycle. + Click on "+1 more" then under "Cycle", click "Choose an iteration" and + select the current development cycle. - .. figure:: figure/4PR-plus-one-set-cycle.png + .. figure:: figure/4PR-plus-one-set-cycle.png - To link the issue that correspondes to the pull request, click on the - gear next to "Development," type the issue number, then select the issue - that is displayed. + To link the issue that corresponds to the pull request, click on the + gear next to "Development," type the issue number, then select the issue + that is displayed. - * An authorized METplus developer will accept the pull request (if - everything meets acceptance criteria) and merge the code into the remote - repository's develop branch. +* An authorized METplus developer will accept the pull request (if + everything meets acceptance criteria) and merge the code into the remote + repository's develop branch. Approve a pull request using a browser ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -755,8 +761,8 @@ sub-tasks to be complete, then it may be best to wait to create a pull request until the other sub-tasks are also complete. -Creating a pull request -^^^^^^^^^^^^^^^^^^^^^^^ +Reviewing a pull request +^^^^^^^^^^^^^^^^^^^^^^^^ 1. Click on the “Pull requests” tab in the GitHub repository and click on the assigned pull request. diff --git a/docs/Contributors_Guide/testing.rst b/docs/Contributors_Guide/testing.rst index 1229359804..f0e10804d4 100644 --- a/docs/Contributors_Guide/testing.rst +++ b/docs/Contributors_Guide/testing.rst @@ -31,7 +31,7 @@ Running To run the unit tests, set the environment variable **METPLUS_TEST_OUTPUT_BASE** to a path where the user running has write -permissions, nativate to the METplus directory, then call pytest:: +permissions, navigate to the METplus directory, then call pytest:: export METPLUS_TEST_OUTPUT_BASE=/d1/personal/${USER}/pytest cd METplus @@ -66,30 +66,46 @@ To run only the GridStat unit tests:: Subsetting Tests by Marker ^^^^^^^^^^^^^^^^^^^^^^^^^^ -Unit tests can include one of the custom markers listed in the -internal/tests/pytests/pytest.ini file. Some examples include: - - * diff - * run_metplus - * util - * wrapper_a - * wrapper_b - * wrapper_c - * wrapper_d - * wrapper - * long - * plotting - -To apply a marker to a unit test function, add the following on the line before -the function definition:: +Pytest allows contributors to use markers on test functions. Markers are used +to set various features/attributes to test functions so that users can easily +exclude or include some tests from the test execution. Pytest provides many +inbuilt markers such as xfail, skip and parametrize. Apart from the inbuilt +marker names, users can create their own custom marker names. + +**Creating Pytest Markers** + +The METplus team has defined various custom marker names for the unit tests +and contributors can add to these markers. These custom markers are listed +with a brief description, in the *internal/tests/pytests/pytest.ini* file. +Some examples include: + + * diff (custom marker for diff util tests that require additional packages) + * run_metplus (custom marker for testing run_metplus.py script) + * util (custom marker for testing metplus/util logic) + * long (custom marker for tests that take a long time to run) + * plotting (custom marker for tests that involve plotting) + +To set up unit test functions so that they can be easily excluded or included +from text execution, contributors can add the following on the line before +the function definition in the Python test file:: + @pytest.mark. -where is one of the custom marker strings listed in pytest.ini. +where is one of the custom marker strings listed in the +*internal/tests/pytests/pytest.ini* file. -New pytest markers should be added to the pytest.ini file with a brief -description. If they are not added to the markers list, then a warning will -be output when running the tests. +To see an example of the “util” custom marker (@pytest.mark.util), +look in the file *internal/tests/pytests/util/time_util/test_time_util.py*. + +New pytest markers should be added to the *internal/tests/pytests/pytest.ini* +file with a brief description. If they are not added to the markers list, +then a warning will be output when running the tests. + +**Using Pytest Markers** + +The tests that have associated marker names can be now run to include +or exclude tests based on the given name. To run only tests with a given marker, run:: diff --git a/docs/Users_Guide/quicksearch.rst b/docs/Users_Guide/quicksearch.rst index 48303cdd57..15cf5dce50 100644 --- a/docs/Users_Guide/quicksearch.rst +++ b/docs/Users_Guide/quicksearch.rst @@ -151,7 +151,9 @@ Use Cases by METplus Feature: | `Looping by Month or Year <../search.html?q=LoopByMonthFeatureUseCase&check_keywords=yes&area=default>`_ | `List Expansion (using begin_end_incr syntax) <../search.html?q=ListExpansionFeatureUseCase&check_keywords=yes&area=default>`_ | `Masking for Regions of Interest <../search.html?q=MaskingFeatureUseCase&check_keywords=yes&area=default>`_ + | `METcalcpy <../search.html?q=METcalcpyUseCase&check_keywords=yes&area=default>`_ | `METdbLoad <../search.html?q=METdbLoadUseCase&check_keywords=yes&area=default>`_ + | `METplotpy <../search.html?q=METplotpyUseCase&check_keywords=yes&area=default>`_ | `MET_PYTHON_EXE Environment Variable <../search.html?q=MET_PYTHON_EXEUseCase&check_keywords=yes&area=default>`_ | `Multiple Conf File Use <../search.html?q=MultiConfUseCase&check_keywords=yes&area=default>`_ | `Observation Time Summary <../search.html?q=ObsTimeSummaryUseCase&check_keywords=yes&area=default>`_ @@ -180,7 +182,9 @@ Use Cases by METplus Feature: | **Looping by Month or Year**: *LoopByMonthFeatureUseCase* | **List Expansion (using begin_end_incr syntax)**: *ListExpansionFeatureUseCase* | **Masking for Regions of Interest**: *MaskingFeatureUseCase* + | **METcalcpy**: *METcalcpyUseCase* | **METdbLoad**: *METdbLoadUseCase* + | **METplotpy**: *METplotpyUseCase* | **MET_PYTHON_EXE Environment Variable**: *MET_PYTHON_EXEUseCase* | **Multiple Conf File Use**: *MultiConfUseCase* | **Observation Time Summary**: *ObsTimeSummaryUseCase* diff --git a/docs/Users_Guide/release-notes.rst b/docs/Users_Guide/release-notes.rst index 9c6825a9e4..afbdbf872a 100644 --- a/docs/Users_Guide/release-notes.rst +++ b/docs/Users_Guide/release-notes.rst @@ -30,7 +30,7 @@ When applicable, release notes are followed by the `GitHub issue `__ number which describes the bugfix, enhancement, or new feature. -METplus Version 5.1.0 Release Notes (2023-07-21) +METplus Version 5.1.0 Release Notes (2023-07-31) ------------------------------------------------ .. dropdown:: Enhancements @@ -42,7 +42,10 @@ METplus Version 5.1.0 Release Notes (2023-07-21) * Update use cases to use new Python directory structure in MET (`#2115 `_) * Add support for new multivariate MODE settings - (`#2197 `_) + (`#2197 `_, + `#2210 `_, + `#2230 `_, + `#2235 `_) .. dropdown:: Bugfix @@ -56,16 +59,10 @@ METplus Version 5.1.0 Release Notes (2023-07-21) (`#2137 `_) * Allow setting of convert, censor_thresh, and censor_val in regrid dictionary (`#2082 `_) - * METplus run errors if no commands were run - (`#2098 `_) * TCPairs setting -diag option causes failure (`#2179 `_) * Define the order of the forecast variables numerically rather than alphabetically (`#2070 `_) - * Allow setting of convert, censor_thresh, and censor_val in regrid dictionary - (`#2082 `_) - * Skip-if-output-exists logic incorrectly skips files - (`#2096 `_) * Prevent error if no commands were run because they were skipped (`#2098 `_) * Allow spaces for complex categorical thresholds @@ -74,6 +71,14 @@ METplus Version 5.1.0 Release Notes (2023-07-21) (`#2161 `_) * StatAnalysis time shifting failure (`#2168 `_) + * TCPairs: skip times logic incorrectly skips additional times + (`#2212 `_) + * TCStat fails if -out_stat directory does not exist + (`#2241 `_) + * Ensure log instances for concurrent runs are unique + (`#2245 `_) + * PointStat - Support running with no MASK provided + (`#1853 `_) .. dropdown:: New Wrappers @@ -85,11 +90,25 @@ METplus Version 5.1.0 Release Notes (2023-07-21) * Multi-Variate MODE (`#1516 `_) * Read in Argo profile data netCDF files for use in METplus with python embedding (`#1977 `_) + * PANDA-C: MPAS to SATCORPS + (`#2188 `_) + * PANDA-C: MPAS to MERRA2 + (`#2188 `_) + * PANDA-C: MPAS to ERA5 + (`#2188 `_) + * PANDA-C: GFS to SATCORPS + (`#2188 `_) + * PANDA-C: GFS to MERRA2 + (`#2188 `_) + * PANDA-C: GFS to ERA5 + (`#2188 `_) + * PointStat AMDAR PBLH with python embedding + (`#2198 `_) + * Space-time cross-spectra for S2S forecasts + (`#2136 `_) .. dropdown:: Documentation - * Update the METplus Components Python Requirements Documentation - (`#2016 `_) * Enhance the Release Notes by adding dropdown menus (`#2076 `_) * Update the METplus Components Python Requirements @@ -97,6 +116,8 @@ METplus Version 5.1.0 Release Notes (2023-07-21) `#2016 `_) * Add documentation on support for releases to the Release Guide (`#2106 `_) + * Add use case quick search keywords for METplotpy and METcalcpy + (`#2151 `_) .. dropdown:: Internal @@ -108,8 +129,6 @@ METplus Version 5.1.0 Release Notes (2023-07-21) (`#2015 `_) * **Upgrade to using Python 3.10.4** (`#2022 `_) - * Add 'License.txt' to the METplus repo - (`#2058 `_) * Add 'LICENSE.md' to the METplus repo (`#2058 `_) * Update Contributor's Guide to use GH Action to update truth data @@ -120,6 +139,15 @@ METplus Version 5.1.0 Release Notes (2023-07-21) (`#2159 `_) * Refactored code to resolve many SonarQube items (`#1610 `_) + * Improve Contributor's Guide + (`#2138 `_, + `#2207 `_) + * Bugfix: Fix difference test logic + (`#2244 `_) + * Remove base environment from Docker Conda images + (`#2249 `_) + * PR Templates (all METplus component repos): Improve language about linking relevant issue(s) + (`#2257 `_) METplus Version 5.0.0 Release Notes (2022-12-09) diff --git a/docs/Users_Guide/wrappers.rst b/docs/Users_Guide/wrappers.rst index b90bfcf5e3..5ddc6b018f 100644 --- a/docs/Users_Guide/wrappers.rst +++ b/docs/Users_Guide/wrappers.rst @@ -6181,7 +6181,7 @@ see :ref:`How METplus controls MET config file settings`. * - :term:`OBS_WINDOW_END` - obs_window.end -**${METPLUS_MASK_GRID}** +**${METPLUS_MASK_DICT}** .. list-table:: :widths: 5 5 @@ -6191,41 +6191,13 @@ see :ref:`How METplus controls MET config file settings`. - MET Config File * - :term:`POINT_STAT_MASK_GRID` - mask.grid - -**${METPLUS_MASK_POLY}** - -.. list-table:: - :widths: 5 5 - :header-rows: 0 - - * - METplus Config(s) - - MET Config File * - :term:`POINT_STAT_MASK_POLY` - mask.poly - -**${METPLUS_MASK_SID}** - -.. list-table:: - :widths: 5 5 - :header-rows: 0 - - * - METplus Config(s) - - MET Config File * - :term:`POINT_STAT_MASK_SID` - mask.sid - -**${METPLUS_MASK_LLPNT}** - -.. list-table:: - :widths: 5 5 - :header-rows: 0 - - * - METplus Config(s) - - MET Config File * - :term:`POINT_STAT_MASK_LLPNT` - mask.llpnt - **${METPLUS_OUTPUT_PREFIX}** .. list-table:: diff --git a/docs/conf.py b/docs/conf.py index 4c920764c4..35a1790c21 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -30,7 +30,6 @@ # To rotate this list of authors for each official release, # move the first author to the end of the list CURRENT_AUTHORS = [ - 'Julie Prestopnik', 'John Opatz', 'John Halley Gotway', 'Tara Jensen', @@ -42,6 +41,7 @@ 'Dan Adriaansen', 'Minna Win-Gildenmeister', 'George McCabe', + 'Julie Prestopnik', ] # list of former contributors who are no longer involved with the project diff --git a/docs/use_cases/met_tool_wrapper/TCDiag/TCDiag.py b/docs/use_cases/met_tool_wrapper/TCDiag/TCDiag.py index d525cbc8f8..7b2b1791ac 100644 --- a/docs/use_cases/met_tool_wrapper/TCDiag/TCDiag.py +++ b/docs/use_cases/met_tool_wrapper/TCDiag/TCDiag.py @@ -15,7 +15,7 @@ # tc_diag tool for a tropical cyclone forecast case and generating # intermediate NetCDF output files of the input model's data transformed # onto an azimuth-range grid. When the full functionality of the -# tc_diag tool is released in MET v12.0.0, this use case will be also +# tc_diag tool is released in MET v12.0.0, this use case will also # output environmental diagnostics computed from callable Python scripts. # # The diagnostics are computed on a range-azimuth grid that follows the @@ -67,7 +67,7 @@ # ------------------ # # This use case utilizes the METplus TCDiag wrapper to search for -# the desired ADECK file and forecast files that are correspond to the track. +# the desired ADECK file and forecast files that correspond to the track. # It generates a command to run tc_diag if all required files are found. ############################################################################## diff --git a/docs/use_cases/model_applications/medium_range/UserScript_fcstGEFS_Difficulty_Index.py b/docs/use_cases/model_applications/medium_range/UserScript_fcstGEFS_Difficulty_Index.py index 70ed34c635..f738570adf 100644 --- a/docs/use_cases/model_applications/medium_range/UserScript_fcstGEFS_Difficulty_Index.py +++ b/docs/use_cases/model_applications/medium_range/UserScript_fcstGEFS_Difficulty_Index.py @@ -185,6 +185,8 @@ # * UserScriptUseCase # * MediumRangeAppUseCase # * NRLOrgUseCase +# * METcalcpyUseCase +# * METplotpyUseCase # # Navigate to the :ref:`quick-search` page to discover other similar use cases. # diff --git a/docs/use_cases/model_applications/s2s/TCGen_fcstGFSO_obsBDECKS_GDF_TDF.py b/docs/use_cases/model_applications/s2s/TCGen_fcstGFSO_obsBDECKS_GDF_TDF.py index 4785bf3995..89625a4d52 100644 --- a/docs/use_cases/model_applications/s2s/TCGen_fcstGFSO_obsBDECKS_GDF_TDF.py +++ b/docs/use_cases/model_applications/s2s/TCGen_fcstGFSO_obsBDECKS_GDF_TDF.py @@ -212,7 +212,8 @@ # # * TCGenToolUseCase # * S2SAppUseCase -# * UserScriptUseCase +# * UserScriptUseCase +# * METplotpyUseCase # # Navigate to the :ref:`quick-search` page to discover other similar use cases. # diff --git a/docs/use_cases/model_applications/s2s/UserScript_fcstS2S_obsERAI_CrossSpectra.py b/docs/use_cases/model_applications/s2s/UserScript_fcstS2S_obsERAI_CrossSpectra.py index 14991f557c..8733d24c41 100644 --- a/docs/use_cases/model_applications/s2s/UserScript_fcstS2S_obsERAI_CrossSpectra.py +++ b/docs/use_cases/model_applications/s2s/UserScript_fcstS2S_obsERAI_CrossSpectra.py @@ -140,6 +140,8 @@ # # * UserScriptUseCase # * S2SAppUseCase +# * METcalcpyUseCase +# * METplotpyUseCase # # Navigate to the :ref:`quick-search` page to discover other similar use cases. # diff --git a/docs/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_Stratosphere.py b/docs/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_Stratosphere.py index eac201a110..9f3fda7684 100644 --- a/docs/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_Stratosphere.py +++ b/docs/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_Stratosphere.py @@ -126,6 +126,7 @@ # # * UserScriptUseCase # * S2SAppUseCase +# * METcalcpyUseCase # # Navigate to the :ref:`quick-search` page to discover other similar use cases. # diff --git a/docs/use_cases/model_applications/s2s/UserScript_obsPrecip_obsOnly_Hovmoeller.py b/docs/use_cases/model_applications/s2s/UserScript_obsPrecip_obsOnly_Hovmoeller.py index cbc5031ddf..2a9046a6ef 100644 --- a/docs/use_cases/model_applications/s2s/UserScript_obsPrecip_obsOnly_Hovmoeller.py +++ b/docs/use_cases/model_applications/s2s/UserScript_obsPrecip_obsOnly_Hovmoeller.py @@ -139,6 +139,7 @@ # # * UserScriptUseCase # * S2SAppUseCase +# * METplotpyUseCase # # Navigate to the :ref:`quick-search` page to discover other similar use cases. # diff --git a/docs/use_cases/model_applications/s2s_mid_lat/UserScript_obsERA_obsOnly_Blocking.py b/docs/use_cases/model_applications/s2s_mid_lat/UserScript_obsERA_obsOnly_Blocking.py index e93bbe4a59..68af4fa32b 100644 --- a/docs/use_cases/model_applications/s2s_mid_lat/UserScript_obsERA_obsOnly_Blocking.py +++ b/docs/use_cases/model_applications/s2s_mid_lat/UserScript_obsERA_obsOnly_Blocking.py @@ -181,6 +181,8 @@ # * S2SMidLatAppUseCase # * NetCDFFileUseCase # * GRIB2FileUseCase +# * METcalcpyUseCase +# * METplotpyUseCase # # Navigate to the :ref:`quick-search` page to discover other similar use cases. # diff --git a/docs/use_cases/model_applications/s2s_mid_lat/UserScript_obsERA_obsOnly_WeatherRegime.py b/docs/use_cases/model_applications/s2s_mid_lat/UserScript_obsERA_obsOnly_WeatherRegime.py index c605c27727..2502c32968 100644 --- a/docs/use_cases/model_applications/s2s_mid_lat/UserScript_obsERA_obsOnly_WeatherRegime.py +++ b/docs/use_cases/model_applications/s2s_mid_lat/UserScript_obsERA_obsOnly_WeatherRegime.py @@ -194,6 +194,8 @@ # * S2SMidLatAppUseCase # * NetCDFFileUseCase # * GRIB2FileUseCase +# * METcalcpyUseCase +# * METplotpyUseCase # # Navigate to the :ref:`quick-search` page to discover other similar use cases. # diff --git a/docs/use_cases/model_applications/s2s_mjo/UserScript_fcstGFS_obsERA_OMI.py b/docs/use_cases/model_applications/s2s_mjo/UserScript_fcstGFS_obsERA_OMI.py index 64cacbe08d..4ef6a1123a 100644 --- a/docs/use_cases/model_applications/s2s_mjo/UserScript_fcstGFS_obsERA_OMI.py +++ b/docs/use_cases/model_applications/s2s_mjo/UserScript_fcstGFS_obsERA_OMI.py @@ -135,6 +135,8 @@ # * S2SMJOAppUseCase # * RegridDataPlaneUseCase # * PCPCombineUseCase +# * METcalcpyUseCase +# * METplotpyUseCase # # Navigate to :ref:`quick-search` to discover other similar use cases. # diff --git a/docs/use_cases/model_applications/s2s_mjo/UserScript_obsCFSR_obsOnly_MJO_ENSO.py b/docs/use_cases/model_applications/s2s_mjo/UserScript_obsCFSR_obsOnly_MJO_ENSO.py index 7af5b8f966..ffe6a06e12 100644 --- a/docs/use_cases/model_applications/s2s_mjo/UserScript_obsCFSR_obsOnly_MJO_ENSO.py +++ b/docs/use_cases/model_applications/s2s_mjo/UserScript_obsCFSR_obsOnly_MJO_ENSO.py @@ -138,6 +138,8 @@ # * NetCDFFileUseCase # * RegridDataPlaneUseCase # * PCPCombineUseCase +# * METcalcpyUseCase +# * METplotpyUseCase # # Navigate to :ref:`quick-search` to discover other similar use cases. # diff --git a/docs/use_cases/model_applications/s2s_mjo/UserScript_obsERA_obsOnly_PhaseDiagram.py b/docs/use_cases/model_applications/s2s_mjo/UserScript_obsERA_obsOnly_PhaseDiagram.py index 7d27c127d0..9a07d4f8ec 100644 --- a/docs/use_cases/model_applications/s2s_mjo/UserScript_obsERA_obsOnly_PhaseDiagram.py +++ b/docs/use_cases/model_applications/s2s_mjo/UserScript_obsERA_obsOnly_PhaseDiagram.py @@ -134,6 +134,7 @@ # # * S2SAppUseCase # * S2SMJOAppUseCase +# * METplotpyUseCase # # Navigate to :ref:`quick-search` to discover other similar use cases. # diff --git a/docs/use_cases/model_applications/s2s_mjo/UserScript_obsERA_obsOnly_RMM.py b/docs/use_cases/model_applications/s2s_mjo/UserScript_obsERA_obsOnly_RMM.py index 15e2b7446f..5475735876 100644 --- a/docs/use_cases/model_applications/s2s_mjo/UserScript_obsERA_obsOnly_RMM.py +++ b/docs/use_cases/model_applications/s2s_mjo/UserScript_obsERA_obsOnly_RMM.py @@ -141,6 +141,8 @@ # * NetCDFFileUseCase # * RegridDataPlaneUseCase # * PCPCombineUseCase +# * METcalcpyUseCase +# * METplotpyUseCase # # Navigate to :ref:`quick-search` to discover other similar use cases. # diff --git a/docs/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_Planview.py b/docs/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_Planview.py index 78b79292d3..09c3176765 100644 --- a/docs/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_Planview.py +++ b/docs/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_Planview.py @@ -165,6 +165,7 @@ # * ValidationUseCase # * ShortRangeAppUseCase # * S2SAppUseCase +# * METplotpyUseCase # # Navigate to the :ref:`quick-search` page to discover other similar # use cases. diff --git a/docs/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalCrossSection.py b/docs/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalCrossSection.py index e876ad80e7..3a4036f197 100644 --- a/docs/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalCrossSection.py +++ b/docs/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalCrossSection.py @@ -156,6 +156,7 @@ # * ValidationUseCase # * ShortRangeAppUseCase # * S2SAppUseCase +# * METplotpyUseCase # # Navigate to the :ref:`quick-search` page to discover other similar use cases. # diff --git a/docs/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalProfile.py b/docs/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalProfile.py index 43b3b0db9b..33a4cb9f84 100644 --- a/docs/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalProfile.py +++ b/docs/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalProfile.py @@ -167,6 +167,7 @@ # * ValidationUseCase # * ShortRangeAppUseCase # * S2SAppUseCase +# * METplotpyUseCase # # Navigate to the :ref:`quick-search` page to discover other similar use cases. # diff --git a/internal/scripts/aws/EC2_README b/internal/scripts/aws/EC2_README new file mode 100644 index 0000000000..e7ff1649ab --- /dev/null +++ b/internal/scripts/aws/EC2_README @@ -0,0 +1,48 @@ +These instructions are for a debian install but in general as long as you install all of these packages, you should be fine + + +sudo apt install ant +sudo apt install apache2 +sudo apt install awscli +sudo apt install bzip2 +sudo apt install ca-certificates +sudo apt install cpio +sudo apt install curl +sudo apt install g++ +sudo apt install gcc +sudo apt install gfortran +sudo apt install ghostscript +sudo apt install git +sudo apt install gnupg +sudo apt install gzip +sudo apt install hdf5-helpers +sudo apt install gv +sudo apt install python3 +sudo apt install unzip +sudo apt install wget +sudo apt install + +sudo groupadd tomcat +sudo useradd -s /bin/false -g tomcat -d /opt/tomcat tomcat +curl -O https://dlcdn.apache.org/tomcat/tomcat-9/v9.0.56/bin/apache-tomcat-9.0.56.tar.gz +Install Tomcat in /opt with these instructions https://phoenixnap.com/kb/install-tomcat-ubuntu + +wget http://www.ftp.cpc.ncep.noaa.gov/wd51we/wgrib2/wgrib2.tgz +make install and put in /opt/bin using these instructions https://www.ftp.cpc.ncep.noaa.gov/wd51we/wgrib2/INSTALLING + +go to https://docs.conda.io/en/latest/miniconda.html#linux-installers +download the miniconda install you want +install miniconda + +Install goofys to mount an s3 bitbucket +http://www.whiteboardcoder.com/2017/12/install-and-set-up-goofys-in-ubuntu.html +sudo curl -O https://storage.googleapis.com/golang/go1.8.linux-amd64.tar.gz + > sudo tar -xvf go1.8.linux-amd64.tar.gz + > sudo mv go /usr/local + > sudo ln -s /usr/local/go/bin/go /usr/bin/go +export GOPATH=$HOME/work + > go get github.com/kahing/goofys + > go install github.com/kahing/goofys + > sudo cp work/bin/goofys /usr/bin/ + + diff --git a/internal/scripts/installation/metplus_components_v5.1_py3.10.sh b/internal/scripts/installation/metplus_components_v5.1_py3.10.sh new file mode 100644 index 0000000000..9ed5724cc9 --- /dev/null +++ b/internal/scripts/installation/metplus_components_v5.1_py3.10.sh @@ -0,0 +1,36 @@ +#! /bin/sh + +ENV_NAME=metplus_v5.1_py3.10 +MINICONDA_PATH=/path/to/miniconda3 + +${MINICONDA_PATH}/bin/conda create -y --name ${ENV_NAME} -c conda-forge python=3.10.4 +${MINICONDA_PATH}/bin/conda install -y --name ${ENV_NAME} -c conda-forge python-dateutil==2.8.2 +${MINICONDA_PATH}/bin/conda install -y --name ${ENV_NAME} -c conda-forge matplotlib==3.6.3 +${MINICONDA_PATH}/bin/conda install -y --name ${ENV_NAME} -c conda-forge scipy==1.9.3 +${MINICONDA_PATH}/bin/conda install -y --name ${ENV_NAME} -c conda-forge plotly==5.13.0 +${MINICONDA_PATH}/bin/conda install -y --name ${ENV_NAME} -c conda-forge xarray==2023.1.0 +${MINICONDA_PATH}/bin/conda install -y --name ${ENV_NAME} -c conda-forge netcdf4==1.6.2 +${MINICONDA_PATH}/bin/conda install -y --name ${ENV_NAME} -c conda-forge pyyaml==6.0 +${MINICONDA_PATH}/bin/conda install -y --name ${ENV_NAME} -c conda-forge statsmodels==0.13.2 +${MINICONDA_PATH}/bin/conda install -y --name ${ENV_NAME} -c conda-forge python-kaleido==0.2.1 +${MINICONDA_PATH}/bin/conda install -y --name ${ENV_NAME} -c conda-forge imageio==2.25.0 +${MINICONDA_PATH}/bin/conda install -y --name ${ENV_NAME} -c conda-forge imutils==0.5.4 +${MINICONDA_PATH}/bin/conda install -y --name ${ENV_NAME} -c conda-forge scikit-image==0.19.3 +${MINICONDA_PATH}/bin/conda install -y --name ${ENV_NAME} -c conda-forge pint==0.20.1 +${MINICONDA_PATH}/bin/conda install -y --name ${ENV_NAME} -c conda-forge metpy=1.4.0 +${MINICONDA_PATH}/bin/conda install -y --name ${ENV_NAME} -c conda-forge pyngl==1.6.1 +${MINICONDA_PATH}/bin/conda install -y --name ${ENV_NAME} -c conda-forge scikit-learn==1.2.1 +${MINICONDA_PATH}/bin/conda install -y --name ${ENV_NAME} -c conda-forge eofs==1.4.0 +${MINICONDA_PATH}/bin/conda install -y --name ${ENV_NAME} -c conda-forge cmocean==2.0 +${MINICONDA_PATH}/bin/conda install -y --name ${ENV_NAME} -c conda-forge xesmf==0.3.0 +${MINICONDA_PATH}/bin/conda install -y --name ${ENV_NAME} -c conda-forge lxml==4.9.1 +${MINICONDA_PATH}/bin/conda install -y --name ${ENV_NAME} -c conda-forge pymysql==1.0.2 +${MINICONDA_PATH}/bin/conda install -y --name ${ENV_NAME} -c conda-forge pandas==1.5.2 +${MINICONDA_PATH}/bin/conda install -y --name ${ENV_NAME} -c conda-forge h5py==3.6.0 +${MINICONDA_PATH}/bin/conda install -y --name ${ENV_NAME} -c conda-forge cartopy==0.20.3 +${MINICONDA_PATH}/bin/conda install -y --name ${ENV_NAME} -c conda-forge psutil==5.7.2 +${MINICONDA_PATH}/bin/conda install -y --name ${ENV_NAME} -c conda-forge pytest==7.2.1 +${MINICONDA_PATH}/bin/conda install -y --name ${ENV_NAME} -c conda-forge pytest-cov +${MINICONDA_PATH}/bin/conda install -y --name ${ENV_NAME} -c conda-forge numpy==1.24.2 +${MINICONDA_PATH}/bin/conda install -y --name ${ENV_NAME} -c conda-forge libstdcxx-ng==12.1.0 +${MINICONDA_PATH}/bin/conda install -y --name ${ENV_NAME} -c conda-forge opencv-python==4.7.0 diff --git a/internal/scripts/installation/modulefiles/5.1.0.lua_wcoss2 b/internal/scripts/installation/modulefiles/5.1.0.lua_wcoss2 new file mode 100644 index 0000000000..5a41532114 --- /dev/null +++ b/internal/scripts/installation/modulefiles/5.1.0.lua_wcoss2 @@ -0,0 +1,26 @@ +help([[ +]]) + +local pkgName = myModuleName() +local pkgVersion = myModuleVersion() +local pkgNameVer = myModuleFullName() + +local hierA = hierarchyA(pkgNameVer,1) +local compNameVer = hierA[1] + +conflict(pkgName) + +local opt = os.getenv("HPC_OPT") or os.getenv("OPT") or "/opt/modules" + +local base = pathJoin(opt,compNameVer,pkgName,pkgVersion) + +prepend_path("PATH", pathJoin(base,"ush")) + +setenv("METPLUS_ROOT", base) +setenv("METPLUS_VERSION", pkgVersion) +setenv("METPLUS_PATH", base) + +whatis("Name: ".. pkgName) +whatis("Version: " .. pkgVersion) +whatis("Category: application") +whatis("Description: Model Evaluation Tools Plus (METplus)") \ No newline at end of file diff --git a/internal/scripts/installation/modulefiles/5.1.0_acorn b/internal/scripts/installation/modulefiles/5.1.0_acorn new file mode 100644 index 0000000000..d5409853a9 --- /dev/null +++ b/internal/scripts/installation/modulefiles/5.1.0_acorn @@ -0,0 +1,22 @@ +#%Module###################################################################### +## +## METplus +## +proc ModulesHelp { } { + puts stderr "Sets up the paths and environment variables to use the METplus-5.1.0 + *** For help see the official MET webpage at http://www.dtcenter.org/met/users ***" +} + +module load intel python/3.10.4 +module use /apps/dev/modulefiles/ +module load ve/evs/1.0 + +module use /apps/sw_review/emc/MET/modulefiles +module load met/11.1.0 +module load nco +module load libjpeg +module load grib_util +module load wgrib2 + +setenv METPLUS_PATH /apps/sw_review/emc/METplus/METplus-5.1.0 +prepend-path PATH /apps/sw_review/emc/METplus/METplus-5.1.0/ush diff --git a/internal/scripts/installation/modulefiles/5.1.0_cheyenne b/internal/scripts/installation/modulefiles/5.1.0_cheyenne new file mode 100644 index 0000000000..4c204e2708 --- /dev/null +++ b/internal/scripts/installation/modulefiles/5.1.0_cheyenne @@ -0,0 +1,17 @@ +#%Module###################################################################### +## +## METplus +## +proc ModulesHelp { } { + puts stderr "Sets up the paths and environment variables to use the METplus-5.1.0 + *** For help see the official MET webpage at http://www.dtcenter.org/met/users ***" +} + +module use /glade/p/ral/jntp/MET/MET_releases/modulefiles +module load met/11.1.0 +module load nco +module load grib-bins/1.3 +module load R + +setenv METPLUS_PATH /glade/p/ral/jntp/MET/METplus/METplus-5.1.0 +prepend-path PATH /glade/p/ral/jntp/MET/METplus/METplus-5.1.0/ush:/glade/p/ral/jntp/MET/METplus/miniconda/miniconda3/envs/metplus_v5.1_py3.10/bin diff --git a/internal/scripts/installation/modulefiles/5.1.0_orion b/internal/scripts/installation/modulefiles/5.1.0_orion new file mode 100644 index 0000000000..58ea4803fa --- /dev/null +++ b/internal/scripts/installation/modulefiles/5.1.0_orion @@ -0,0 +1,25 @@ +#%Module###################################################################### +## +## METplus +## +proc ModulesHelp { } { + puts stderr "Sets up the paths and environment variables to use the METplus-5.1.0 + *** For help see the official MET webpage at http://www.dtcenter.org/met/users ***" +} +#%Module###################################################################### +## +## METplus +## +proc ModulesHelp { } { + puts stderr "Sets up the paths and environment variables to use the METplus-5.1.0 + *** For help see the official MET webpage at http://www.dtcenter.org/met/users ***" +} + +module load contrib +module load intel/2020.2 +module load met/11.1.0 +module load nco/4.8.1 +module load wgrib/2.0.8 +prepend-path PATH /work/noaa/ovp/miniconda/miniconda3/envs/metplus_v5.1_py3.10/bin +setenv METPLUS_PATH /apps/contrib/MET/METplus/METplus-5.1.0 +prepend-path PATH /apps/contrib/MET/METplus/METplus-5.1.0/ush diff --git a/internal/tests/pytests/conftest.py b/internal/tests/pytests/conftest.py index 968f96b736..47c7127c2c 100644 --- a/internal/tests/pytests/conftest.py +++ b/internal/tests/pytests/conftest.py @@ -6,6 +6,7 @@ import getpass import shutil from pathlib import Path +from netCDF4 import Dataset # add METplus directory to path so the wrappers and utilities can be found metplus_dir = str(Path(__file__).parents[3]) @@ -13,25 +14,48 @@ from metplus.util import config_metplus -output_base = os.environ.get('METPLUS_TEST_OUTPUT_BASE') +output_base = os.environ.get("METPLUS_TEST_OUTPUT_BASE") if not output_base: - print('ERROR: METPLUS_TEST_OUTPUT_BASE must be set to a path to write') + print("ERROR: METPLUS_TEST_OUTPUT_BASE must be set to a path to write") sys.exit(1) try: - test_output_dir = os.path.join(output_base, 'test_output') + test_output_dir = os.path.join(output_base, "test_output") if os.path.exists(test_output_dir): - print(f'Removing test output dir: {test_output_dir}') + print(f"Removing test output dir: {test_output_dir}") shutil.rmtree(test_output_dir) if not os.path.exists(test_output_dir): - print(f'Creating test output dir: {test_output_dir}') + print(f"Creating test output dir: {test_output_dir}") os.makedirs(test_output_dir) except PermissionError: - print(f'ERROR: Cannot write to $METPLUS_TEST_OUTPUT_BASE: {output_base}') + print(f"ERROR: Cannot write to $METPLUS_TEST_OUTPUT_BASE: {output_base}") sys.exit(2) +@pytest.fixture(scope="session", autouse=True) +def custom_tmpdir(): + """! Set the default output location for temp files produced by pytest + to METPLUS_TEST_OUTPUT_BASE env var. Causes default pytest fixtures, + such as tmpdir_factory, to write temp files to this dir. + + A sub directory named 'pytest_tmp' is created to ensure there will never + be name conflicts between files created by default pytest fixture and + other methods for writing temp files. + """ + original_var = os.environ.get("PYTEST_DEBUG_TEMPROOT", None) + output_base = os.environ.get("METPLUS_TEST_OUTPUT_BASE") + temp_base = os.path.join(output_base, "pytest_tmp") + + os.environ["PYTEST_DEBUG_TEMPROOT"] = temp_base + if not os.path.exists(temp_base): + os.mkdir(temp_base) + yield + # Restore original value, if existed + if original_var: + os.environ["PYTEST_DEBUG_TEMPROOT"] = original_var + + @pytest.hookimpl(tryfirst=True, hookwrapper=True) def pytest_runtest_makereport(item, call): """! This is used to capture the status of a test so the metplus_config @@ -58,19 +82,19 @@ def metplus_config(request): config = metplus_config. """ script_dir = os.path.dirname(__file__) - args = [os.path.join(script_dir, 'minimum_pytest.conf')] + args = [os.path.join(script_dir, "minimum_pytest.conf")] config = config_metplus.setup(args) yield config # don't remove output base if test fails if request.node.rep_call.failed: return - config_output_base = config.getdir('OUTPUT_BASE') + config_output_base = config.getdir("OUTPUT_BASE") if config_output_base and os.path.exists(config_output_base): shutil.rmtree(config_output_base) -@pytest.fixture(scope='function') +@pytest.fixture(scope="function") def metplus_config_files(): """! Create a METplus configuration object using minimum_pytest.conf settings and any list of config files.The metplus_config fixture is @@ -79,13 +103,56 @@ def metplus_config_files(): metplus_config_files as an argument to the test function and pass in a list of config files to it. Example: config = metplus_config_files([my_file]) """ + def read_configs(extra_configs): # Read in minimum pytest config file and any other extra configs script_dir = os.path.dirname(__file__) - minimum_conf = os.path.join(script_dir, 'minimum_pytest.conf') + minimum_conf = os.path.join(script_dir, "minimum_pytest.conf") args = extra_configs.copy() args.append(minimum_conf) config = config_metplus.setup(args) return config return read_configs + +@pytest.fixture(scope="module") +def make_dummy_nc(): + return make_nc + + +def make_nc(tmp_path, lon, lat, z, data, variable='Temp', file_name='fake.nc'): + """! Make a dummy netCDF file for use in tests. Populates a generic single + variable netcdf is dimension, lat, lon, z. + + @param tmp_path directory to write this netCDF to. + @param lon list of longitude values. + @param lat list of latitude values. + @param z list of pressure levels. + @param data array of values with dimesions (lat, lon, z) + @param variable (optional) string name of variable, defualt 'Temp' + @param file_name (optional) string name of file, defualt 'fake.nc' + + @returns path to netCDF file + """ + file_name = tmp_path / file_name + with Dataset(file_name, "w", format="NETCDF4") as rootgrp: + # Some tools (i.e. diff_util) can't deal with groups, + # so attach dimensions and variables to the root group. + rootgrp.createDimension("lon", len(lon)) + rootgrp.createDimension("lat", len(lat)) + rootgrp.createDimension("z", len(z)) + rootgrp.createDimension("time", None) + + # create variables + longitude = rootgrp.createVariable("Longitude", "f4", "lon") + latitude = rootgrp.createVariable("Latitude", "f4", "lat") + levels = rootgrp.createVariable("Levels", "i4", "z") + temp = rootgrp.createVariable(variable, "f4", ("time", "lon", "lat", "z")) + time = rootgrp.createVariable("Time", "i4", "time") + + longitude[:] = lon + latitude[:] = lat + levels[:] = z + temp[0, :, :, :] = data + + return file_name diff --git a/internal/tests/pytests/util/diff_util/test_diff_util.py b/internal/tests/pytests/util/diff_util/test_diff_util.py index 7ac2c837b4..9222ed3388 100644 --- a/internal/tests/pytests/util/diff_util/test_diff_util.py +++ b/internal/tests/pytests/util/diff_util/test_diff_util.py @@ -1,14 +1,15 @@ import pytest +from netCDF4 import Dataset import os import shutil -import uuid +from unittest import mock +from PIL import Image +import numpy as np -from metplus.util.diff_util import dirs_are_equal, ROUNDING_OVERRIDES +from metplus.util import diff_util as du from metplus.util import mkdir_p -test_output_dir = os.path.join(os.environ['METPLUS_TEST_OUTPUT_BASE'], - 'test_output') stat_header = 'VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE' mpr_line_1 = 'V11.1.0 HRRR ALL_1.25 120000 20220701_200000 20220701_200000 000000 20220701_200000 20220701_200000 HPBL m L0 HPBL m L0 ADPSFC DENVER BILIN 4 NA NA NA NA MPR 5 4 DENVER 39.78616 -104.41425 0 0 2160.80324 1498.06763 AMDAR NA NA NA' @@ -21,12 +22,47 @@ csv_val_2 = 'Kenny-Smith, Ambrose, 0.8977' -def create_diff_files(files_a, files_b): - unique_id = str(uuid.uuid4())[0:8] - dir_a = os.path.join(test_output_dir, f'diff_{unique_id}', 'a') - dir_b = os.path.join(test_output_dir, f'diff_{unique_id}', 'b') - mkdir_p(dir_a) - mkdir_p(dir_b) +DEFAULT_NC = [ + [359, 0, 1], # lon + [-1, 0, 1], # lat + [0, 1], # z + [ # data + [[1, 2], [3, 4], [5, 6]], + [[2, 3], [4, 5], [6, 7]], + [[30, 31], [33, 32], [34, 39]], + ], + "Temp", # variable +] + + +@pytest.fixture() +def dummy_nc1(tmp_path_factory, make_dummy_nc): + # Construct a temporary netCDF file + return make_dummy_nc( + tmp_path_factory.mktemp("data1"), + DEFAULT_NC[0], + DEFAULT_NC[1], + DEFAULT_NC[2], + DEFAULT_NC[3], + DEFAULT_NC[4], + # Note: "nc5" is not included in NETCDF_EXTENSIONS, hence + # we use it here to specifically trigger the call to + # netCDF.Dataset in get_file_type. + file_name= "fake.nc5" + ) + + +def _statment_in_capfd(capfd, check_print): + out, _ = capfd.readouterr() + print("out: ", out) + for statement in check_print: + assert statement in out + + +def create_diff_files(tmp_path_factory, files_a, files_b): + dir_a = tmp_path_factory.mktemp('dir_a') + dir_b = tmp_path_factory.mktemp('dir_b') + write_test_files(dir_a, files_a) write_test_files(dir_b, files_b) return dir_a, dir_b @@ -134,22 +170,417 @@ def write_test_files(dirname, files): ({'file_list.csv': [csv_header, csv_val_1, csv_val_2]}, {'file_list.csv': [csv_header, csv_val_1.replace('Mackenzie', 'Art'), csv_val_2]}, None, False), - ] + # csv diff trunc not equal round + ({'file_list.csv': [csv_header, csv_val_1, csv_val_2]}, + {'file_list.csv': [csv_header, csv_val_1.replace('0.9999', '1.0001'), csv_val_2,]}, + 3, True), + ], ) @pytest.mark.diff -def test_diff_dir_text_files(a_files, b_files, rounding_override, expected_is_equal): +def test_diff_dir_text_files(tmp_path_factory, a_files, b_files, rounding_override, expected_is_equal): if rounding_override: for filename in a_files: - ROUNDING_OVERRIDES[filename] = rounding_override + du.ROUNDING_OVERRIDES[filename] = rounding_override - a_dir, b_dir = create_diff_files(a_files, b_files) - assert dirs_are_equal(a_dir, b_dir) == expected_is_equal + a_dir, b_dir = create_diff_files(tmp_path_factory, a_files, b_files) + assert du.dirs_are_equal(str(a_dir), str(b_dir)) == expected_is_equal # pass individual files instead of entire directory for filename in a_files: if filename in b_files: a_path = os.path.join(a_dir, filename) b_path = os.path.join(b_dir, filename) - assert dirs_are_equal(a_path, b_path) == expected_is_equal + assert du.dirs_are_equal(a_path, b_path) == expected_is_equal + + +@pytest.mark.parametrize( + "path,expected", + [ + ("/path/to/file.csv", "csv"), + ("/path/to/file.jpeg", "image"), + ("/path/to/file.jpg", "image"), + ("/path/to/file.nc", "netcdf"), + ("/path/to/file.cdf", "netcdf"), + ("/path/to/file.pdf", "pdf"), + ("/path/to/file.zip", "skip .zip"), + ("/path/to/file.png", "image"), + ("/path/to/file.bigfoot", "unknown"), + ], +) +@pytest.mark.util +def test_get_file_type(path, expected): + actual = du.get_file_type(path) + assert actual == expected + - shutil.rmtree(os.path.dirname(a_dir)) +@pytest.mark.util +def test_get_file_type_netCDF4(dummy_nc1): + actual = du.get_file_type(dummy_nc1) + assert actual == 'netcdf' + + +@mock.patch.object(du, "UNSUPPORTED_EXTENSIONS", [".foo"]) +@pytest.mark.util +def test_get_file_type_unsupported(): + actual = du.get_file_type("/path/to/file.foo") + assert actual == "unsupported .foo" + + +@pytest.mark.util +def test_get_file_type_extensions(): + # Check all extensions are unique, otherwise we may + # get unexpected result from get_file_type + extensions = [ + du.IMAGE_EXTENSIONS, + du.NETCDF_EXTENSIONS, + du.SKIP_EXTENSIONS, + du.PDF_EXTENSIONS, + du.CSV_EXTENSIONS, + du.UNSUPPORTED_EXTENSIONS, + ] + flat_list = [ext for x in extensions for ext in x] + assert len(set(flat_list)) == len(flat_list) + + +@pytest.mark.parametrize( + "nc_data,fields,expected,check_print", + [ + ( + # Compare exact same data + [ + DEFAULT_NC[0], + DEFAULT_NC[1], + DEFAULT_NC[2], + DEFAULT_NC[3], + DEFAULT_NC[4], + ], + None, + True, + None, + ), + # Field name differ + ( + [ + DEFAULT_NC[0], + DEFAULT_NC[1], + DEFAULT_NC[2], + DEFAULT_NC[3], + "Foo", + ], + None, + False, + [ + "ERROR: Field list differs between files", + "File_A: ['Latitude', 'Levels', 'Longitude', 'Temp', 'Time']", + "File_B:['Foo', 'Latitude', 'Levels', 'Longitude', 'Time']", + ], + ), + # One small value change + ( + [ + DEFAULT_NC[0], + DEFAULT_NC[1], + DEFAULT_NC[2], + [ + [[1, 2], [3, 4], [5, 6]], + [[2, 3], [4, 5], [6, 7]], + [[30, 31], [33, 32], [34, 39.1]], + ], + DEFAULT_NC[4], + ], + None, + False, + [ + "ERROR: Field (Temp) values differ", + ], + ), + # Value changed but not comparing that field + ( + [ + DEFAULT_NC[0], + DEFAULT_NC[1], + DEFAULT_NC[2], + [ + [[1, 2], [3, 4], [5, 6]], + [[2, 3], [4, 5], [6, 7]], + [[30, 31], [33, 32], [34, 39.001]], + ], + DEFAULT_NC[4], + ], + ["Longitude", "Latitude", "Levels"], + True, + None, + ), + # Contains nan difference + ( + [ + DEFAULT_NC[0], + DEFAULT_NC[1], + DEFAULT_NC[2], + [ + [[1, 2], [3, 4], [5, 6]], + [[2, 3], [4, 5], [6, 7]], + [[30, 31], [33, 32], [34, np.nan]], + ], + DEFAULT_NC[4], + ], + None, + False, + ["Variable Temp contains NaN. Comparing each value"], + ), + # Field doesn't exist + ( + [ + DEFAULT_NC[0], + DEFAULT_NC[1], + DEFAULT_NC[2], + DEFAULT_NC[3], + DEFAULT_NC[4], + ], + "Bar", + False, + ["ERROR: Field Bar not found"], + ), + ], +) +@pytest.mark.util +def test_nc_is_equal( + capfd, tmp_path_factory, make_dummy_nc, dummy_nc1, nc_data, fields, expected, check_print +): + # make a dummy second file to compare to dummy_nc1 + dummy_nc2 = make_dummy_nc(tmp_path_factory.mktemp("data2"), *nc_data) + assert du.nc_is_equal(dummy_nc1, dummy_nc2, fields=fields, debug=True) == expected + + if check_print: + _statment_in_capfd(capfd, check_print) + +@pytest.mark.parametrize( + "nc_data,fields,expected,check_print", + [ + ( + [ + DEFAULT_NC[0], + DEFAULT_NC[1], + DEFAULT_NC[2], + [ + [[1, 2], [3, 4], [5, 6]], + [[2, 3], [4, 5], [6, 7]], + [[30, 31], [33, 32], [34, np.nan]], + ], + DEFAULT_NC[4], + ], + None, + True, + ["Variable Temp contains NaN. Comparing each value"], + ), + ] +) +@pytest.mark.util +def test_nc_is_equal_both_nan( + capfd, tmp_path_factory, make_dummy_nc, nc_data, fields, expected, check_print +): + dummy_nc = make_dummy_nc(tmp_path_factory.mktemp("data2"), *nc_data) + assert du.nc_is_equal(dummy_nc, dummy_nc, fields=fields, debug=True) == expected + + if check_print: + _statment_in_capfd(capfd, check_print) + + +@pytest.mark.parametrize( + "val,expected",[ + (np.float32(44.54), True), + (-0.15, True), + ("-123,456.5409", False), + ("2345j", False), + ("-12345.244", True), + ("foo", False) + ] +) +@pytest.mark.util +def test__is_number(val, expected): + assert du._is_number(val) == expected + + +@pytest.mark.parametrize( + 'func, args, patch_func, patch_return, expected', + [ + ( + du._handle_csv_files, + ['path/file1.csv', 'path/file2.csv'], + 'compare_csv_files', + True, + True, + ), + ( + du._handle_csv_files, + ['path/file1.csv', 'path/file2.csv'], + 'compare_csv_files', + False, + ('path/file1.csv', 'path/file2.csv', 'CSV diff', ''), + ), + ( + du._handle_netcdf_files, + ['path/file1.nc', 'path/file2.nc'], + 'nc_is_equal', + True, + True, + ), + ( + du._handle_netcdf_files, + ['path/file1.nc', 'path/file2.nc'], + 'nc_is_equal', + False, + ('path/file1.nc', 'path/file2.nc', 'NetCDF diff', ''), + ), + ( + du._handle_pdf_files, + ['path/file1.pdf', 'path/file2.pdf', True], + 'compare_pdf_as_images', + True, + True, + ), + ( + du._handle_pdf_files, + ['path/file1.pdf', 'path/file2.pdf', True], + 'compare_pdf_as_images', + False, + ('path/file1.pdf', 'path/file2.pdf', 'PDF diff', ''), + ), + ( + du._handle_image_files, + ['path/file1.png', 'path/file2.png', True], + 'compare_image_files', + True, + True, + ), + ( + du._handle_image_files, + ['path/file1.png', 'path/file2.png', True], + 'compare_image_files', + False, + ('path/file1.png', 'path/file2.png', 'Image diff', ''), + ), + ], +) +@pytest.mark.util +def test__handle_funcs(func, args, patch_func, patch_return, expected): + with mock.patch.object(du, patch_func, return_value=patch_return): + actual = func(*args) + assert actual == expected + + +@pytest.mark.parametrize( + 'cmp_return, comp_txt_return, expected', + [ + (True, True, True), + (False, True, True), + (False, False, ('file1.txt', 'file2.txt', 'Text diff', '')), + ], +) +@pytest.mark.util +def test__handle_text_files(cmp_return, comp_txt_return, expected): + with mock.patch.object(du.filecmp, 'cmp', return_value=cmp_return): + with mock.patch.object(du, 'compare_txt_files', return_value=comp_txt_return): + actual = du._handle_text_files( + 'file1.txt', 'file2.txt', '/dir_a/', '/dir_b/' + ) + assert actual == expected + + +@pytest.mark.parametrize( + 'colour_A, colour_B, save_diff, expected, check_print', + [ + ( + 255, + 255, + False, + True, + None + ), + ( + 255, + 253, + False, + False, + ['Difference pixel: (1, 1, 0)'], + ), + ( + 255, + 0, + True, + False, + ['Difference pixel: (254, 0, 0)'], + ), + ], +) +@pytest.mark.util +def test_compare_image_files( + capfd, tmp_path_factory, colour_A, colour_B, save_diff, expected, check_print +): + image_dir = tmp_path_factory.mktemp('images') + image1 = image_dir / 'img1.jpg' + image2 = image_dir / 'img2.jpg' + + expected_diff = os.path.join(image_dir, 'img2_diff.png') + + def _make_test_img(file_path, col): + im = Image.new('RGB', [1, 1], col) + im.save(file_path) + im.close() + + _make_test_img(image1, colour_A) + _make_test_img(image2, colour_B) + + actual = du.compare_image_files(image1, image2, save_diff) + + if save_diff: + assert actual == expected_diff + assert os.path.exists(actual) + else: + assert actual == expected + + # Just to check the diffs are correctly output + if check_print: + _statment_in_capfd(capfd, check_print) + + +@pytest.mark.parametrize( + 'array_a, array_b, expected, check_print', + [ + # basic test + ( + np.array([1, 2, 3.9]), + np.array([1, 2, 3.9]), + True, + None, + ), + # diff test + ( + np.array([1, 2, 3.9]), + np.array([1, 2, 4]), + False, + ["val_a: 3.9, val_b: 4"], + ), + # stored as strings + ( + '[1, 2, 3.9]', + '[1, 2, 3.9]', + True, + None, + ), + # multi dimentional with nan + ( + np.array([[1, 2, 3.9],[np.nan, 5, 6]]), + np.array([[1, 2, 3.9],[np.nan, 5, 6]]), + True, + None, + ), + ], +) +@pytest.mark.util +def test__all_values_are_equal(capfd, array_a, array_b, expected, check_print): + + actual = du._all_values_are_equal(array_a, array_b) + assert actual == expected + if check_print: + _statment_in_capfd(capfd, check_print) diff --git a/internal/tests/pytests/wrappers/point_stat/__init__.py b/internal/tests/pytests/wrappers/point_stat/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/internal/tests/pytests/wrappers/point_stat/point_stat_test_conus_sfc.conf b/internal/tests/pytests/wrappers/point_stat/point_stat_test_conus_sfc.conf deleted file mode 100644 index dac6370021..0000000000 --- a/internal/tests/pytests/wrappers/point_stat/point_stat_test_conus_sfc.conf +++ /dev/null @@ -1,138 +0,0 @@ -[dir] -# This is the location of your input files for METplus -PROJ_DIR = /path/to -TMP_DIR = /path/to -OUTPUT_BASE = /path/to -METPLUS_BASE = /path/to -MET_BUILD_BASE = /usr/local/met-8.0 -MET_INSTALL_DIR = /usr/local/met-8.0 -PARM_BASE = /path/to - -# Forecast and Obs file input directories (netCDF files) -FCST_INPUT_DIR = /d1/METplus_Mallory/data/gfs -OBS_INPUT_DIR = /d1/METplus_Mallory/output_for_testing/grid2obs_metplustest.2/prepbufr - -# Final output directory for point_stat output -POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE}/{OBTYPE} - -[config] -## LOOP_METHOD -## Options are: processes, times -## Looping by time- runs all items in the PROCESS_LIST for each -## initialization time and repeats until all times have been evaluated. -## Looping by processes- run each item in the PROCESS_LIST for all -## specified initialization times then repeat for the next item in the -## PROCESS_LIST. -LOOP_METHOD = processes - -# MET point_stat config file -#POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/Mallory_PointStatConfig_conus_sfc -POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/PointStatConfig_conus_sfc - -## in MET, the output files are named with tool and timing information to -## specify an output prefix string. If you wish to override this- assign a -## string to the OUTPUT_PREFIX value. This can be -## helpful when running multiple runs of the same tool. Otherwise, leave this -## empty. -OUTPUT_PREFIX = - -LOG_LEVEL = DEBUG -## Configuration-related settings such as the process list, begin and end times, etc. -PROCESS_LIST = PointStat - -# For processing by init time or valid time, indicate the start and end hours -# in HH format -START_HOUR = 00 -END_HOUR = 23 - -# Indicate the begin and end date -BEG_TIME = 20170601 -END_TIME = 20170603 - -# Start and end dates are created by combining the date with -# start and end hours (format can be hh, hhmm, or hhmmss. -START_DATE = {BEG_TIME}{START_HOUR} -END_DATE = {END_TIME}{END_HOUR} - -# Forecast hour start, end, and interval. Interval is the forecast interval in -# hours. -FCST_HR_START = 0 -FCST_HR_END = 120 -FCST_HR_INTERVAL = 24 - -# For both pb2nc and point_stat, the obs_window dictionary: -OBS_WINDOW_BEGIN = -2700 -OBS_WINDOW_END = 2700 - - -# Model/fcst and obs name, e.g. GFS, NAM, GDAS, etc. -MODEL = GFS -OBTYPE = NAM - -# Prefix to add to the MET point_stat output file: -# point_stat_PREFIX_HHMMSSL_YYYYMMDD_HHMMSSV.stat -# Leave empty if no prefix is requested -POINT_STAT_OUTPUT_PREFIX = - -# Regrid to specified grid. Indicate NONE if no regridding, or the grid id -# (e.g. G212) -REGRID_TO_GRID = G104 - -# Verification Masking regions -# Indicate which grid and polygon masking region, if applicable -POINT_STAT_GRID = FULL -# List of full path to poly masking files. NOTE: Only short lists of poly -# files work (those that fit on one line), a long list will result in an -# environment variable that is too long, resulting in an error. For long -# lists of poly masking files (i.e. all the mask files in the NCEP_mask -# directory), define these in the MET point_stat configuration file. -POINT_STAT_POLY = -POINT_STAT_STATION_ID = - -# Message types, if all message types are to be returned, leave this empty, -# otherwise indicate the message types of interest. -POINT_STAT_MESSAGE_TYPE = ONLYSF -# Variables and levels as specified in the field dictionary of the MET -# point_stat configuration file. Specify as FCST_VARn_NAME, -# FCST_VARn_LEVELS, (optional) FCST_VARn_OPTION - -FCST_VAR1_NAME = TMP -FCST_VAR1_LEVELS = Z2 - -FCST_VAR2_NAME = RH -FCST_VAR2_LEVELS = Z2 - -FCST_VAR3_NAME = DPT -FCST_VAR3_LEVELS = Z2 - -FCST_VAR4_NAME = UGRD -FCST_VAR4_LEVELS = Z10 - -FCST_VAR5_NAME = VGRD -FCST_VAR5_LEVELS = Z10 - -FCST_VAR6_NAME = TCDC -FCST_VAR6_LEVELS = L0 -FCST_VAR6_OPTIONS = GRIB_lvl_typ = 200 - -FCST_VAR7_NAME = PRMSL -FCST_VAR7_LEVELS = Z0 - - -# REGEX PATTERNS -[regex_pattern] -# Regular expressions for files relevant to this use case. Use parentheses -# around any date, cycle and offset time. -OBS_INPUT_FILE_REGEX = .*prepbufr.nam.(2[0-9]{7}).t([0-9]{2})z.tm([0-9]{2}).nc -OBS_INPUT_DIR_REGEX = -FCST_INPUT_FILE_REGEX = .*pgbf([0-9]{1,3}).gfs.(2[0-9]{9}) -FCST_INPUT_DIR_REGEX = - - -# -# FILENAME TEMPLATES -# -[filename_templates] -# NOTE: These are EXAMPLE FILENAME TEMPLATES -# - diff --git a/internal/tests/pytests/wrappers/point_stat/point_stat_test_upper_air.conf b/internal/tests/pytests/wrappers/point_stat/point_stat_test_upper_air.conf deleted file mode 100644 index 87464d992c..0000000000 --- a/internal/tests/pytests/wrappers/point_stat/point_stat_test_upper_air.conf +++ /dev/null @@ -1,128 +0,0 @@ -[dir] -## Input data directories - -# This is the location of your input files for METplus -PROJ_DIR = /path/to -TMP_DIR = /path/to -OUTPUT_BASE = /path/to -METPLUS_BASE = /path/to -MET_BUILD_BASE = /usr/local/met-8.0 -MET_INSTALL_DIR = /usr/local/met-8.0 -PARM_BASE = /path/to - -# Forecast and Obs file input directories (netCDF files) -FCST_INPUT_DIR = /d1/METplus_Mallory/data/gfs -OBS_INPUT_DIR = /d1/METplus_Mallory/output_for_testing/grid2obs_metplustest.2/prepbufr - -# Final output directory for point_stat output -POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE}/{MODEL} - -[config] -## LOOP_METHOD -## Options are: processes, times -## Looping by time- runs all items in the PROCESS_LIST for each -## initialization time and repeats until all times have been evaluated. -## Looping by processes- run each item in the PROCESS_LIST for all -## specified initialization times then repeat for the next item in the -## PROCESS_LIST. -LOOP_METHOD = processes - -# MET point_stat config file -POINT_STAT_CONFIG_FILE = {PARM_BASE}/met_config/PointStatConfig_upper_air - -## in MET, the output files are named with tool and timing information to -## specify an output prefix string. If you wish to override this- assign a -## string to the OUTPUT_PREFIX value. This can be -## helpful when running multiple runs of the same tool. Otherwise, leave this -## empty. -OUTPUT_PREFIX = - -LOG_LEVEL = DEBUG -## Configuration-related settings such as the process list, begin and end times, etc. -PROCESS_LIST = PointStat - -# For processing by init time or valid time, indicate the start and end hours -# in HH format -START_HOUR = 00 -END_HOUR = 23 - -# Indicate the begin and end date -BEG_TIME = 20170601 -END_TIME = 20170603 - -# Start and end dates are created by combining the date with -# start and end hours (format can be hh, hhmm, or hhmmss. -START_DATE = {BEG_TIME}{START_HOUR} -END_DATE = {END_TIME}{END_HOUR} - -# Forecast hour start, end, and interval. Interval is the forecast interval in -# hours. -FCST_HR_START = 0 -FCST_HR_END = 120 -FCST_HR_INTERVAL = 24 - -# For both pb2nc and point_stat, the obs_window dictionary: -OBS_WINDOW_BEGIN = -2700 -OBS_WINDOW_END = 2700 - -# Model/fcst and obs name, e.g. nam, gdas, etc. -MODEL = gfs -OBTYPE = gdas - -# Prefix to add to the MET point_stat output file: -# point_stat_PREFIX_HHMMSSL_YYYYMMDD_HHMMSSV.stat -# Leave empty if no prefix is requested -POINT_STAT_OUTPUT_PREFIX = - -# Regrid to specified grid. Indicate NONE if no regridding, or the grid id -# (e.g. G212) -REGRID_TO_GRID = G003 - -# Verification masking regions for point stat, different from pb2nc in that -# these are lists of values. Can be left empty. -POINT_STAT_GRID = FULL -# List of full path to poly masking files. NOTE: Only short lists of poly -# files work (those that fit on one line), a long list will result in an -# environment variable that is too long, resulting in an error. For long -# lists of poly masking files (i.e. all the mask files in the NCEP_mask -# directory), define these in the MET point_stat configuration file. -POINT_STAT_POLY = -POINT_STAT_STATION_ID = - -# a list of message types e.g. ADPUPA, ADPSFC. Leave empty to retain all message types. -POINT_STAT_MESSAGE_TYPE = ADPUPA - -# Variables and levels as specified in the field dictionary of the MET -# point_stat configuration file. - -FCST_VAR1_NAME = TMP -FCST_VAR1_LEVELS = P1000, P925, P850, P700, P500, P400, P300, P250, P200, P150, P100, P50, P20, P10 - -FCST_VAR2_NAME = RH -FCST_VAR2_LEVELS = P1000, P925, P850, P700, P500, P400, P300 - -FCST_VAR3_NAME = UGRD -FCST_VAR3_LEVELS = P1000, P925, P850, P700, P500, P400, P300, P250, P200, P150, P100, P50, P20, P10 - -FCST_VAR4_NAME = VGRD -FCST_VAR4_LEVELS = P1000, P925, P850, P700, P500, P400, P300, P250, P200, P150, P100, P50, P20, P10 - -FCST_VAR5_NAME = HGT -FCST_VAR5_LEVELS = P1000, P950, P925, P850, P700, P500, P400, P300, P250, P200, P150, P100, P50, P20, P10 - - -# REGEX PATTERNS -[regex_pattern] -# Regular expressions for files relevant to this use case. Use parentheses -# around any date, cycle and offset time. -OBS_INPUT_DIR_REGEX = -FCST_INPUT_DIR_REGEX = - -# -# FILENAME TEMPLATES -# -[filename_templates] -# NOTE: These are EXAMPLE FILENAME TEMPLATES -# -OBS_INPUT_FILE_TMPL = prepbufr.gdas.{valid?fmt=%Y%m%d%H}.nc -FCST_INPUT_FILE_TMPL = pgbf{lead?fmt=%H}.gfs.{valid?fmt=%Y%m%d%H} diff --git a/internal/tests/pytests/wrappers/point_stat/refactor_point_stat_test_conus_sfc.conf b/internal/tests/pytests/wrappers/point_stat/refactor_point_stat_test_conus_sfc.conf deleted file mode 100644 index 830fdd1db6..0000000000 --- a/internal/tests/pytests/wrappers/point_stat/refactor_point_stat_test_conus_sfc.conf +++ /dev/null @@ -1,111 +0,0 @@ -[exe] -# NON-MET executables -WGRIB2 = /usr/local/bin/wgrib2 -RM_EXE = /bin/rm -CUT_EXE = /usr/bin/cut -TR_EXE = /usr/bin/tr -NCAP2_EXE =/usr/local/nco/bin/ncap2 -CONVERT_EXE =/usr/bin/convert -NCDUMP_EXE = /usr/local/bin/ncdump -EGREP_EXE = /bin/egrep - -[dir] -## Input data directories - -# This is the location of your input files for METplus -PROJ_DIR = /d1/minnawin/wip -TMP_DIR = /d1/minnawin/wip/tmp -OUTPUT_BASE = /d1/minnawin/point_stat_crow_test -METPLUS_BASE = /home/minnawin/wip/METplus -MET_BUILD_BASE = /usr/local/met-6.1 -MET_INSTALL_DIR = /usr/local/met-6.1 -PARM_BASE = /home/minnawin/wip/METplus/parm - -# Forecast and Obs file input directories (netCDF files) -FCST_INPUT_DIR = /d1/METplus_Mallory/data/gfs -OBS_INPUT_DIR = /d1/minnawin/pb2nc_crow_test/nam/conus_sfc - -# Final output directory for point_stat output -POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE}/{OBTYPE} - -[config] -## LOOP_METHOD -## Options are: processes, times -## Looping by time- runs all items in the PROCESS_LIST for each -## initialization time and repeats until all times have been evaluated. -## Looping by processes- run each item in the PROCESS_LIST for all -## specified initialization times then repeat for the next item in the -## PROCESS_LIST. -LOOP_METHOD = processes - -# MET point_stat config file -POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/Mallory_PointStatConfig_conus_sfc - -## in MET, the output files are named with tool and timing information to -## specify an output prefix string. If you wish to override this- assign a -## string to the OUTPUT_PREFIX value. This can be -## helpful when running multiple runs of the same tool. Otherwise, leave this -## empty. -OUTPUT_PREFIX = - -LOG_LEVEL = DEBUG -## Configuration-related settings such as the process list, begin and end times, etc. -PROCESS_LIST = PointStat - -# For processing by init time or valid time, indicate the start and end hours -# in HH format -START_HOUR = 00 -END_HOUR = 23 - -# Indicate the begin and end date -BEG_TIME = 20170601 -END_TIME = 20170630 - -# Start and end dates are created by combining the date with -# start and end hours (format can be hh, hhmm, or hhmmss. -START_DATE = {BEG_TIME}{START_HOUR} -END_DATE = {END_TIME}{END_HOUR} - -# Forecast hour start, end, and interval. Interval is the forecast interval in -# hours. -FCST_HR_START_DATE = 0 -FCST_HR_END_DATE = 130 -FCST_HR_INTERVAL = 24 - -# Model/fcst and obs name, e.g. GFS, NAM, GDAS, etc. -MODEL = gfs -OBTYPE = nam - -# Prefix to add to the MET point_stat output file: -# point_stat_PREFIX_HHMMSSL_YYYYMMDD_HHMMSSV.stat -# Leave empty if no prefix is requested -POINT_STAT_OUTPUT_PREFIX = - -# Indicate which grid and polygon masking region, if applicable -GRID_MASK = G003 -#MASK_POLY = APL, ATC, CAM, CAR, CONUS, EAST, ECA, GLF, GMC, GRB, HWI, LMY, -#MDW, MEX, NAK, NAO, NEC, NMT, NPL, NSA, NWC, PRI, SAO, SEC, SMT, SPL, SWC, -#SWD, WCA, WEST -MASK_POLY = CONUS - -# TODO replace this with updated syntax which supports options and other ways -# of defining fields and levels. -# Variables and levels as specified in the field dictionary of the MET -# point_stat configuration file. Specify as FCST_VARn_NAME, FCST_VARn_LEVELS, -# (optional) FCST_VARn_OPTION where the le -# VAR_LIST = TMP/Z2, RH/Z2, DPT/Z2, UGRD/Z10, VGRD/Z10, PRMSL/Z0 - -# REGEX PATTERNS -[regex_pattern] -# Regular expressions for files relevant to this use case. Use parentheses -# around any date, cycle and offset time. -OBS_INPUT_FILE_REGEX = .*prepbufr.nam.(2[0-9]{7}).t([0-9]{2})z.tm([0-9]{2}).nc -FCST_INPUT_FILE_REGEX = .*pgbf([0-9]{1,3}).gfs.(2[0-9]{9}) - -# -# FILENAME TEMPLATES -# -[filename_templates] -# NOTE: These are EXAMPLE FILENAME TEMPLATES -# - diff --git a/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py b/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py index 0a5f9b8df1..8dcf879b14 100755 --- a/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py +++ b/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py @@ -20,6 +20,7 @@ valid = valid.strftime(time_fmt) valids.append(valid) + def set_minimum_config_settings(config): # set config variables to prevent command from running and bypass check # if input files actually exist @@ -129,6 +130,7 @@ def test_met_dictionary_in_var_options(metplus_config): }, {'METPLUS_MASK_GRID': 'grid = ["FULL"];', 'METPLUS_MASK_POLY': 'poly = ["one", "two"];', + 'METPLUS_MASK_DICT': 'mask = {grid = ["FULL"];poly = ["one", "two"];}', }), # mask grid and poly (new config var) ({'POINT_STAT_MASK_GRID': 'FULL', @@ -136,12 +138,12 @@ def test_met_dictionary_in_var_options(metplus_config): }, {'METPLUS_MASK_GRID': 'grid = ["FULL"];', 'METPLUS_MASK_POLY': 'poly = ["one", "two"];', + 'METPLUS_MASK_DICT': 'mask = {grid = ["FULL"];poly = ["one", "two"];}', }), # mask grid value - ({'POINT_STAT_MASK_GRID': 'FULL', - }, - {'METPLUS_MASK_GRID': - 'grid = ["FULL"];', + ({'POINT_STAT_MASK_GRID': 'FULL', }, + {'METPLUS_MASK_GRID': 'grid = ["FULL"];', + 'METPLUS_MASK_DICT': 'mask = {grid = ["FULL"];}', }), # mask.poly complex example ({'POINT_STAT_MASK_POLY': ('["{ENV[MET_BUILD_BASE]}/share/met/poly/CAR.poly", ' @@ -154,30 +156,31 @@ def test_met_dictionary_in_var_options(metplus_config): '"{ENV[MET_BUILD_BASE]}/share/met/poly/GLF.poly", ' '"{ENV[MET_BUILD_BASE]}/share/met/poly/NAO.poly", ' '"{ENV[MET_BUILD_BASE]}/share/met/poly/SAO.poly"];', + 'METPLUS_MASK_DICT': + 'mask = {poly = ["{ENV[MET_BUILD_BASE]}/share/met/poly/CAR.poly", ' + '"{ENV[MET_BUILD_BASE]}/share/met/poly/GLF.poly", ' + '"{ENV[MET_BUILD_BASE]}/share/met/poly/NAO.poly", ' + '"{ENV[MET_BUILD_BASE]}/share/met/poly/SAO.poly"];}', }), # mask grid empty string (should create empty list) - ({'POINT_STAT_MASK_GRID': '', - }, - {'METPLUS_MASK_GRID': - 'grid = [];', + ({'POINT_STAT_MASK_GRID': '', }, + {'METPLUS_MASK_GRID': 'grid = [];', + 'METPLUS_MASK_DICT': 'mask = {grid = [];}', }), # mask poly (old config var) - ({'POINT_STAT_VERIFICATION_MASK_TEMPLATE': 'one, two', - }, - {'METPLUS_MASK_POLY': - 'poly = ["one", "two"];', + ({'POINT_STAT_VERIFICATION_MASK_TEMPLATE': 'one, two', }, + {'METPLUS_MASK_POLY': 'poly = ["one", "two"];', + 'METPLUS_MASK_DICT': 'mask = {poly = ["one", "two"];}', }), # mask poly (new config var) - ({'POINT_STAT_MASK_POLY': 'one, two', - }, - {'METPLUS_MASK_POLY': - 'poly = ["one", "two"];', + ({'POINT_STAT_MASK_POLY': 'one, two', }, + {'METPLUS_MASK_POLY': 'poly = ["one", "two"];', + 'METPLUS_MASK_DICT': 'mask = {poly = ["one", "two"];}', }), - ({'POINT_STAT_MASK_SID': 'one, two', - }, - {'METPLUS_MASK_SID': - 'sid = ["one", "two"];', + ({'POINT_STAT_MASK_SID': 'one, two', }, + {'METPLUS_MASK_SID': 'sid = ["one", "two"];', + 'METPLUS_MASK_DICT': 'mask = {sid = ["one", "two"];}', }), ({'POINT_STAT_OUTPUT_PREFIX': 'my_output_prefix'}, @@ -189,10 +192,8 @@ def test_met_dictionary_in_var_options(metplus_config): ({'OBS_POINT_STAT_WINDOW_BEGIN': '-2700', 'OBS_POINT_STAT_WINDOW_END': '2700', }, - {'METPLUS_OBS_WINDOW_DICT': - 'obs_window = {beg = -2700;end = 2700;}', - 'OBS_WINDOW_BEGIN': '-2700', - 'OBS_WINDOW_END': '2700' + {'METPLUS_OBS_WINDOW_DICT': 'obs_window = {beg = -2700;end = 2700;}', + 'OBS_WINDOW_BEGIN': '-2700', 'OBS_WINDOW_END': '2700' }), # test that {app}_OBS_WINDOW are preferred over # OBS_{app}_WINDOW and generic OBS_WINDOW @@ -203,8 +204,7 @@ def test_met_dictionary_in_var_options(metplus_config): 'OBS_WINDOW_BEGIN': '-900', 'OBS_WINDOW_END': '900', }, - {'METPLUS_OBS_WINDOW_DICT': - 'obs_window = {beg = -1800;end = 1800;}', + {'METPLUS_OBS_WINDOW_DICT': 'obs_window = {beg = -1800;end = 1800;}', }), ({'POINT_STAT_CLIMO_CDF_CDF_BINS': '1', }, @@ -485,7 +485,8 @@ def test_met_dictionary_in_var_options(metplus_config): {'METPLUS_HSS_EC_VALUE': 'hss_ec_value = 0.5;'}), ({'POINT_STAT_MASK_LLPNT': ('{ name = "LAT30TO40"; lat_thresh = >=30&&<=40; lon_thresh = NA; },' '{ name = "BOX"; lat_thresh = >=20&&<=40; lon_thresh = >=-110&&<=-90; }')}, - {'METPLUS_MASK_LLPNT': 'llpnt = [{ name = "LAT30TO40"; lat_thresh = >=30&&<=40; lon_thresh = NA; }, { name = "BOX"; lat_thresh = >=20&&<=40; lon_thresh = >=-110&&<=-90; }];'}), + {'METPLUS_MASK_LLPNT': 'llpnt = [{ name = "LAT30TO40"; lat_thresh = >=30&&<=40; lon_thresh = NA; }, { name = "BOX"; lat_thresh = >=20&&<=40; lon_thresh = >=-110&&<=-90; }];', + 'METPLUS_MASK_DICT': 'mask = {llpnt = [{ name = "LAT30TO40"; lat_thresh = >=30&&<=40; lon_thresh = NA; }, { name = "BOX"; lat_thresh = >=20&&<=40; lon_thresh = >=-110&&<=-90; }];}'}), ({'POINT_STAT_HIRA_FLAG': 'False', }, {'METPLUS_HIRA_DICT': 'hira = {flag = FALSE;}'}), @@ -531,6 +532,64 @@ def test_met_dictionary_in_var_options(metplus_config): ({'POINT_STAT_OBS_VALID_END': '{valid?fmt=%Y%m%d_%H?shift=6H}', }, {}), ({'POINT_STAT_OBS_VALID_BEG': '{valid?fmt=%Y%m%d_%H?shift=-6H}', 'POINT_STAT_OBS_VALID_END': '{valid?fmt=%Y%m%d_%H?shift=6H}'}, {}), + # complex mask example + ({'POINT_STAT_MASK_GRID': 'FULL', + 'POINT_STAT_MASK_POLY': ('["{ENV[MET_BUILD_BASE]}/share/met/poly/CAR.poly", ' + '"{ENV[MET_BUILD_BASE]}/share/met/poly/GLF.poly", ' + '"{ENV[MET_BUILD_BASE]}/share/met/poly/NAO.poly", ' + '"{ENV[MET_BUILD_BASE]}/share/met/poly/SAO.poly" ];'), + 'POINT_STAT_MASK_SID': 'one, two', + 'POINT_STAT_MASK_LLPNT': ( + '{ name = "LAT30TO40"; lat_thresh = >=30&&<=40; lon_thresh = NA; },' + '{ name = "BOX"; lat_thresh = >=20&&<=40; lon_thresh = >=-110&&<=-90; }')}, + {'METPLUS_MASK_DICT': ( + 'mask = {grid = ["FULL"];' + 'poly = ["{ENV[MET_BUILD_BASE]}/share/met/poly/CAR.poly", ' + '"{ENV[MET_BUILD_BASE]}/share/met/poly/GLF.poly", ' + '"{ENV[MET_BUILD_BASE]}/share/met/poly/NAO.poly", ' + '"{ENV[MET_BUILD_BASE]}/share/met/poly/SAO.poly"];' + 'sid = ["one", "two"];' + 'llpnt = [{ name = "LAT30TO40"; lat_thresh = >=30&&<=40; lon_thresh = NA; }, { name = "BOX"; lat_thresh = >=20&&<=40; lon_thresh = >=-110&&<=-90; }];}' + ), + 'METPLUS_MASK_LLPNT': 'llpnt = [{ name = "LAT30TO40"; lat_thresh = >=30&&<=40; lon_thresh = NA; }, { name = "BOX"; lat_thresh = >=20&&<=40; lon_thresh = >=-110&&<=-90; }];', + 'METPLUS_MASK_POLY': + 'poly = ["{ENV[MET_BUILD_BASE]}/share/met/poly/CAR.poly", ' + '"{ENV[MET_BUILD_BASE]}/share/met/poly/GLF.poly", ' + '"{ENV[MET_BUILD_BASE]}/share/met/poly/NAO.poly", ' + '"{ENV[MET_BUILD_BASE]}/share/met/poly/SAO.poly"];', + 'METPLUS_MASK_GRID': 'grid = ["FULL"];', + 'METPLUS_MASK_SID': 'sid = ["one", "two"];', + }), + # complex mask example, empty grid value + ({'POINT_STAT_MASK_GRID': '', + 'POINT_STAT_MASK_POLY': ( + '["{ENV[MET_BUILD_BASE]}/share/met/poly/CAR.poly", ' + '"{ENV[MET_BUILD_BASE]}/share/met/poly/GLF.poly", ' + '"{ENV[MET_BUILD_BASE]}/share/met/poly/NAO.poly", ' + '"{ENV[MET_BUILD_BASE]}/share/met/poly/SAO.poly" ];'), + 'POINT_STAT_MASK_SID': 'one, two', + 'POINT_STAT_MASK_LLPNT': ( + '{ name = "LAT30TO40"; lat_thresh = >=30&&<=40; lon_thresh = NA; },' + '{ name = "BOX"; lat_thresh = >=20&&<=40; lon_thresh = >=-110&&<=-90; }')}, + {'METPLUS_MASK_DICT': ( + 'mask = {grid = [];' + 'poly = ["{ENV[MET_BUILD_BASE]}/share/met/poly/CAR.poly", ' + '"{ENV[MET_BUILD_BASE]}/share/met/poly/GLF.poly", ' + '"{ENV[MET_BUILD_BASE]}/share/met/poly/NAO.poly", ' + '"{ENV[MET_BUILD_BASE]}/share/met/poly/SAO.poly"];' + 'sid = ["one", "two"];' + 'llpnt = [{ name = "LAT30TO40"; lat_thresh = >=30&&<=40; lon_thresh = NA; }, { name = "BOX"; lat_thresh = >=20&&<=40; lon_thresh = >=-110&&<=-90; }];}' + ), + 'METPLUS_MASK_LLPNT': 'llpnt = [{ name = "LAT30TO40"; lat_thresh = >=30&&<=40; lon_thresh = NA; }, { name = "BOX"; lat_thresh = >=20&&<=40; lon_thresh = >=-110&&<=-90; }];', + 'METPLUS_MASK_POLY': + 'poly = ["{ENV[MET_BUILD_BASE]}/share/met/poly/CAR.poly", ' + '"{ENV[MET_BUILD_BASE]}/share/met/poly/GLF.poly", ' + '"{ENV[MET_BUILD_BASE]}/share/met/poly/NAO.poly", ' + '"{ENV[MET_BUILD_BASE]}/share/met/poly/SAO.poly"];', + 'METPLUS_MASK_GRID': 'grid = [];', + 'METPLUS_MASK_SID': 'sid = ["one", "two"];', + }), + ] ) @pytest.mark.wrapper_a diff --git a/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py b/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py index 7f0b7e14db..b1f7599dbf 100644 --- a/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py +++ b/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py @@ -356,7 +356,28 @@ def set_minimum_config_settings(config): 'SERIES_ANALYSIS_MASK_POLY': 'MET_BASE/poly/EAST.poly', }, {'METPLUS_MASK_DICT': 'mask = {grid = "FULL";poly = "MET_BASE/poly/EAST.poly";}'}), - + # check tags are resolved and animation config works + ({ + 'FCST_VAR1_LEVELS': 'A0{init?fmt=3}', + 'SERIES_ANALYSIS_GENERATE_PLOTS': 'True', + 'SERIES_ANALYSIS_GENERATE_ANIMATIONS': 'True', + 'CONVERT_EXE': 'animation_exe' + }, + {},), + # check 'BOTH_*' and '*INPUT_FILE_LIST' config + ({'SERIES_ANALYSIS_REGRID_TO_GRID': 'FCST', + 'BOTH_SERIES_ANALYSIS_INPUT_TEMPLATE': 'True', + }, + {'METPLUS_REGRID_DICT': 'regrid = {to_grid = FCST;}'}), + ({'SERIES_ANALYSIS_REGRID_TO_GRID': 'FCST', + 'BOTH_SERIES_ANALYSIS_INPUT_FILE_LIST': 'True', + }, + {'METPLUS_REGRID_DICT': 'regrid = {to_grid = FCST;}'}), + ({'SERIES_ANALYSIS_REGRID_TO_GRID': 'FCST', + 'FCST_SERIES_ANALYSIS_INPUT_FILE_LIST': 'True', + 'OBS_SERIES_ANALYSIS_INPUT_FILE_LIST': 'True', + }, + {'METPLUS_REGRID_DICT': 'regrid = {to_grid = FCST;}'}), ] ) @pytest.mark.wrapper_a @@ -374,6 +395,8 @@ def test_series_analysis_single_field(metplus_config, config_overrides, wrapper = SeriesAnalysisWrapper(config) assert wrapper.isOK + is_both = wrapper.c_dict.get('USING_BOTH') + app_path = os.path.join(config.getdir('MET_BIN_DIR'), wrapper.app_name) verbosity = f"-v {wrapper.c_dict['VERBOSITY']}" @@ -383,14 +406,21 @@ def test_series_analysis_single_field(metplus_config, config_overrides, suffix = '_init_20050807000000_valid_ALL_lead_ALL.txt' fcst_file = f'{prefix}fcst{suffix}' obs_file = f'{prefix}obs{suffix}' - expected_cmds = [(f"{app_path} " + + if is_both: + expected_cmds = [(f"{app_path} " + f"-both {out_dir}/{fcst_file} " + f"-out {out_dir}/2005080700 " + f"-config {config_file} {verbosity}"), + ] + else: + expected_cmds = [(f"{app_path} " f"-fcst {out_dir}/{fcst_file} " f"-obs {out_dir}/{obs_file} " f"-out {out_dir}/2005080700 " f"-config {config_file} {verbosity}"), ] - all_cmds = wrapper.run_all_times() print(f"ALL COMMANDS: {all_cmds}") @@ -899,20 +929,56 @@ def test_get_output_dir(metplus_config, template, storm_id, label, expected_resu assert(actual_result == os.path.join(output_dir, expected_result)) +@pytest.mark.parametrize( + 'data,expected_min,expected_max,variable_name', [ + ( [ + [[1, 2], [3, 4], [5, 6]], + [[2, 3], [4, 5], [6, 7]], + [[30, 31], [33, 32], [34, 39]], + ], + 1, + 39, + 'Temp' + ), + ( + [ + [[1, 1], [1, 1], [1, 1]] + ], + 1, + 1, + 'Temp' + ), + ( + [ + [[1, 1], [1, 1], [1, 1]] + ], + None, + None, + 'Foo' + ), + ] + +) @pytest.mark.wrapper_a -def test_get_netcdf_min_max(metplus_config): - pytest.skip('Rewrite this test to write a NetCDF file and check vals instead of using file in met install dir') - expected_min = 0.0 - expected_max = 8.0 - +def test_get_netcdf_min_max(tmp_path_factory, + metplus_config, + make_dummy_nc, + data, + expected_min, + expected_max, + variable_name): + + filepath = make_dummy_nc( + tmp_path_factory.mktemp("data1"), + [359, 0, 1], + [-1, 0, 1], + [0, 1], + data, + "Temp" + ) + wrapper = series_analysis_wrapper(metplus_config) - met_install_dir = wrapper.config.getdir('MET_INSTALL_DIR') - filepath = os.path.join(met_install_dir, - 'share', - 'met', - 'tc_data', - 'basin_global_tenth_degree.nc') - variable_name = 'basin' + min, max = wrapper._get_netcdf_min_max(filepath, variable_name) assert min == expected_min assert max == expected_max diff --git a/metplus/RELEASE_DATE b/metplus/RELEASE_DATE index 69447068db..06f50e8cbf 100644 --- a/metplus/RELEASE_DATE +++ b/metplus/RELEASE_DATE @@ -1 +1 @@ -20221209 \ No newline at end of file +20230731 \ No newline at end of file diff --git a/metplus/VERSION b/metplus/VERSION index ae4be6b042..bc0aed6c89 100644 --- a/metplus/VERSION +++ b/metplus/VERSION @@ -1 +1 @@ -5.1.0-dev \ No newline at end of file +6.0.0-beta1-dev \ No newline at end of file diff --git a/metplus/util/diff_util.py b/metplus/util/diff_util.py index 4278206c6d..a2124f4742 100755 --- a/metplus/util/diff_util.py +++ b/metplus/util/diff_util.py @@ -5,6 +5,7 @@ import netCDF4 import filecmp import csv +from numbers import Number from PIL import Image, ImageChops from pandas import isnull from numpy.ma import is_masked @@ -43,6 +44,11 @@ # PBL use case can be removed after dtcenter/METplus#2246 is completed SKIP_KEYWORDS = [ 'PointStat_fcstHRRR_obsAMDAR_PBLH_PyEmbed', + 'CyclonePlotter/cyclone/20150301.png', + 'plots/obs_elbow.png', + 'plots/fcst_elbow.png', + 'CyclonePlotter_fcstGFS_obsGFS_UserScript_ExtraTC/cyclone/20201007', + 'plots/MAKE_MAKI_timeseries', ] @@ -94,7 +100,7 @@ def get_file_type(filepath): return 'pdf' if file_extension in UNSUPPORTED_EXTENSIONS: - return f'unsupported{file_extension}' + return f'unsupported {file_extension}' return 'unknown' @@ -478,6 +484,8 @@ def _is_equal_rounded(value_a, value_b): def _is_number(value): + if isinstance(value, Number): + return True return value.replace('.', '1').replace('-', '1').strip().isdigit() diff --git a/metplus/wrappers/command_builder.py b/metplus/wrappers/command_builder.py index 1adc35a01f..893562d069 100755 --- a/metplus/wrappers/command_builder.py +++ b/metplus/wrappers/command_builder.py @@ -1553,12 +1553,13 @@ def handle_time_summary_dict(self): [f'{app_upper}_TIME_SUMMARY_VALID_THRESH']), }) - def handle_mask(self, single_value=False, get_flags=False): + def handle_mask(self, single_value=False, get_flags=False, get_point=False): """! Read mask dictionary values and set them into env_var_list @param single_value if True, only a single value for grid and poly are allowed. If False, they should be treated as as list @param get_flags if True, read grid_flag and poly_flag values + @param get_point if True, read sid and llpnt values """ data_type = 'string' if single_value else 'list' app_upper = self.app_name.upper() @@ -1574,6 +1575,10 @@ def handle_mask(self, single_value=False, get_flags=False): items['grid_flag'] = ('string', 'remove_quotes,uppercase') items['poly_flag'] = ('string', 'remove_quotes,uppercase') + if get_point: + items['sid'] = 'list' + items['llpnt'] = ('list', 'remove_quotes') + self.add_met_config_dict('mask', items) def add_met_config_dict(self, dict_name, items): diff --git a/metplus/wrappers/point_stat_wrapper.py b/metplus/wrappers/point_stat_wrapper.py index 2ca1137ae7..a5848b3613 100755 --- a/metplus/wrappers/point_stat_wrapper.py +++ b/metplus/wrappers/point_stat_wrapper.py @@ -28,10 +28,7 @@ class PointStatWrapper(CompareGriddedWrapper): 'METPLUS_OBS_FIELD', 'METPLUS_MESSAGE_TYPE', 'METPLUS_OBS_WINDOW_DICT', - 'METPLUS_MASK_GRID', - 'METPLUS_MASK_POLY', - 'METPLUS_MASK_SID', - 'METPLUS_MASK_LLPNT', + 'METPLUS_MASK_DICT', 'METPLUS_OUTPUT_PREFIX', 'METPLUS_CLIMO_CDF_DICT', 'METPLUS_OBS_QUALITY_INC', @@ -56,6 +53,10 @@ class PointStatWrapper(CompareGriddedWrapper): 'POINT_STAT_GRID', 'POINT_STAT_STATION_ID', 'POINT_STAT_MESSAGE_TYPE', + 'METPLUS_MASK_GRID', # deprecated in v5.1.0 + 'METPLUS_MASK_POLY', # deprecated in v5.1.0 + 'METPLUS_MASK_SID', # deprecated in v5.1.0 + 'METPLUS_MASK_LLPNT', # deprecated in v5.1.0 ] OUTPUT_FLAGS = [ @@ -154,6 +155,9 @@ def create_c_dict(self): self.add_met_config_window('obs_window') self.handle_obs_window_legacy(c_dict) + self.handle_mask(get_point=True) + + # handle legacy mask dictionary variables self.add_met_config(name='grid', data_type='list', env_var_name='METPLUS_MASK_GRID', @@ -183,6 +187,7 @@ def create_c_dict(self): metplus_configs=['POINT_STAT_MASK_LLPNT'], extra_args={'allow_empty': True, 'remove_quotes': True}) + # end of handling legacy mask dictionary variables self.add_met_config(name='message_type', data_type='list') diff --git a/parm/met_config/PointStatConfig_wrapped b/parm/met_config/PointStatConfig_wrapped index a65ff43b3a..fb9fe5e906 100644 --- a/parm/met_config/PointStatConfig_wrapped +++ b/parm/met_config/PointStatConfig_wrapped @@ -113,13 +113,8 @@ ${METPLUS_OBS_WINDOW_DICT} // // Verification masking regions // -mask = { - ${METPLUS_MASK_GRID} - ${METPLUS_MASK_POLY} - ${METPLUS_MASK_SID} - //llpnt = - ${METPLUS_MASK_LLPNT} -} +//mask = { +${METPLUS_MASK_DICT} //////////////////////////////////////////////////////////////////////////////// diff --git a/parm/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsNDBC_WaveHeight.conf b/parm/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsNDBC_WaveHeight.conf index fb9dd20544..f09d28b90c 100644 --- a/parm/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsNDBC_WaveHeight.conf +++ b/parm/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsNDBC_WaveHeight.conf @@ -135,3 +135,5 @@ POINT_STAT_MASK_GRID = FULL POINT_STAT_MASK_POLY = POINT_STAT_MASK_SID = +[user_env_vars] +MET_NDBC_STATIONS = {INPUT_BASE}/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsNDBC_WaveHeight/ndbc_stations.20220928.xml diff --git a/requirements.txt b/requirements.txt index e6c17278dd..3ec037fa46 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -certifi==2022.12.7 +certifi==2023.7.22 python-dateutil==2.8.2 six==1.16.0