diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 6c4450e814..118348bafe 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -65,6 +65,7 @@ Branch name: `bugfix__main__` - [ ] Add/update log messages for easier debugging. - [ ] Add/update unit tests. - [ ] Add/update documentation. +- [ ] Add any new Python packages to the [METplus Components Python Requirements](https://metplus.readthedocs.io/en/develop/Users_Guide/overview.html#metplus-components-python-requirements) table. - [ ] Push local changes to GitHub. - [ ] Submit a pull request to merge into **main_\**. Pull request: `bugfix main_ ` diff --git a/.github/ISSUE_TEMPLATE/enhancement_request.md b/.github/ISSUE_TEMPLATE/enhancement_request.md index 3133dec892..85a64e9da3 100644 --- a/.github/ISSUE_TEMPLATE/enhancement_request.md +++ b/.github/ISSUE_TEMPLATE/enhancement_request.md @@ -54,6 +54,7 @@ Branch name: `feature__` - [ ] Add/update log messages for easier debugging. - [ ] Add/update unit tests. - [ ] Add/update documentation. +- [ ] Add any new Python packages to the [METplus Components Python Requirements](https://metplus.readthedocs.io/en/develop/Users_Guide/overview.html#metplus-components-python-requirements) table. - [ ] Push local changes to GitHub. - [ ] Submit a pull request to merge into **develop**. Pull request: `feature ` diff --git a/.github/ISSUE_TEMPLATE/new_feature_request.md b/.github/ISSUE_TEMPLATE/new_feature_request.md index b37efe8aef..1fe07f63dd 100644 --- a/.github/ISSUE_TEMPLATE/new_feature_request.md +++ b/.github/ISSUE_TEMPLATE/new_feature_request.md @@ -58,6 +58,7 @@ Branch name: `feature__` - [ ] Add/update log messages for easier debugging. - [ ] Add/update unit tests. - [ ] Add/update documentation. +- [ ] Add any new Python packages to the [METplus Components Python Requirements](https://metplus.readthedocs.io/en/develop/Users_Guide/overview.html#metplus-components-python-requirements) table. - [ ] Push local changes to GitHub. - [ ] Submit a pull request to merge into **develop**. Pull request: `feature ` diff --git a/.github/ISSUE_TEMPLATE/new_use_case.md b/.github/ISSUE_TEMPLATE/new_use_case.md index d16e19960f..6d8be0973e 100644 --- a/.github/ISSUE_TEMPLATE/new_use_case.md +++ b/.github/ISSUE_TEMPLATE/new_use_case.md @@ -67,6 +67,7 @@ Branch name: `feature__` - [ ] Add/update log messages for easier debugging. - [ ] Add/update unit tests. - [ ] Add/update documentation. +- [ ] Add any new Python packages to the [METplus Components Python Requirements](https://metplus.readthedocs.io/en/develop/Users_Guide/overview.html#metplus-components-python-requirements) table. - [ ] Push local changes to GitHub. - [ ] Submit a pull request to merge into **develop**. Pull request: `feature ` diff --git a/.github/ISSUE_TEMPLATE/task.md b/.github/ISSUE_TEMPLATE/task.md index ce3a6fb09a..24aa7eaf91 100644 --- a/.github/ISSUE_TEMPLATE/task.md +++ b/.github/ISSUE_TEMPLATE/task.md @@ -54,6 +54,7 @@ Branch name: `feature__` - [ ] Add/update log messages for easier debugging. - [ ] Add/update unit tests. - [ ] Add/update documentation. +- [ ] Add any new Python packages to the [METplus Components Python Requirements](https://metplus.readthedocs.io/en/develop/Users_Guide/overview.html#metplus-components-python-requirements) table. - [ ] Push local changes to GitHub. - [ ] Submit a pull request to merge into **develop**. Pull request: `feature ` diff --git a/.github/jobs/docker_setup.sh b/.github/jobs/docker_setup.sh index 46e1e61e84..f5c923dad9 100755 --- a/.github/jobs/docker_setup.sh +++ b/.github/jobs/docker_setup.sh @@ -36,11 +36,17 @@ MET_TAG=`${GITHUB_WORKSPACE}/scripts/docker/hooks/get_met_version` MET_DOCKER_REPO=met-dev if [ "${MET_TAG}" != "develop" ]; then MET_DOCKER_REPO=met +elif [ "${EXTERNAL_TRIGGER}" == "true" ]; then + # if MET tag is develop and external repo triggered workflow + # then append -lite to MET tag name to use tag generated by + # MET GHA workflow that does not include MET unit test tools + MET_TAG=${MET_TAG}-lite fi # if MET_FORCE_TAG variable is set and not empty, use that version instead if [ ! -z "$MET_FORCE_TAG" ]; then MET_TAG=$MET_FORCE_TAG + MET_DOCKER_REPO=met fi echo Using MET_DOCKER_REPO=$MET_DOCKER_REPO diff --git a/.github/jobs/get_use_cases_to_run.sh b/.github/jobs/get_use_cases_to_run.sh index bfd2991038..39c250474c 100755 --- a/.github/jobs/get_use_cases_to_run.sh +++ b/.github/jobs/get_use_cases_to_run.sh @@ -9,7 +9,7 @@ run_all_use_cases=$2 run_unit_tests=$3 echo Run use cases: $run_use_cases -echo Run All use cases: $run_all_use_cases +echo Run all use cases: $run_all_use_cases echo Run unit tests: $run_unit_tests # if running use cases, generate JQ filter to use diff --git a/.github/jobs/set_job_controls.sh b/.github/jobs/set_job_controls.sh index 007adb50eb..e8015992c1 100755 --- a/.github/jobs/set_job_controls.sh +++ b/.github/jobs/set_job_controls.sh @@ -94,3 +94,11 @@ echo ::set-output name=branch_name::$branch_name # get use cases to run .github/jobs/get_use_cases_to_run.sh $run_use_cases $run_all_use_cases $run_unit_tests + +# echo output variables to review in logs +echo branch_name: $branch_name +echo run_diff: $run_diff +echo run_save_truth_data: $run_save_truth_data +echo external_trigger: $external_trigger +echo run_get_image: $run_get_image +echo run_get_input_data: $run_get_input_data diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index 025054ccc9..372caf3779 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -61,12 +61,12 @@ }, { "category": "marine_and_cryosphere", - "index_list": "3-4", + "index_list": "3-5", "run": false }, { "category": "marine_and_cryosphere", - "index_list": "5", + "index_list": "6", "run": false }, { diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index c94d25f2a8..1e30a18207 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -15,6 +15,7 @@ If **yes**, describe the new output and/or changes to the existing output:
## Pull Request Checklist ## See the [METplus Workflow](https://metplus.readthedocs.io/en/latest/Contributors_Guide/github_workflow.html) for details. +- [ ] Add any new Python packages to the [METplus Components Python Requirements](https://metplus.readthedocs.io/en/develop/Users_Guide/overview.html#metplus-components-python-requirements) table. - [ ] Review the source issue metadata (required labels, projects, and milestone). - [ ] Complete the PR definition above. - [ ] Ensure the PR title matches the feature or bugfix branch name. diff --git a/.github/workflows/release_published.yml b/.github/workflows/release_published.yml deleted file mode 100644 index 8a7ce83ae2..0000000000 --- a/.github/workflows/release_published.yml +++ /dev/null @@ -1,15 +0,0 @@ -name: Release Published -on: - release: - types: [published] - -jobs: - send-message: - name: Send Slack Message for New Release - runs-on: ubuntu-latest - steps: - - name: Run cURL Command - run: > - curl -X POST -H 'Content-type: application/json' --data - '{"text":"METplus ${{ github.event.release.tag_name }} release was created: <${{ github.event.release.html_url }}>"}' - ${{ secrets.SLACK_WEBHOOK }} diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 858fb6fbe2..1b47909874 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -84,6 +84,7 @@ jobs: env: DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + EXTERNAL_TRIGGER: ${{ needs.job_control.outputs.external_trigger }} #MET_FORCE_TAG: 10.0.0 update_data_volumes: diff --git a/README.md b/README.md index 65d62d2072..10f4a11a5f 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ METplus is a Python scripting infrastructure around the MET verification tools Documentation for both Users and Contributors can be found [here](https://metplus.readthedocs.io). For more information about the entire suite of tools, please visit the -[MET website](https://dtcenter.org/community-code/model-evaluation-tools-met). +[DTC METplus website](https://dtcenter.org/community-code/metplus). Support for the METplus components is provided through the [METplus Discussions](https://github.com/dtcenter/METplus/discussions) forum. diff --git a/build_components/Externals_stable.cfg b/build_components/Externals_stable.cfg index afc09b80e4..123ebeaff8 100644 --- a/build_components/Externals_stable.cfg +++ b/build_components/Externals_stable.cfg @@ -3,35 +3,35 @@ local_path = ../MET protocol = git required = True repo_url = https://github.com/dtcenter/MET -branch = v10.0.0 +tag = v10.1.0 [METviewer] local_path = ../METviewer protocol = git required = True repo_url = https://github.com/dtcenter/METviewer -branch = v4.0.0 +tag = v4.1.0 [METplotpy] local_path = ../METplotpy protocol = git required = True repo_url = https://github.com/dtcenter/METplotpy -branch = v1.0.0 +tag = v1.1.0 [METcalcpy] local_path = ../METcalcpy protocol = git required = True repo_url = https://github.com/dtcenter/METcalcpy -branch = v1.0.0 +tag = v1.1.0 [METdatadb] local_path = ../METdatadb protocol = git required = True repo_url = https://github.com/dtcenter/METdatadb -branch = develop +tag = v1.1.0 [externals_description] diff --git a/docs/Contributors_Guide/add_use_case.rst b/docs/Contributors_Guide/add_use_case.rst index fcd423fbcb..8d02d21e3d 100644 --- a/docs/Contributors_Guide/add_use_case.rst +++ b/docs/Contributors_Guide/add_use_case.rst @@ -210,15 +210,14 @@ use case OR category directory for a model_applications use case * Update the description of the input data in the Datasets section. * Update the list of External Dependencies (if applicable) to include any - required Python packages. Update the - `METplus Components Python Requirements `_ - spreadsheet. If the package is already listed in the spreadsheet, add + required Python packages. Update the :ref:`python_requirements_table` + table. If the package is already listed in the spreadsheet, add a link to the documentation page for this new use case, following the - format in the spreadsheet. If the package is not already listed, update - the spreadsheet to include the name of the required package, the version, + format in the table. If the package is not already listed, update + the table to include the name of the required package, the version, the METplus component (e.g. METplus wrappers, METcalcpy, METplotpy), the - source, a brief description, and a link to this new use case that uses - this new Python package. + source, a brief description of the package, and a link to this new use + case that uses this new Python package. * Update the list of tools used in the METplus Components section. @@ -905,14 +904,14 @@ Compare the volume_mount_directories file Compare the feature branch file to the upcoming METplus version directory file:: - diff ${METPLUS_FEATURE_BRANCH}/volume_mount_directories ${METPLUS_VERSION}/volume_mount_directories + diff ${METPLUS_FEATURE_BRANCH}/volume_mount_directories v${METPLUS_VERSION}/volume_mount_directories **ONLY RUN THE COMMAND THAT IS APPROPRIATE TO YOUR USE CASE. READ CAREFULLY!** **CONDITION 1: IF there is a new entry or change in the feature version**, copy the feature file into the upcoming METplus version directory and the develop directory:: - cp ${METPLUS_FEATURE_BRANCH}/volume_mount_directories ${METPLUS_VERSION}/volume_mount_directories + cp ${METPLUS_FEATURE_BRANCH}/volume_mount_directories v${METPLUS_VERSION}/volume_mount_directories cp ${METPLUS_FEATURE_BRANCH}/volume_mount_directories develop/volume_mount_directories Copy data from the feature directory into the next version directory diff --git a/docs/Release_Guide/met_official.rst b/docs/Release_Guide/met_official.rst index 1cf1457509..7bef52530a 100644 --- a/docs/Release_Guide/met_official.rst +++ b/docs/Release_Guide/met_official.rst @@ -13,6 +13,7 @@ Create a new vX.Y.Z official release from the develop branch. .. include:: release_steps/update_release_notes_official.rst .. include:: release_steps/rotate_authorship.rst .. include:: release_steps/merge_release_issue.rst +.. include:: release_steps/met/create_release_reference_branch.rst .. include:: release_steps/create_release_branch.rst .. include:: release_steps/create_release_on_github.rst .. include:: release_steps/create_release_extra.rst diff --git a/docs/Release_Guide/metplus_official.rst b/docs/Release_Guide/metplus_official.rst index 675a3012a4..47a1ba344d 100644 --- a/docs/Release_Guide/metplus_official.rst +++ b/docs/Release_Guide/metplus_official.rst @@ -12,9 +12,9 @@ Create a new vX.Y.Z official release from the develop branch. .. include:: release_steps/metplus/update_release_date.rst .. include:: release_steps/update_release_notes_official.rst .. include:: release_steps/rotate_authorship.rst +.. include:: release_steps/metplus/update_manage_externals.rst .. include:: release_steps/merge_release_issue.rst .. include:: release_steps/metplus/create_release_reference_branch.rst -.. include:: release_steps/metplus/update_release_content.rst .. include:: release_steps/push_release_branch.rst .. include:: release_steps/create_release_on_github.rst .. include:: release_steps/metplus/create_release_extra.rst diff --git a/docs/Release_Guide/release_steps/create_release_on_github.rst b/docs/Release_Guide/release_steps/create_release_on_github.rst index a20cd4f28a..c1346b2cd0 100644 --- a/docs/Release_Guide/release_steps/create_release_on_github.rst +++ b/docs/Release_Guide/release_steps/create_release_on_github.rst @@ -13,7 +13,6 @@ Create Release on GitHub https://|projectRepo|.readthedocs.io/en/vX.Y.Z-betaN/Users_Guide/release-notes.html (Note: the URL will not be active until the release is created) -* Add a link to the PDF of the |projectRepo| User's Guide, if available. - The PDF can be downloaded from ReadTheDocs if it is available, i.e. +* Add links to the HTML and PDF versions of the |projectRepo| User's Guide on ReadTheDocs. https://|projectRepo|.readthedocs.io/_/downloads/en/vX.Y.Z-betaN/pdf/ (Note: the URL will not be active until the release is created) diff --git a/docs/Release_Guide/release_steps/finalize_release_on_github_official.rst b/docs/Release_Guide/release_steps/finalize_release_on_github_official.rst index ca2abafbd7..45123c695d 100644 --- a/docs/Release_Guide/release_steps/finalize_release_on_github_official.rst +++ b/docs/Release_Guide/release_steps/finalize_release_on_github_official.rst @@ -1,14 +1,33 @@ Finalize Release on GitHub -------------------------- -* Close the GitHub issue for creating this official release. -* Edit the milestone for the current release by updating the *Due date* with the actual release date. -* If necessary, create a new milestone for the next official release (e.g. next vX.Y.Z release). -* If necessary, reassign any remaining issues for the current milestone to the next one. -* Close the current milestone. -* Confirm that all existing development projects for the current milestone are closed. -* If necessary, create development projects for the next milestone (e.g. |projectRepo|-X.Y.Z-beta1, beta2, beta3). -* Update the repository settings by resetting the *Default branch* to the new main_vX.Y branch: +* Update milestones: + + * Edit the milestone for the current release by updating the *Due date* with the actual release date. + + * Close the current milestone. + + * Create a new milestone for the first bugfix release (e.g. first vX.Y.1 (bugfix) release). + + * If necessary, create a new milestone for the next official release (e.g. next vX.Y.Z release). + +* Update issues: + + * Close the GitHub issue for creating this official release. + + * If necessary, reassign any remaining issues for the current milestone to other milestones. + +* Update projects: + + * Confirm that all existing development projects for the current milestone are closed. + + * If necessary, create development projects for the next milestone (e.g. |projectRepo|-X.Y.Z-beta1, beta2, beta3). + +* Update branches: + + * Remove any remaining stale development branches from the new release. + + * Update the repository settings by resetting the *Default branch* to the new main_vX.Y branch: .. parsed-literal:: @@ -16,4 +35,3 @@ Finalize Release on GitHub -> Settings -> Branches (tab on left) -> change the drop down to new branch - diff --git a/docs/Release_Guide/release_steps/met/create_release_reference_branch.rst b/docs/Release_Guide/release_steps/met/create_release_reference_branch.rst new file mode 100644 index 0000000000..c4018922da --- /dev/null +++ b/docs/Release_Guide/release_steps/met/create_release_reference_branch.rst @@ -0,0 +1,35 @@ +Create Release Reference Branch +------------------------------- + +* Create a branch from the develop branch for the reference branch for the + new official release and push it to GitHub. The branch name should match + the format main_vX.Y-ref where X.Y is the major/minor release number. + +.. parsed-literal:: + + cd |projectRepo| + git checkout develop + git pull + git checkout -b main_vX.Y-ref + +Push Reference Branch to GitHub +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +:: + + git push -u origin main_vX.Y-ref + +Pushing this branch to GitHub should trigger the GitHub Actions automation +that runs all of the use cases and creates Docker data volumes with the output +data. These data will be used to verify that any bugfixes applied to the +main_vX.Y branch does not break any of existing logic. + +Monitor GitHub Actions Workflow +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Navigate to https://github.com/dtcenter/MET/actions and verify that a +*Testing* workflow was triggered on the *main_vX.Y-ref* branch. + +* Wait until the entire workflow has run successfully. The final job entitled + "Create Output Docker Data Volumes" should create Docker data volumes for + each use case category on DockerHub (dtcenter/met-dev). diff --git a/docs/Release_Guide/release_steps/metplus/create_release_reference_branch.rst b/docs/Release_Guide/release_steps/metplus/create_release_reference_branch.rst index 94845f5361..db2a4db7ec 100644 --- a/docs/Release_Guide/release_steps/metplus/create_release_reference_branch.rst +++ b/docs/Release_Guide/release_steps/metplus/create_release_reference_branch.rst @@ -1,6 +1,13 @@ Create Release Reference Branch ------------------------------- +* For METplus, the corresponding MET release must be created before starting + this step. Specifically, a DockerHub tag on dtcenter/met that is named + (X+6).Y-latest must already exist. For example, for METplus 4.1.0, the MET + DockerHub tag for 10.1-latest must be built. + See https://hub.docker.com/repository/docker/dtcenter/met/general for + a list of existing tags. + * Create a branch from the develop branch for the reference branch for the new official release and push it to GitHub. The branch name should match the format main_vX.Y-ref where X.Y is the major/minor release number. @@ -11,6 +18,31 @@ Create Release Reference Branch git checkout develop git pull git checkout -b main_vX.Y-ref + +Update the version number +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Update the content that should go into the release version but remain unchanged +in the develop branch. + +Remove **-dev** from the version number: + +* As of METplus 4.0.0, we are naming releases with X.Y.Z format even if Z is 0. +* As of METplus v4.0.0, the file containing the version number is located at + **metplus/VERSION** (in earlier releases, the file was located at + docs/version or doc/version). +* In the develop branch, the version should match the upcoming release + with -dev added to the end like X.Y.Z-betaN-dev, i.e. 4.0.0-beta1-dev +* Remove **-dev** from the version number so that it matches the release + you are creating. + +Commit these changes. + +Push Reference Branch to GitHub +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +:: + git push -u origin main_vX.Y-ref Pushing this branch to GitHub should trigger the GitHub Actions automation @@ -18,6 +50,9 @@ that runs all of the use cases and creates Docker data volumes with the output data. These data will be used to verify that any bugfixes applied to the main_vX.Y branch does not break any of existing logic. +Monitor GitHub Actions Workflow +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + Navigate to https://github.com/dtcenter/METplus/actions and verify that a *Testing* workflow was triggered on the *main_vX.Y-ref* branch. @@ -28,6 +63,9 @@ Navigate to https://github.com/dtcenter/METplus/actions and verify that a each use case category on DockerHub (dtcenter/metplus-data-dev). The names of these volumes start with *output-*. +Create main branch and push to GitHub +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + * After the truth data volumes have been generated, create the main_vX.Y branch off of the -ref branch. diff --git a/docs/Release_Guide/release_steps/metplus/update_manage_externals.rst b/docs/Release_Guide/release_steps/metplus/update_manage_externals.rst new file mode 100644 index 0000000000..c32b3f0e8a --- /dev/null +++ b/docs/Release_Guide/release_steps/metplus/update_manage_externals.rst @@ -0,0 +1,9 @@ +Update the version numbers in the manage externals files +-------------------------------------------------------- + +build_components/Externals_stable.cfg + +Each of the components in these files has a tag associated with them. +Update the value for tag to the tag associated with the release for each +METplus component that is part of this METplus coordinated release, i.e. +MET should be v10.0.0 for the METplus 4.0.0 coordinated release. diff --git a/docs/Release_Guide/release_steps/metplus/update_release_content.rst b/docs/Release_Guide/release_steps/metplus/update_release_content.rst deleted file mode 100644 index 92ba48afda..0000000000 --- a/docs/Release_Guide/release_steps/metplus/update_release_content.rst +++ /dev/null @@ -1,29 +0,0 @@ -Update Release Content ----------------------- - -Update content that should go into the release version but remain unchanged -in the develop branch. - -Update the version number -^^^^^^^^^^^^^^^^^^^^^^^^^ - -Remove **-dev** from the version number: - -* As of METplus 4.0.0, we are naming releases with X.Y.Z format even if Z is 0. -* As of METplus v4.0.0, the file containing the version number is located at - **metplus/VERSION** (in earlier releases, the file was located at - docs/version or doc/version). -* In the develop branch, the version should match the upcoming release - with -dev added to the end like X.Y.Z-betaN-dev, i.e. 4.0.0-beta1-dev -* Remove **-dev** from the version number so that it matches the release - you are creating. - -Update the version numbers in the manage externals files -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -build_components/Externals_stable.cfg - -Each of the components in these files has a branch associated with them. -Update the value for branch to the tag associated with the release for each -METplus component that is part of this METplus coordinated release, i.e. -MET should be 10.0.0 for the METplus 4.0.0 coordinated release. diff --git a/docs/Release_Guide/release_steps/open_release_issue.rst b/docs/Release_Guide/release_steps/open_release_issue.rst index a1ea1abaec..46cdc15d83 100644 --- a/docs/Release_Guide/release_steps/open_release_issue.rst +++ b/docs/Release_Guide/release_steps/open_release_issue.rst @@ -7,7 +7,7 @@ Open Release Issue * "Create the |projectRepo| vX.Y.Z-betaN (or -rcN) release" for a development release. -* For an official or development release, assign the issue to the current |projectRepo| development project. +* For an official or development release, assign the issue to the corresponding |projectRepo| development project. * For a bugfix release, assign the issue to the organization level support project. diff --git a/docs/Release_Guide/release_steps/rotate_authorship.rst b/docs/Release_Guide/release_steps/rotate_authorship.rst index cf6f9627b7..e0507210b0 100644 --- a/docs/Release_Guide/release_steps/rotate_authorship.rst +++ b/docs/Release_Guide/release_steps/rotate_authorship.rst @@ -14,5 +14,16 @@ for each official release: "First Initial. Last Name". Please maintain that format as it is the most common format for citations. +* The author list is typically found in the conf.py file in the documentation + directory, i.e. *docs/conf.py*. + Most of the component repositories store the list of authors in a variable + named **author_list**. Please ensure that changes to this list match the + correct format listed above. + In the METplus repository, the conf.py file has variable named + CURRENT_AUTHORS that is a list of the authors to rotate. + To rotate, move the first item in the list to the end of the list. + There is logic in this file to read the list and format it properly to match + the expected format for citations. + * Review the list of authors in the citation and at the top level of the documentation and update as needed. diff --git a/docs/Users_Guide/overview.rst b/docs/Users_Guide/overview.rst index d76e2c94eb..b6f30e4025 100644 --- a/docs/Users_Guide/overview.rst +++ b/docs/Users_Guide/overview.rst @@ -85,6 +85,8 @@ METplus Wrappers configuration files. Some Python wrappers do not correspond to a particular MET tool, but wrap utilities to extend METplus functionality. +.. _python_requirements_table: + METplus Components Python Requirements ====================================== @@ -96,7 +98,6 @@ METplus Components Python Requirements Description - no more than 22 (was 20) characters Use Cases - no more than 17 (was 10) characters - .. role:: raw-html(raw) :format: html @@ -414,6 +415,14 @@ METplus Components Python Requirements Python programming :raw-html:`
` language - For using MET Python embedding functionality in use cases + * - pint + - >=0.18 + - METcalcpy + - https://github.com/hgrecco/pint + - Python package to define, :raw-html:`
` + operate and manipulate :raw-html:`
` + physical quantities + - * - plotly - >=4.9.0 - METcalcpy, :raw-html:`
` diff --git a/docs/Users_Guide/release-notes.rst b/docs/Users_Guide/release-notes.rst index 6f6c3a7ba9..3443efde4c 100644 --- a/docs/Users_Guide/release-notes.rst +++ b/docs/Users_Guide/release-notes.rst @@ -39,168 +39,192 @@ describes the bugfix, enhancement, or new feature: https://github.com/dtcenter/METplus/issues -METplus Version 4.1.0-beta6 Release Notes (2022-03-01) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -* Enhancements: - - * **Add support for setting file_type for fcst/obs for applications that process gridded data** (`#1165 `_) - * **Enhance logic for setting mask.poly to allow MET list characters (square braces and semi-colon)** (`#966 `_) - * Enhance TCGen wrapper to add support for new configurations (`#1273 `_) - * Add support for new climo_cdf.direct_prob flag (`#1392 `_) - * Implement various enhancements to climatology settings (`#1247 `_) - * Enhance logic to set climatology info for Python embedding (`#944 `_) - * Incorporate basic zonal and meridional means into METplus (`#1230 `_) - * Add support for explicitly setting file list file paths in wrappers that support multiple input files (`#1289 `_) - * Add support for setting grid_weight_flag in EnsembleStat (`#1369 `_) - * Add option to PCPCombine to force using 0 hr accum in subtract mode (`#1368 `_) - * Fix logic to use fcst dictionary if ens dictionary is not set in EnsembleStat wrapper (`#1421 `_) - * Fix logic to create instances of other wrappers within wrappers to avoid modifying global configurations (`#1356 `_) - * Remove support for PB2NC_OBS_PREPBUFR_MAP (`#1431 `_) - - -* New Use Cases: +METplus Version 4.1.0 Release Notes (2022-03-14) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - * Precipitation-type comparison across 3 models (`#1408 `_) - * TC Verification Compare ADECK vs BDECK (`#911 `_) +* Enhancements: -* Internal: + * General: - * Document Continous Integration Functionality in the METplus Contributor's Guide (`#675 `_) + * **Add support for setting control members in EnsembleStat and GenEnsProd** (`#1236 `_) + * **Create an Amazon AMI containing all METplus components** (`#506 `_) + * Modify wrappers that use wrapped MET config files to default to parm/met_config versions if unset (`#931 `_) + * Add support for setting hss_ec_value in MET config files (`#951 `_) + * Added support for setting a dictionary value for time_summary.width (`#1252 `_) + * Properly handle list values that include square braces (`#1212 `_) + * Update wrapped MET config files to reference MET_TMP_DIR in tmp value (`#1101 `_) + * Add support for setting INIT_LIST and VALID_LIST for irregular time intervals (`#1286 `_) + * Support setting the OMP_NUM_THREADS environment variable (`#1320 `_) + * Add support for commonly changed MET config variables (`#896 `_) + * Prevent wildcard character from being used in output file path (`#1291 `_) + * Add support for setting file_type for fcst/obs for applications that process gridded data (`#1165 `_) + * Enhance logic for setting mask.poly to allow MET list characters (square braces and semi-colon) (`#966 `_) + * Add support for new climo_cdf.direct_prob flag (`#1392 `_) + * Implement various enhancements to climatology settings (`#1247 `_) + * Enhance logic to set climatology info for Python embedding (`#944 `_) + * Updated logic for handling _CLIMO_MEAN_FIELD variables for specifying climatology fields (`#1021 `_) + * Incorporate basic zonal and meridional means into METplus (`#1230 `_) + * Add support for explicitly setting file list file paths in wrappers that support multiple input files (`#1289 `_) + * Add support for setting -out argument in TCStat and StatAnalysis wrappers (`#1102 `_) -METplus Version 4.1.0-beta5 Release Notes (2022-01-14) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + * PointStat: -* Enhancements: + * Make output_flag.orank configurable for Point-Stat (`#1103 `_) + * Added support for setting obs_quality_inc/exc in PointStat (`#1213 `_) - * **Add support for setting control members in EnsembleStat and GenEnsProd** (`#1236 `_) - * **Enhance SeriesAnalysis wrapper to allow different field info values for each file in a list** (`#1166 `_) - * Add support for setting INIT_LIST and VALID_LIST for irregular time intervals (`#1286 `_) - * Support setting the OMP_NUM_THREADS environment variable (`#1320 `_) - * Enhance ExtractTiles using MTD input to properly match times (`#1285 `_) - * Add support for commonly changed MET config variables part 2 (`#896 `_) - * Prevent wildcard character from being used in output file path (`#1291 `_) + * GridStat: -* New Use Cases: + * Add Grid-Stat configuration options for distance_map dictionary (`#1089 `_) - * Satellite verification of sea surface salinity: SMOS vs RTOFS output (`#1116 `_) + * EnsembleStat: -* Internal: + * Add support for setting grid_weight_flag in EnsembleStat (`#1369 `_) + * Fix logic to use fcst dictionary if ens dictionary is not set in EnsembleStat wrapper (`#1421 `_) + * Add support for probabilistic verification to the Ensemble-Stat wrapper (`#1464 `_) - * **Create guidance for memory-intensive use cases, introduce Python memory profiler** (`#1183 `_) - * **Identify code throughout METplus components that are common utilities** (`#799 `_) - * **Add definitions to the Release Guide for the stages of the release cycle** (`#934 `_) + * GenEnsProd: -METplus Version 4.1.0-beta4 Release Notes (2021-11-16) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + * Add support for the normalize option to the Gen-Ens-Prod wrapper (`#1445 `_) -* Enhancements: + * TCPairs: - * **Create an Amazon AMI containing all METplus components** (`#506 `_) - * Added support for setting a dictionary value for time_summary.width (`#1252 `_) - * Added support for setting obs_quality_inc/exc in PointStat (`#1213 `_) - * Properly handle list values that include square braces (`#1212 `_) - * Reorganize the Cryosphere and Marine and Coastal use case categories into one group (`#1200 `_) - * Update wrapped MET config files to reference MET_TMP_DIR in tmp value (`#1101 `_) - * CyclonePlotter, create options to format output grid area to user-desired area (`#1091 `_) - * CyclonePlotter, connected lines run over the Prime Meridian (`#1000 `_) - * Add harmonic pre-processing to the RMM use case (`#1019 `_) + * Enhance TC-Pairs wrapper to make valid_inc, valid_exc, and write_valid configurable options (`#1069 `_) + * Improve logic of TCPairs wrapper (`#749 `_) + * Enhance TCPairs to loop by valid time and allow looping when LOOP_ORDER = processes (`#986 `_) -* New Wrappers: + * TCGen: - * **IODA2NC** (`#1203 `_) - * **GenEnsProd** (`#1180 `_, `#1266 `_) + * Enhance TCGen wrapper to add support for new configurations (`#1273 `_) -* New Use Cases: + * SeriesAnalysis: - * **IODA2NC** (`#1204 `_) - * **GenEnsProd** (`#1180 `_, `#1266 `_) + * **Enhance SeriesAnalysis wrapper to allow different field info values for each file in a list** (`#1166 `_) + * Add support for probability field threshold in SeriesAnalysis (`#875 `_) + * RegridDataPlane: -METplus Version 4.1.0-beta3 Release Notes (2021-10-06) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + * **Add support for extra field options in RegridDataPlane wrapper** (`#924 `_) -* Enhancements: + * PCPCombine: - * Add Grid-Stat configuration options for distance_map dictionary (`#1089 `_) - * Fix installation instructions in User's Guide (`#1067 `_) - * Add instructions to update old METplus configuration files that reference user-defined wrapped MET config files (`#1147 `_) + * Improve PCPCombine derive mode logic to skip lookback (`#928 `_) + * Add support for using filename templates for defining input level in PCPCombine (`#1062 `_) + * Add option to PCPCombine to force using 0 hr accum in subtract mode (`#1368 `_) -* New Use Cases: + * GenVxMask: - * Satellite verification of sea surface temperature (GHRSST) against RTOFS output (`#1004 `_) - * Feature Relative using MTD output for feature centroid lat/lon (`#641 `_) + * Update GenVxMask wrapper to require setting -type (`#960 `_) -* Internal: + * UserScript: - * Transition Community and Developer Support to Github Discussions (`#932 `_) - * Add documentation about the Release Guide and Verification Datasets Guide (`#874 `_) + * **Enhance UserScript to get a list of files that match the run times instead of using a wildcard** (`#1002 `_) -METplus Version 4.1.0-beta2 Release Notes (2021-08-31) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + * ExtractTiles: -* Enhancements: + * Enhance ExtractTiles using MTD input to properly match times (`#1285 `_) + + * TCMPRPlotter: + + * Improvements to TCMPRPlotter wrapper logging and output control (`#926 `_) + * Add option to TCMPRPlotter to pass in directory to tc_stat instead of individual files (`#1057 `_) + * Add option to pass in the input directory to TCMPRPlotter instead of finding all tcst files and passing the list (`#1084 `_) + + * CyclonePlotter: + + * Update CyclonePlotter for offline/HPC usage (`#933 `_) + * CyclonePlotter, create options to format output grid area to user-desired area (`#1091 `_) + * CyclonePlotter, connected lines run over the Prime Meridian (`#1000 `_) + + * Use Cases: + + * Add stat_analysis to the Blocking and Weather Regime processing (`#1001 `_) + * Modify user diagnostic feature relative use case to use MetPy Python package (`#759 `_) + * Reorganize the Cryosphere and Marine and Coastal use case categories into one group (`#1200 `_) + * Add harmonic pre-processing to the RMM use case (`#1019 `_) - * Add stat_analysis to the Blocking and Weather Regime processing (`#1001 `_) - * Make output_flag.orank configurable for Point-Stat (`#1103 `_) - * Enhance TC-Pairs wrapper to make valid_inc, valid_exc, and write_valid configurable options (`#1069 `_) - * Add option to TCMPRPlotter to pass in directory to tc_stat instead of individual files (`#1057 `_) - * **Add support for using filename templates for defining input level in PCPCombine** (`#1062 `_) - * Modify wrappers that use wrapped MET config files to default to parm/met_config versions if unset (`#931 `_) - * Modify user diagnostic feature relative use case to use MetPy Python package (`#759 `_) - * Add option to pass in the input directory to TCMPRPlotter instead of finding all tcst files and passing the list (`#1084 `_) - * Updated logic for handling _CLIMO_MEAN_FIELD variables for specifying climatology fields (`#1021 `_) - * Add support for setting hss_ec_value in MET config files (`#951 `_) - * Update documentation to reference GitHub Discussions instead of MET Help (`#956 `_) * New Wrappers: + * GenEnsProd (`#1180 `_, `#1266 `_) + * GFDLTracker (`#615 `_) + * IODA2NC (`#1203 `_) + + * New Use Cases: - * GFDL tracker for TC genesis (`#616 `_) - * GFDL tracker for Extra-TC tracking (`#617 `_) - * RMM and OMI (driver scripts) (`#892 `_) + * MET Tool Wrapper: -* Internal: + * **PointStat: Python Embedding for Point Observations** (`#1490 `_) + * IODA2NC (`#1204 `_) + * GenEnsProd (`#1180 `_, `#1266 `_) + * GFDLTracker for TropicalCyclone (`#615 `_) + * GFDLTracker for TC Genesis (`#616 `_) + * GFDLTracker for Extra-TC Tracking (`#617 `_) - * Make updates to the Release Guide (`#935 `_) - * Clean up GitHub wiki broken links and out-of-date information (`#237 `_) - * Add option to override MET version used for automated tests (`#936 `_) + * Marine and Cryosphere: -METplus Version 4.1.0-beta1 Release Notes (2021-07-21) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + * GridStat_fcstRTOFS_obsOSTIA_iceCover (`#834 `_) + * Satellite verification of sea surface temperature (GHRSST) against RTOFS output (`#1004 `_) + * Satellite verification of sea surface salinity: SMOS vs RTOFS output (`#1116 `_) + * Satellite verification of sea surface salinity: AVISO vs RTOFS output HYCOM climo (`#1318 `_) + * Satellite verification of sea surface salinity: SMAP vs RTOFS output (`#1216 `_) -* Enhancements: - * **Improve logic of TCPairs wrapper** (`#749 `_) - * **Add support for probability field threshold in SeriesAnalysis** (`#875 `_) - * **Add support for extra field options in RegridDataPlane wrapper** (`#924 `_) - * **Improvements to TCMPRPlotter wrapper logging and output control** (`#926 `_) - * **Improve PCPCombine derive mode logic to skip lookback** (`#928 `_) - * **Update CyclonePlotter for offline/HPC usage** (`#933 `_) - * **Update GenVxMask wrapper to require setting -type** (`#960 `_) - * **Enhance TCPairs to loop by valid time and allow looping when LOOP_ORDER = processes** (`#986 `_) - * **Enhance UserScript to get a list of files that match the run times instead of using a wildcard** (`#1002 `_) + * Medium Range: + * Feature Relative using MTD output for feature centroid lat/lon (`#641 `_) -* New Wrappers: - * **GFDLTracker** + * Precipitation: + * Precipitation-type comparison across 3 models (`#1408 `_) + + + * Seasonal to Subseasonal (S2S): + + * UserScript_fcstGFS_obsERA_OMI (`#892 `_) + * UserScript_fcstGFS_obsERA_PhaseDiagram (`#1019 `_) + * UserScript_fcstGFS_obsERA_RMM (`#892 `_) + * RMM and OMI (driver scripts) (`#892 `_) + + + * Tropical Cyclone and Extra Tropical Cyclone (tc_and_extra_tc): + + * TC Verification Compare ADECK vs BDECK (`#911 `_) + * TCGen Verify Deterministic Genesis Forecasts and Probabilities from ATCF e-deck files (`#1274 `_) + + +* Bugfixes: + + * Fix read of PB2NC_FILE_WINDOW_[BEGIN/END] configuration variables (`#1486 `_) + * Fix use of current field info in output prefix when using process list instances (`#1471 `_) + * Fix logic to create instances of other wrappers within wrappers to avoid modifying global configurations (`#1356 `_) -* New Use Cases: - * Marine and Cryosphere: GridStat_fcstRTOFS_obsOSTIA_iceCover (`#834 `_) - * met_tool_wrapper: GFDLTracker_TC (`#615 `_) - * Seasonal to Subseasonal: UserScript_fcstGFS_obsERA_OMI - * Seasonal to Subseasonal: UserScript_fcstGFS_obsERA_PhaseDiagram - * Seasonal to Subseasonal: UserScript_fcstGFS_obsERA_RMM +* Documentation: + * Add list of METplus statistics to documentation (`#1049 `_) + * Update documentation to reference GitHub Discussions instead of MET Help (`#956 `_) + * Fix installation instructions in User's Guide (`#1067 `_) + * Add instructions to update old METplus configuration files that reference user-defined wrapped MET config files (`#1147 `_) * Internal: * Improve approach to obtain additional python packages needed for some use cases (`#839 `_) + * Make updates to the Release Guide (`#935 `_) + * Clean up GitHub wiki broken links and out-of-date information (`#237 `_) + * Add option to override MET version used for automated tests (`#936 `_) + * Transition Community and Developer Support to Github Discussions (`#932 `_) + * Add documentation about the Release Guide and Verification Datasets Guide (`#874 `_) + * Create guidance for memory-intensive use cases, introduce Python memory profiler (`#1183 `_) + * Identify code throughout METplus components that are common utilities (`#799 `_) + * Add definitions to the Release Guide for the stages of the release cycle (`#934 `_) + * Document Continous Integration Functionality in the METplus Contributor's Guide (`#675 `_) + * Update Contributor's Guide for new removing/adding data protocols (`#1227 `_) + * Add recording of Python packages to Adding Use Cases documentation (`#1374 `_) + * Remove public-facing access to outdated use case categories (Cryosphere, marine_and_coastal) (`#1226 `_) + METplus Version 4.0.0 Release Notes (2021-05-10) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/Users_Guide/statistics_list.rst b/docs/Users_Guide/statistics_list.rst index 856f073ee1..320c570334 100644 --- a/docs/Users_Guide/statistics_list.rst +++ b/docs/Users_Guide/statistics_list.rst @@ -1,19 +1,26 @@ -****************************** -METplus Database of Statistics -****************************** +******************************** +METplus Statistics & Diagnostics +******************************** .. Number of characters per line: Statistic Name - no more that 32 characters METplus Name - no more than 17 characters Statistic Type - no more than 19 characters + Tools - approx 18 characters? METplus Line Type - currently unlimited (approx 33 characters) +Statistics Database +=================== + + +Statistics List A-B +___________________ .. role:: raw-html(raw) :format: html -.. list-table:: Statistics List +.. list-table:: Statistics List A-B :widths: auto :header-rows: 1 @@ -34,7 +41,8 @@ METplus Database of Statistics MCTS :raw-html:`
` NBRCTS :raw-html:`
` MODE cts - * - Asymptotic Fractions Skill Score + * - Asymptotic :raw-html:`
` + Fractions Skill Score - AFSS - Neighborhood - Grid-Stat @@ -46,12 +54,6 @@ METplus Database of Statistics TC-Stat - TCMPR :raw-html:`
` TCST - * - Difference between the axis :raw-html:`
` - angles of two objects (in degrees) - - ANGLE_DIFF - - Diagnostic - - MODE - - MODE * - Anomaly Correlation :raw-html:`
` including mean error - ANOM_CORR @@ -64,56 +66,14 @@ METplus Database of Statistics * - Uncentered Anomaly :raw-html:`
` Correlation excluding mean :raw-html:`
` error - - ANOM_CORR :raw-html:`
` _UNCNTR + - ANOM_CORR :raw-html:`
` + _UNCNTR - Continuous - Point-Stat :raw-html:`
` Grid-Stat :raw-html:`
` Series-Analysis :raw-html:`
` Stat-Analysis - CNT - * - Object area (in grid squares) - - AREA - - Diagnostic - - MODE :raw-html:`
` - MTD - - MODE obj - * - Forecast object area :raw-html:`
` - divided by the observation :raw-html:`
` - object area (unitless) - - AREA_RATIO - - Diagnostic - - MODE - - MODE obj - * - Area of the object :raw-html:`
` - that meet the object :raw-html:`
` - definition threshold :raw-html:`
` - criteria (in grid squares) - - AREA_THRESH - - Diagnostic - - MODE - - MODE obj - * - Absolute value of :raw-html:`
` - the difference :raw-html:`
` - between the aspect :raw-html:`
` - ratios of two objects :raw-html:`
` - (unitless) - - ASPECT_DIFF - - Diagnostic - - MODE - - MODE obj - * - Object axis angle :raw-html:`
` - (in degrees) - - AXIS_ANG - - Diagnostic - - MODE :raw-html:`
` - MTD - - MTD obj - * - Difference in spatial :raw-html:`
` - axis plane angles - - AXIS_DIFF - - Diagnostic - - MTD - - MTD obj * - Baddeley’s Delta Metric - BADDELEY - Distance Map @@ -149,13 +109,6 @@ METplus Database of Statistics Ensemble-Stat - CNT :raw-html:`
` SSVAR - * - Minimum distance between :raw-html:`
` - the boundaries of two objects - - BOUNDARY :raw-html:`
` - _DIST - - Diagnostic - - MODE - - MODE obj * - Brier Score - BRIER - Probability @@ -182,6 +135,24 @@ METplus Database of Statistics - Point-Stat :raw-html:`
` Grid-Stat - PSTD + +Statistics List C-E +___________________ + +.. role:: raw-html(raw) + :format: html + +.. list-table:: Statistics List C-E + :widths: auto + :header-rows: 1 + + * - Statistics :raw-html:`
` + Long Name + - METplus Name + - Statistic Type + - Tools + - METplus :raw-html:`
` + Line Type * - Calibration when forecast :raw-html:`
` is between the ith and :raw-html:`
` i+1th probability :raw-html:`
` @@ -192,58 +163,6 @@ METplus Database of Statistics - Point-Stat :raw-html:`
` Grid-Stat - PJC - * - Total great circle distance :raw-html:`
` - travelled by the 2D spatial :raw-html:`
` - centroid over the lifetime :raw-html:`
` - of the 3D object - - CDIST :raw-html:`
` - _TRAVELLED - - Diagnostic - - MTD - - MTD 3D obj - * - Distance between two :raw-html:`
` - objects centroids :raw-html:`
` - (in grid units) - - CENTROID :raw-html:`
` - _DIST - - Diagnostic - - MODE - - MODE obj - * - Latitude of centroid :raw-html:`
` - - CENTROID :raw-html:`
` - _LAT - - Diagnostic - - MTD :raw-html:`
` - MODE - - MTD 2D & 3D obj :raw-html:`
` - MODE obj - * - Longitude of centroid :raw-html:`
` - - CENTROID :raw-html:`
` - _LON - - Diagnostic - - MTD :raw-html:`
` - MODE - - MTD 2D & 3D obj :raw-html:`
` - MODE obj - * - Time coordinate of centroid - - CENTROID_T - - Diagnostic - - MTD - - MTD 3D obj - * - X coordinate of centroid :raw-html:`
` - - CENTROID_X - - Diagnostic - - MTD :raw-html:`
` - MODE - - MTD 2D & 3D obj :raw-html:`
` - MODE obj - * - Y coordinate of centroid :raw-html:`
` - - CENTROID_Y - - Diagnostic - - MTD :raw-html:`
` - MODE - - MTD 2D & 3D obj :raw-html:`
` - MODE obj * - Climatological mean value - CLIMO_MEAN - Continuous @@ -261,35 +180,6 @@ METplus Database of Statistics Ensemble-Stat - MPR :raw-html:`
` ORANK - * - Ratio of the difference :raw-html:`
` - between the area of an :raw-html:`
` - object and the area of :raw-html:`
` - its convex hull divided :raw-html:`
` - by the area of the :raw-html:`
` - complex hull (unitless) - - COMPLEXITY - - Diagnostic - - MODE - - MODE obj - * - Ratio of complexities of :raw-html:`
` - two objects defined as :raw-html:`
` - the lesser of the forecast :raw-html:`
` - complexity divided by the :raw-html:`
` - observation complexity or :raw-html:`
` - its reciprocal (unitless) - - COMPLEXITY :raw-html:`
` - _RATIO - - Diagnostic - - MODE - - MODE obj - * - Minimum distance between :raw-html:`
` - the convex hulls of two :raw-html:`
` - objects (in grid units) - - CONVEX_HULL :raw-html:`
` - _DIST - - Diagnostic - - MODE - - MODE obj * - Continuous Ranked :raw-html:`
` Probability Score :raw-html:`
` (normal dist.) @@ -348,31 +238,6 @@ METplus Database of Statistics - CTS :raw-html:`
` MODE :raw-html:`
` MBRCTCS - * - Radius of curvature - - CURVATURE - - Diagnostic - - MODE - - MODE obj - * - Ratio of the curvature - - CURVATURE :raw-html:`
` - _RATIO - - Diagnostic - - MODE - - MODE obj - * - Center of curvature :raw-html:`
` - (in grid coordinates) - - CURVATURE :raw-html:`
` - _X - - Diagnostic - - MODE - - MODE obj - * - Center of curvature :raw-html:`
` - (in grid coordinates) - - CURVATURE :raw-html:`
` - _Y - - Diagnostic - - MODE - - MODE obj * - Absolute value of :raw-html:`
` DIR_ERR (see below) - DIR_ABSERR @@ -389,21 +254,6 @@ METplus Database of Statistics - Point-Stat :raw-html:`
` Grid-Stat - VCNT - * - Difference in object :raw-html:`
` - direction of movement - - DIRECTION :raw-html:`
` - _DIFF - - Diagnostic - - MTD - - MTD 3D obj - * - Difference in the :raw-html:`
` - lifetimes of the :raw-html:`
` - two objects - - DURATION :raw-html:`
` - _DIFF - - Diagnostic - - MTD - - MTD 3D obj * - Expected correct rate :raw-html:`
` used for MCTS HSS_EC - EC_VALUE @@ -432,18 +282,6 @@ METplus Database of Statistics - Continuous - Grid-Stat - GRAD - * - Object end time - - END_TIME - - Diagnostic - - MTD - - MTD 3D obj - * - Difference in object :raw-html:`
` - ending time steps - - END_TIME :raw-html:`
` - _DELTA - - Diagnostic - - MTD - - MTD 3D obj * - The unperturbed :raw-html:`
` ensemble mean value - ENS_MEAN @@ -467,6 +305,21 @@ METplus Database of Statistics Ensemble-Stat - CNT :raw-html:`
` SSVAR + +Statistics List F +_________________ + +.. list-table:: Statistics List F + :widths: auto + :header-rows: 1 + + * - Statistics :raw-html:`
` + Long Name + - METplus Name + - Statistic Type + - Tools + - METplus :raw-html:`
` + Line Type * - Forecast rate/event :raw-html:`
` frequency - F_RATE @@ -478,7 +331,7 @@ METplus Database of Statistics * - Mean forecast wind speed - F_SPEED :raw-html:`
` _BAR - - Continous + - Continuous - Point-Stat :raw-html:`
` Grid-Stat - VL1L2 @@ -533,193 +386,6 @@ METplus Database of Statistics - Continuous - Grid-Stat - NBRCNT - * - Number of forecast :raw-html:`
` - clusters - - fcst_clus - - Diagnostic - - MODE - - MODE obj - * - Number of points used to :raw-html:`
` - define the hull of all :raw-html:`
` - of the cluster forecast :raw-html:`
` - objects - - fcst_clus :raw-html:`
` - _hull - - Diagnostic - - MODE - - MODE obj - * - Forecast Cluster Convex :raw-html:`
` - Hull Point Latitude - - fcst_clus :raw-html:`
` - _hull_lat - - Diagnostic - - MODE - - MODE obj - * - Forecast Cluster Convex :raw-html:`
` - Hull Point Longitude - - fcst_clus :raw-html:`
` - _hull _lon - - Diagnostic - - MODE - - MODE obj - * - Number of Forecast :raw-html:`
` - Cluster Convex Hull Points - - fcst_clus :raw-html:`
` - _hull_npts - - Diagnostic - - MODE - - MODE obj - * - Forecast Cluster Convex :raw-html:`
` - Hull Starting Index - - fcst_clus :raw-html:`
` - _hull_start - - Diagnostic - - MODE - - MODE obj - * - Forecast Cluster Convex :raw-html:`
` - Hull Point X-Coordinate - - fcst_clus :raw-html:`
` - _hull_x - - Diagnostic - - MODE - - MODE obj - * - Forecast Cluster Convex :raw-html:`
` - Hull Point Y-Coordinate - - fcst_clus :raw-html:`
` - _hull_y - - Diagnostic - - MODE - - MODE obj - * - Forecast Object Raw :raw-html:`
` - Values - - fcst_obj :raw-html:`
` - _raw - - Diagnostic - - MODE - - MODE obj - * - Number of simple :raw-html:`
` - forecast objects - - fcst_simp - - Diagnostic - - MODE - - MODE obj - * - Number of points used :raw-html:`
` - to define the boundaries :raw-html:`
` - of all of the simple :raw-html:`
` - forecast objects - - fcst_simp :raw-html:`
` - _bdy - - Diagnostic - - MODE - - MODE obj - * - Forecast Simple :raw-html:`
` - Boundary Latitude - - fcst_simp :raw-html:`
` - _bdy_lat - - Diagnostic - - MODE - - MODE obj - * - Forecast Simple :raw-html:`
` - Boundary Longitude - - fcst_simp :raw-html:`
` - _bdy_lon - - Diagnostic - - MODE - - MODE obj - * - Number of Forecast :raw-html:`
` - Simple Boundary Points - - fcst_simp :raw-html:`
` - _bdy_npts - - Diagnostic - - MODE - - MODE obj - * - Forecast Simple :raw-html:`
` - Boundary Starting Index - - fcst_simp :raw-html:`
` - _bdy_start - - Diagnostic - - MODE - - MODE obj - * - Forecast Simple :raw-html:`
` - Boundary X-Coordinate - - fcst_simp :raw-html:`
` - _bdy_x - - Diagnostic - - MODE - - MODE obj - * - Forecast Simple :raw-html:`
` - Boundary Y-Coordinate - - fcst_simp :raw-html:`
` - _bdy_y - - Diagnostic - - MODE - - MODE obj - * - Number of points used to :raw-html:`
` - define the hull of all :raw-html:`
` - of the simple forecast :raw-html:`
` - objects - - fcst_simp :raw-html:`
` - _hull - - Diagnostic - - MODE - - MODE obj - * - Forecast Simple Convex :raw-html:`
` - Hull Point Latitude - - fcst_simp :raw-html:`
` - _hull_lat - - Diagnostic - - MODE - - MODE obj - * - Forecast Simple Convex :raw-html:`
` - Hull Point Longitude - - fcst_simp :raw-html:`
` - _hull_lon - - Diagnostic - - MODE - - MODE obj - * - Number of Forecast :raw-html:`
` - Simple Convex Hull Points - - fcst_simp :raw-html:`
` - _hull_npts - - Diagnostic - - MODE - - MODE obj - * - Forecast Simple Convex :raw-html:`
` - Hull Starting Index - - fcst_simp :raw-html:`
` - _hull_start - - Diagnostic - - MODE - - MODE obj - * - Forecast Simple Convex :raw-html:`
` - Hull Point X-Coordinate - - fcst_simp :raw-html:`
` - _hull_x - - Diagnostic - - MODE - - MODE obj - * - Forecast Simple Convex :raw-html:`
` - Hull Point Y-Coordinate - - fcst_simp :raw-html:`
` - _hull_y - - Diagnostic - - MODE - - MODE obj - * - Number of thresholds :raw-html:`
` - applied to the forecast - - fcst :raw-html:`
` - _thresh :raw-html:`
` - _length - - Diagnostic - - MODE - - MODE obj - * - Number of thresholds :raw-html:`
` - applied to the forecast - - fcst_thresh :raw-html:`
` - _length - - Diagnostic - - MODE - - MODE obj * - Direction of the average :raw-html:`
` forecast wind vector - FDIR @@ -727,12 +393,6 @@ METplus Database of Statistics - Point-Stat :raw-html:`
` Grid-Stat - VCNT - * - Forecast energy squared :raw-html:`
` - for this scale - - FENERGY - - - - Wavelet-Stat - - ISC * - Mean Forecast Anomaly Squared - FFABAR - Continuous @@ -748,18 +408,6 @@ METplus Database of Statistics Grid-Stat - SSVAR :raw-html:`
` SL1L2 - * - Mean of absolute value :raw-html:`
` - of forecast gradients - - FGBAR - - - - Grid-Stat - - GRAD - * - Ratio of forecast and :raw-html:`
` - observed gradients - - FGOG_RATIO - - - - Grid-Stat - - GRAD * - Count of events in :raw-html:`
` forecast category i and :raw-html:`
` observation category j @@ -823,38 +471,6 @@ METplus Database of Statistics Grid-Stat - SSVAR :raw-html:`
` SL1L2 - * - Pratt’s Figure of Merit :raw-html:`
` - from observation to :raw-html:`
` - forecast - - FOM_FO - - Diagnostic - - Grid-Stat - - DMAP - * - Maximum of FOM_FO :raw-html:`
` - and FOM_OF - - FOM_MAX - - Diagnostic - - Grid-Stat - - DMAP - * - Mean of FOM_FO :raw-html:`
` - and FOM_OF :raw-html:`
` - - FOM_MEAN - - Diagnostic - - Grid-Stat - - DMAP - * - Minimum of FOM_FO :raw-html:`
` - and FOM_OF - - FOM_MIN - - Diagnostic - - Grid-Stat - - DMAP - * - Pratt’s Figure of Merit :raw-html:`
` - from forecast to :raw-html:`
` - observation - - FOM_OF - - Diagnostic - - Grid-Stat - - DMAP * - Number of tied forecast :raw-html:`
` ranks used in computing :raw-html:`
` Kendall’s tau statistic @@ -910,20 +526,25 @@ METplus Database of Statistics - MODE :raw-html:`
` CTC :raw-html:`
` NBRCTC - * - Distance between the :raw-html:`
` - forecast and Best track :raw-html:`
` - genesis events (km) - - GEN_DIST - - Diagnostic - - TC-Gen - - GENMPR - * - Forecast minus Best track :raw-html:`
` - genesis time in HHMMSS :raw-html:`
` - format - - GEN_TDIFF - - Diagnostic - - TC-Gen - - GENMPR + +Statistics List G-M +___________________ + +.. role:: raw-html(raw) + :format: html + +.. list-table:: Statistics List G-M + :widths: auto + :header-rows: 1 + + * - Statistics :raw-html:`
` + Long Name + - METplus Name + - Statistic Type + - Tools + - METplus :raw-html:`
` + Line Type + * - Gerrity Score and :raw-html:`
` bootstrap confidence limits - GER @@ -946,11 +567,6 @@ METplus Database of Statistics - Point-Stat :raw-html:`
` Grid-Stat - FHO - * - Hausdorff Distance - - HAUSDORFF - - Diagnostic - - Grid-Stat - - DMAP * - Hanssen and Kuipers :raw-html:`
` Discriminant - HK @@ -973,7 +589,7 @@ METplus Database of Statistics CTS :raw-html:`
` NBRCTS * - Heidke Skill Score :raw-html:`
` - user-specific expected :raw-html:`
` + user-specific expected :raw-html:`
` correct - HSS_EC - Categorical @@ -985,102 +601,18 @@ METplus Database of Statistics - Ensemble - Ensemble-Stat - ECNT - * - Best track genesis minus :raw-html:`
` - forecast initialization :raw-html:`
` - time in HHMMSS format - - INIT_TDIFF - - Diagnostic - - TC-Gen - - GENMPR - * - 10th, 25th, 50th, 75th, :raw-html:`
` - 90th, and user-specified :raw-html:`
` - percentiles of :raw-html:`
` - intensity of the raw :raw-html:`
` - field within the :raw-html:`
` - object or time slice - - INTENSITY :raw-html:`
` - _10, _25, :raw-html:`
` - _50, _75, :raw-html:`
` - _90, _NN - - Diagnostic - - MODE - - MODE obj - * - Sum of the intensities of :raw-html:`
` - the raw field within the :raw-html:`
` - object (variable units) - - INTENSITY :raw-html:`
` - _SUM - - Diagnostics - - MODE - - MODE obj - * - Total interest for this :raw-html:`
` - object pair - - INTEREST - - Diagnostic - - MTD :raw-html:`
` - MODE - - MTD 3D obj :raw-html:`
` - MODE obj - * - Intersection area of two :raw-html:`
` - objects (in grid squares) - - INTERSECT :raw-html:`
` - ION_AREA - - Diagnostic - - MODE - - MODE obj - * - Ratio of intersection area :raw-html:`
` - to the lesser of the :raw-html:`
` - forecast and observation :raw-html:`
` - object areas (unitless) - - INTERSECT :raw-html:`
` - ION_OVER :raw-html:`
` - _AREA - - Diagnostic - - MODE - - MODE obj - * - “Volume” of object :raw-html:`
` - intersection - - INTERSECT :raw-html:`
` - ION_VOLUME - - Diagnostic - - MTD - - MTD 3D obj * - Interquartile Range :raw-html:`
` - IQR - Continuous - Point-Stat :raw-html:`
` Grid-Stat - - CNT - * - The intensity scale :raw-html:`
` - skill score - - ISC - - - - Wavelet-Stat - - ISC - * - The scale at which all :raw-html:`
` - information following :raw-html:`
` - applies - - ISCALE - - - - Wavelet-Stat - - ISC + - CNT * - Kendall’s tau statistic - KT_CORR - Continuous - Point-Stat :raw-html:`
` Grid-Stat - CNT - * - Dimension of the latitude - - LAT - - Diagnostic - - MODE - - MODE obj - * - Length of the :raw-html:`
` - enclosing rectangle - - LENGTH - - Diagnostic - - MODE - - MODE obj * - Likelihood when forecast :raw-html:`
` is between the ith and :raw-html:`
` i+1th probability :raw-html:`
` @@ -1098,11 +630,6 @@ METplus Database of Statistics Grid-Stat - CTS :raw-html:`
` NBRCTS - * - Dimension of the longitude - - LON - - Diagnostic - - MODE - - MODE obj * - The Median Absolute :raw-html:`
` Deviation - MAD @@ -1179,14 +706,6 @@ METplus Database of Statistics - Distance - Grid-Stat - DMAP - * - Mean of maximum of :raw-html:`
` - absolute values of :raw-html:`
` - forecast and observed :raw-html:`
` - gradients - - MGBAR - - - - Grid-Stat - - GRAD * - Mean squared error - MSE - Continuous @@ -1212,7 +731,25 @@ METplus Database of Statistics - Continuous - Point-Stat :raw-html:`
` Grid-Stat - - VCNT + - VCNT + +Statistics List N-O +___________________ + +.. role:: raw-html(raw) + :format: html + +.. list-table:: Statistics List N-O + :widths: auto + :header-rows: 1 + + * - Statistics :raw-html:`
` + Long Name + - METplus Name + - Statistic Type + - Tools + - METplus :raw-html:`
` + Line Type * - Dimension of the :raw-html:`
` contingency table & the :raw-html:`
` total number of :raw-html:`
` @@ -1224,23 +761,6 @@ METplus Database of Statistics Grid-Stat - MCTC :raw-html:`
` MCTS - * - Number of cluster objects - - N_CLUS - - Diagnostic - - MODE - - MODE obj - * - Number of simple :raw-html:`
` - forecast objects - - N_FCST_SIMP - - Diagnostic - - MODE - - MODE obj - * - Number of simple :raw-html:`
` - observation objects - - N_OBS_SIMP - - Diagnostic - - MODE - - MODE obj * - Observation rate - O_RATE - Categorical @@ -1278,114 +798,6 @@ METplus Database of Statistics - Point-Stat :raw-html:`
` Grid-Stat - VCNT - * - Number of observed :raw-html:`
` - clusters - - obs_clus - - Diagnostic - - MODE - - MODE obj - * - Number of points used to :raw-html:`
` - define the hull of all of :raw-html:`
` - the cluster observation :raw-html:`
` - objects - - obs_clus :raw-html:`
` - _hull - - Diagnostic - - MODE - - MODE obj - * - Observation Cluster Convex :raw-html:`
` - Hull Point Latitude - - obs_clus :raw-html:`
` - _hull_lat - - Diagnostic - - MODE - - MODE obj - * - Observation Cluster Convex :raw-html:`
` - Hull Point Longitude - - obs_clus :raw-html:`
` - _hull_lon - - Diagnostic - - MODE - - MODE obj - * - Number of Observation :raw-html:`
` - Cluster Convex Hull Points - - obs_clus :raw-html:`
` - _hull_npts - - Diagnostic - - MODE - - MODE obj - * - Observation Cluster Convex :raw-html:`
` - Hull Starting Index - - obs_clus :raw-html:`
` - _hull_start - - Diagnostic - - MODE - - MODE obj - * - Observation Cluster Convex :raw-html:`
` - Hull Point X-Coordinate - - obs_clus :raw-html:`
` - _hull_x - - Diagnostic - - MODE - - MODE obj - * - Observation Cluster Convex :raw-html:`
` - Hull Point Y-Coordinate - - obs_clus :raw-html:`
` - _hull_y - - Diagnostic - - MODE - - MODE obj - * - Number of simple :raw-html:`
` - observation objects - - obs_simp - - Diagnostic - - MODE - - MODE obj - * - Number of points used :raw-html:`
` - to define the boundaries :raw-html:`
` - of the simple observation :raw-html:`
` - objects - - obs_simp :raw-html:`
` - _bdy - - Diagnostic - - MODE - - MODE obj - * - Observation Simple :raw-html:`
` - Boundary Point Latitude - - obs_simp :raw-html:`
` - _bdy_lat - - Diagnostic - - MODE - - MODE obj - * - Observation Simple :raw-html:`
` - Boundary Point Longitude - - obs_simp :raw-html:`
` - _bdy_lon - - Diagnostic - - MODE - - MODE obj - * - Number of Observation :raw-html:`
` - Simple Boundary Points - - obs_simp :raw-html:`
` - _bdy_npts - - Diagnostic - - MODE - - MODE obj - * - Number of points used to :raw-html:`
` - define the hull of the :raw-html:`
` - simple observation objects - - obs_simp :raw-html:`
` - _hull - - Diagnostic - - MODE - - MODE obj - * - Number of Observation :raw-html:`
` - Simple Convex Hull Points - - obs_simp :raw-html:`
` - _hull_npts - - Diagnostic - - MODE - - MODE obj * - Odds Ratio - ODDS - Categorical @@ -1401,19 +813,7 @@ METplus Database of Statistics - Continuous - Point-Stat :raw-html:`
` Grid-Stat - - VCNT - * - Observed energy squared :raw-html:`
` - for this scale - - OENERGY - - - - Wavelet-Stat - - ISC - * - Mean of absolute value :raw-html:`
` - of observed gradients - - OGBAR - - - - Grid-Stat - - GRAD + - VCNT * - Number of observation :raw-html:`
` when forecast is between :raw-html:`
` the ith and i+1th :raw-html:`
` @@ -1505,17 +905,27 @@ METplus Database of Statistics - Probability - Point-Stat :raw-html:`
` Grid-Stat - - PJC - * - Ratio of the nth percentile :raw-html:`
` - (INTENSITY_NN column) of :raw-html:`
` - intensity of the two :raw-html:`
` - objects - - PERCENTILE :raw-html:`
` - _INTENSITY :raw-html:`
` - _RATIO - - Diagnostic - - MODE - - MODE obj + - PJC + + +Statistics List P-R +___________________ + + +.. role:: raw-html(raw) + :format: html + +.. list-table:: Statistics List P-R + :widths: auto + :header-rows: 1 + + * - Statistics :raw-html:`
` + Long Name + - METplus Name + - Statistic Type + - Tools + - METplus :raw-html:`
` + Line Type * - Probability Integral :raw-html:`
` Transform - PIT @@ -1628,7 +1038,7 @@ METplus Database of Statistics - RELP_i - Ensemble - Ensemble-Stat - - RELP + - RELP * - Resolution - RESOLUTION - Probability @@ -1664,7 +1074,7 @@ METplus Database of Statistics - Continuous - Point-Stat :raw-html:`
` Grid-Stat - - CNT + - CNT * - Square root of MSVE - RMSVE - Continuous @@ -1717,6 +1127,26 @@ METplus Database of Statistics - Ensemble - Ensemble-Stat - RPS + + +Statistics List S-T +___________________ + + +.. role:: raw-html(raw) + :format: html + +.. list-table:: Statistics List S-T + :widths: auto + :header-rows: 1 + + * - Statistics :raw-html:`
` + Long Name + - METplus Name + - Statistic Type + - Tools + - METplus :raw-html:`
` + Line Type * - S1 score - S1 - Continuous @@ -1757,15 +1187,6 @@ METplus Database of Statistics - Point-Stat :raw-html:`
` Grid-Stat - CNT - * - Spatial distance between :raw-html:`
` - (𝑥,𝑦)(x,y) coordinates of :raw-html:`
` - object spacetime centroid - - SPACE :raw-html:`
` - _CENTROID :raw-html:`
` - _DIST - - Diagnostics - - MTD - - MTD 3D obs * - Absolute value of SPEED_ERR - SPEED :raw-html:`
` _ABSERR @@ -1773,11 +1194,6 @@ METplus Database of Statistics - Point-Stat :raw-html:`
` Grid-Stat - VCNT - * - Difference in object speeds - - SPEED_DELTA - - Diagnostics - - MTD - - MTD 3D obs * - Difference between the :raw-html:`
` length of the average :raw-html:`
` forecast wind vector and :raw-html:`
` @@ -1814,29 +1230,6 @@ METplus Database of Statistics - Ensemble-Stat - ECNT :raw-html:`
` ORANK - * - Difference in object :raw-html:`
` - starting time steps - - START_TIME :raw-html:`
` - _DELTA - - Diagnostic - - MTD - - MTD 3D obj - * - Symmetric difference of :raw-html:`
` - two objects :raw-html:`
` - (in grid squares) - - SYMMETRIC :raw-html:`
` - _DIFF - - Diagnostics - - MODE - - MODE obj - * - Difference in t index of :raw-html:`
` - object spacetime centroid - - TIME :raw-html:`
` - _CENTROID :raw-html:`
` - _DELTA - - Diagnostic - - MTD - - MTD 3D obj * - Track error of adeck :raw-html:`
` relative to bdeck (nm) - TK_ERR @@ -1849,6 +1242,26 @@ METplus Database of Statistics - Continuous - TC-Pairs - TCMPR + + + +Statistics List U-Z +___________________ + +.. role:: raw-html(raw) + :format: html + +.. list-table:: Statistics List U-Z + :widths: auto + :header-rows: 1 + + * - Statistics :raw-html:`
` + Long Name + - METplus Name + - Statistic Type + - Tools + - METplus :raw-html:`
` + Line Type * - Mean U-component :raw-html:`
` Forecast Anomaly - UFABAR @@ -1875,13 +1288,6 @@ METplus Database of Statistics - Point-Stat :raw-html:`
` Grid-Stat - PSTD - * - Union area of :raw-html:`
` - two objects :raw-html:`
` - (in grid squares) - - UNION_AREA - - Diagnostic - - MODE - - MODE obj * - Mean U-component :raw-html:`
` Observation Anomaly - UOABAR @@ -1992,7 +1398,7 @@ METplus Database of Statistics - VCNT * - Mean(vf-vc) - VFABAR - - Continous + - Continuous - Point-Stat :raw-html:`
` Grid-Stat - VAL1L2 @@ -2013,15 +1419,933 @@ METplus Database of Statistics - Continuous - Point-Stat :raw-html:`
` Grid-Stat - - VL1L2 - * - Integer count of the :raw-html:`
` - number of 3D “cells” :raw-html:`
` - in an object - - VOLUME - - Diagnostic - - MTD - - MTD 3D obj - * - Forecast object volume :raw-html:`
` + - VL1L2 + + +Diagnostics Database +==================== + + +.. Number of characters per line: + Statistic Name - no more that 32 characters + METplus Name - no more than 17 characters + Statistic Type - no more than 19 characters + METplus Line Type - currently unlimited (approx 33 characters) + +Diagnostics List A-B +____________________ + +.. role:: raw-html(raw) + :format: html + +.. list-table:: Diagnostics List A-B + :widths: auto + :header-rows: 1 + + * - Statistics :raw-html:`
` + Long Name + - METplus Name + - Statistic Type + - Tools + - METplus :raw-html:`
` + Line Type + * - Difference between the axis :raw-html:`
` + angles of two objects (in degrees) + - ANGLE_DIFF + - Diagnostic + - MODE + - MODE + * - Object area (in grid squares) + - AREA + - Diagnostic + - MODE :raw-html:`
` + MTD + - MODE obj + * - Forecast object area :raw-html:`
` + divided by the observation :raw-html:`
` + object area (unitless) + - AREA_RATIO + - Diagnostic + - MODE + - MODE obj + * - Area of the object :raw-html:`
` + that meet the object :raw-html:`
` + definition threshold :raw-html:`
` + criteria (in grid squares) + - AREA_THRESH + - Diagnostic + - MODE + - MODE obj + * - Absolute value of :raw-html:`
` + the difference :raw-html:`
` + between the aspect :raw-html:`
` + ratios of two objects :raw-html:`
` + (unitless) + - ASPECT_DIFF + - Diagnostic + - MODE + - MODE obj + * - Object axis angle :raw-html:`
` + (in degrees) + - AXIS_ANG + - Diagnostic + - MODE :raw-html:`
` + MTD + - MTD obj + * - Difference in spatial :raw-html:`
` + axis plane angles + - AXIS_DIFF + - Diagnostic + - MTD + - MTD obj + * - Blocking Index + - Blocking :raw-html:`
` + Index + - Diagnostic + - METplus Use :raw-html:`
` + Case + - n/a + * - Minimum distance between :raw-html:`
` + the boundaries of two objects + - BOUNDARY :raw-html:`
` + _DIST + - Diagnostic + - MODE + - MODE obj + +Diagnostics List C-E +____________________ + +.. role:: raw-html(raw) + :format: html + +.. list-table:: Diagnostics List C-E + :widths: auto + :header-rows: 1 + + * - Statistics :raw-html:`
` + Long Name + - METplus Name + - Statistic Type + - Tools + - METplus :raw-html:`
` + Line Type + * - Total great circle distance :raw-html:`
` + travelled by the 2D spatial :raw-html:`
` + centroid over the lifetime :raw-html:`
` + of the 3D object + - CDIST :raw-html:`
` + _TRAVELLED + - Diagnostic + - MTD + - MTD 3D obj + * - Distance between two :raw-html:`
` + objects centroids :raw-html:`
` + (in grid units) + - CENTROID :raw-html:`
` + _DIST + - Diagnostic + - MODE + - MODE obj + * - Latitude of centroid :raw-html:`
` + - CENTROID :raw-html:`
` + _LAT + - Diagnostic + - MTD :raw-html:`
` + MODE + - MTD 2D & 3D obj :raw-html:`
` + MODE obj + * - Longitude of centroid :raw-html:`
` + - CENTROID :raw-html:`
` + _LON + - Diagnostic + - MTD :raw-html:`
` + MODE + - MTD 2D & 3D obj :raw-html:`
` + MODE obj + * - Time coordinate of centroid + - CENTROID_T + - Diagnostic + - MTD + - MTD 3D obj + * - X coordinate of centroid :raw-html:`
` + - CENTROID_X + - Diagnostic + - MTD :raw-html:`
` + MODE + - MTD 2D & 3D obj :raw-html:`
` + MODE obj + * - Y coordinate of centroid :raw-html:`
` + - CENTROID_Y + - Diagnostic + - MTD :raw-html:`
` + MODE + - MTD 2D & 3D obj :raw-html:`
` + MODE obj + * - Space-Time :raw-html:`
` + Coherence Diagram + - Coherence :raw-html:`
` + Diagram + - Diagnostic + - METplus :raw-html:`
` + Use Case + - n/a + * - Ratio of the difference :raw-html:`
` + between the area of an :raw-html:`
` + object and the area of :raw-html:`
` + its convex hull divided :raw-html:`
` + by the area of the :raw-html:`
` + complex hull (unitless) + - COMPLEXITY + - Diagnostic + - MODE + - MODE obj + * - Ratio of complexities of :raw-html:`
` + two objects defined as :raw-html:`
` + the lesser of the forecast :raw-html:`
` + complexity divided by the :raw-html:`
` + observation complexity or :raw-html:`
` + its reciprocal (unitless) + - COMPLEXITY :raw-html:`
` + _RATIO + - Diagnostic + - MODE + - MODE obj + * - Minimum distance between :raw-html:`
` + the convex hulls of two :raw-html:`
` + objects (in grid units) + - CONVEX_HULL :raw-html:`
` + _DIST + - Diagnostic + - MODE + - MODE obj + * - Radius of curvature + - CURVATURE + - Diagnostic + - MODE + - MODE obj + * - Ratio of the curvature + - CURVATURE :raw-html:`
` + _RATIO + - Diagnostic + - MODE + - MODE obj + * - Center of curvature :raw-html:`
` + (in grid coordinates) + - CURVATURE :raw-html:`
` + _X + - Diagnostic + - MODE + - MODE obj + * - Center of curvature :raw-html:`
` + (in grid coordinates) + - CURVATURE :raw-html:`
` + _Y + - Diagnostic + - MODE + - MODE obj + * - Cloud Water / :raw-html:`
` + Precip Relationship + - CW/Precip :raw-html:`
` + Relationship + - Diagnostic + - Grid-Diag + - n/a + * - Difference in object :raw-html:`
` + direction of movement + - DIRECTION :raw-html:`
` + _DIFF + - Diagnostic + - MTD + - MTD 3D obj + * - Difference in the :raw-html:`
` + lifetimes of the :raw-html:`
` + two objects + - DURATION :raw-html:`
` + _DIFF + - Diagnostic + - MTD + - MTD 3D obj + * - Object end time + - END_TIME + - Diagnostic + - MTD + - MTD 3D obj + * - Difference in object :raw-html:`
` + ending time steps + - END_TIME :raw-html:`
` + _DELTA + - Diagnostic + - MTD + - MTD 3D obj + +Diagnostics List F +__________________ + +.. role:: raw-html(raw) + :format: html + +.. list-table:: Diagnostics List F + :widths: auto + :header-rows: 1 + + * - Statistics :raw-html:`
` + Long Name + - METplus Name + - Statistic Type + - Tools + - METplus :raw-html:`
` + Line Type + * - Number of forecast :raw-html:`
` + clusters + - FCST_CLUS + - Diagnostic + - MODE + - MODE obj + * - Number of points used to :raw-html:`
` + define the hull of all :raw-html:`
` + of the cluster forecast :raw-html:`
` + objects + - FCST_CLUS :raw-html:`
` + _HULL + - Diagnostic + - MODE + - MODE obj + * - Forecast Cluster Convex :raw-html:`
` + Hull Point Latitude + - FCST_CLUS :raw-html:`
` + _HULL_LAT + - Diagnostic + - MODE + - MODE obj + * - Forecast Cluster Convex :raw-html:`
` + Hull Point Longitude + - FCST_CLUS :raw-html:`
` + _HULL _LON + - Diagnostic + - MODE + - MODE obj + * - Number of Forecast :raw-html:`
` + Cluster Convex Hull Points + - FCST_CLUS :raw-html:`
` + _HULL_NPTS + - Diagnostic + - MODE + - MODE obj + * - Forecast Cluster Convex :raw-html:`
` + Hull Starting Index + - FCST_CLUS :raw-html:`
` + _HULL_START + - Diagnostic + - MODE + - MODE obj + * - Forecast Cluster Convex :raw-html:`
` + Hull Point X-Coordinate + - FCST_CLUS :raw-html:`
` + _HULL_X + - Diagnostic + - MODE + - MODE obj + * - Forecast Cluster Convex :raw-html:`
` + Hull Point Y-Coordinate + - FCST_CLUS :raw-html:`
` + _HULL_Y + - Diagnostic + - MODE + - MODE obj + * - Forecast Object Raw :raw-html:`
` + Values + - FCST_OBJ :raw-html:`
` + _RAW + - Diagnostic + - MODE + - MODE obj + * - Number of simple :raw-html:`
` + forecast objects + - FCST_SIMP + - Diagnostic + - MODE + - MODE obj + * - Number of points used :raw-html:`
` + to define the boundaries :raw-html:`
` + of all of the simple :raw-html:`
` + forecast objects + - FCST_SIMP :raw-html:`
` + _BDY + - Diagnostic + - MODE + - MODE obj + * - Forecast Simple :raw-html:`
` + Boundary Latitude + - FCST_SIMP :raw-html:`
` + _BDY_LAT + - Diagnostic + - MODE + - MODE obj + * - Forecast Simple :raw-html:`
` + Boundary Longitude + - FCST_SIMP :raw-html:`
` + _BDY_LON + - Diagnostic + - MODE + - MODE obj + * - Number of Forecast :raw-html:`
` + Simple Boundary Points + - FCST_SIMP :raw-html:`
` + _BDY_NPTS + - Diagnostic + - MODE + - MODE obj + * - Forecast Simple :raw-html:`
` + Boundary Starting Index + - FCST_SIMP :raw-html:`
` + _BDY_START + - Diagnostic + - MODE + - MODE obj + * - Forecast Simple :raw-html:`
` + Boundary X-Coordinate + - FCST_SIMP :raw-html:`
` + _BDY_X + - Diagnostic + - MODE + - MODE obj + * - Forecast Simple :raw-html:`
` + Boundary Y-Coordinate + - FCST_SIMP :raw-html:`
` + _BDY_Y + - Diagnostic + - MODE + - MODE obj + * - Number of points used to :raw-html:`
` + define the hull of all :raw-html:`
` + of the simple forecast :raw-html:`
` + objects + - FCST_SIMP :raw-html:`
` + _HULL + - Diagnostic + - MODE + - MODE obj + * - Forecast Simple Convex :raw-html:`
` + Hull Point Latitude + - FCST_SIMP :raw-html:`
` + _HULL_LAT + - Diagnostic + - MODE + - MODE obj + * - Forecast Simple Convex :raw-html:`
` + Hull Point Longitude + - FCST_SIMP :raw-html:`
` + _HULL_LON + - Diagnostic + - MODE + - MODE obj + * - Number of Forecast :raw-html:`
` + Simple Convex Hull Points + - FCST_SIMP :raw-html:`
` + _HULL_NPTS + - Diagnostic + - MODE + - MODE obj + * - Forecast Simple Convex :raw-html:`
` + Hull Starting Index + - FCST_SIMP :raw-html:`
` + _HULL_START + - Diagnostic + - MODE + - MODE obj + * - Forecast Simple Convex :raw-html:`
` + Hull Point X-Coordinate + - FCST_SIMP :raw-html:`
` + _HULL_X + - Diagnostic + - MODE + - MODE obj + * - Forecast Simple Convex :raw-html:`
` + Hull Point Y-Coordinate + - FCST_SIMP :raw-html:`
` + _HULL_Y + - Diagnostic + - MODE + - MODE obj + * - Number of thresholds :raw-html:`
` + applied to the forecast + - FCST :raw-html:`
` + _THRESH :raw-html:`
` + _LENGTH + - Diagnostic + - MODE + - MODE obj + * - Number of thresholds :raw-html:`
` + applied to the forecast + - FCST_THRESH :raw-html:`
` + _LENGTH + - Diagnostic + - MODE + - MODE obj + * - Forecast energy squared :raw-html:`
` + for this scale + - FENERGY + - + - Wavelet-Stat + - ISC + * - Mean of absolute value :raw-html:`
` + of forecast gradients + - FGBAR + - + - Grid-Stat + - GRAD + * - Ratio of forecast and :raw-html:`
` + observed gradients + - FGOG_RATIO + - + - Grid-Stat + - GRAD + * - Pratt’s Figure of Merit :raw-html:`
` + from observation to :raw-html:`
` + forecast + - FOM_FO + - Diagnostic + - Grid-Stat + - DMAP + * - Maximum of FOM_FO :raw-html:`
` + and FOM_OF + - FOM_MAX + - Diagnostic + - Grid-Stat + - DMAP + * - Mean of FOM_FO :raw-html:`
` + and FOM_OF :raw-html:`
` + - FOM_MEAN + - Diagnostic + - Grid-Stat + - DMAP + * - Minimum of FOM_FO :raw-html:`
` + and FOM_OF + - FOM_MIN + - Diagnostic + - Grid-Stat + - DMAP + * - Pratt’s Figure of Merit :raw-html:`
` + from forecast to :raw-html:`
` + observation + - FOM_OF + - Diagnostic + - Grid-Stat + - DMAP + + +Diagnostics List G-L +____________________ + +.. role:: raw-html(raw) + :format: html + +.. list-table:: Diagnostics List G-L + :widths: auto + :header-rows: 1 + + * - Statistics :raw-html:`
` + Long Name + - METplus Name + - Statistic Type + - Tools + - METplus :raw-html:`
` + Line Type + * - Distance between the :raw-html:`
` + forecast and Best track :raw-html:`
` + genesis events (km) + - GEN_DIST + - Diagnostic + - TC-Gen + - GENMPR + * - Forecast minus Best track :raw-html:`
` + genesis time in HHMMSS :raw-html:`
` + format + - GEN_TDIFF + - Diagnostic + - TC-Gen + - GENMPR + * - Hausdorff Distance + - HAUSDORFF + - Diagnostic + - Grid-Stat + - DMAP + * - Hovmoeller Diagram + - Hovmoeller + - Diagnostic + - METplus :raw-html:`
` + Use Case + - n/a + * - Best track genesis minus :raw-html:`
` + forecast initialization :raw-html:`
` + time in HHMMSS format + - INIT_TDIFF + - Diagnostic + - TC-Gen + - GENMPR + * - 10th, 25th, 50th, 75th, :raw-html:`
` + 90th, and user-specified :raw-html:`
` + percentiles of :raw-html:`
` + intensity of the raw :raw-html:`
` + field within the :raw-html:`
` + object or time slice + - INTENSITY :raw-html:`
` + _10, _25, :raw-html:`
` + _50, _75, :raw-html:`
` + _90, _NN + - Diagnostic + - MODE + - MODE obj + * - Sum of the intensities of :raw-html:`
` + the raw field within the :raw-html:`
` + object (variable units) + - INTENSITY :raw-html:`
` + _SUM + - Diagnostic + - MODE + - MODE obj + * - Total interest for this :raw-html:`
` + object pair + - INTEREST + - Diagnostic + - MTD :raw-html:`
` + MODE + - MTD 3D obj :raw-html:`
` + MODE obj + * - Intersection area of two :raw-html:`
` + objects (in grid squares) + - INTERSECT :raw-html:`
` + ION_AREA + - Diagnostic + - MODE + - MODE obj + * - Ratio of intersection area :raw-html:`
` + to the lesser of the :raw-html:`
` + forecast and observation :raw-html:`
` + object areas (unitless) + - INTERSECT :raw-html:`
` + ION_OVER :raw-html:`
` + _AREA + - Diagnostic + - MODE + - MODE obj + * - “Volume” of object :raw-html:`
` + intersection + - INTERSECT :raw-html:`
` + ION_VOLUME + - Diagnostic + - MTD + - MTD 3D obj + * - The intensity scale :raw-html:`
` + skill score + - ISC + - + - Wavelet-Stat + - ISC + * - The scale at which all :raw-html:`
` + information following :raw-html:`
` + applies + - ISCALE + - + - Wavelet-Stat + - ISC + * - Joint Probability :raw-html:`
` + Distribution between :raw-html:`
` + variable + - Joint PDF :raw-html:`
` + to Diagnose :raw-html:`
` + Relationship + - Diagnostic + - Grid-Diag + - n/a + * - Dimension of the latitude + - LAT + - Diagnostic + - MODE + - MODE obj + * - Length of the :raw-html:`
` + enclosing rectangle + - LENGTH + - Diagnostic + - MODE + - MODE obj + * - Dimension of the longitude + - LON + - Diagnostic + - MODE + - MODE obj + + +Diagnostics List M-O +____________________ + +.. role:: raw-html(raw) + :format: html + +.. list-table:: Diagnostics List M-O + :widths: auto + :header-rows: 1 + + * - Statistics :raw-html:`
` + Long Name + - METplus Name + - Statistic Type + - Tools + - METplus :raw-html:`
` + Line Type + * - Meridional Means + - Meridional Means + - Diagnostic + - METplus Use Case + - n/a + * - Mean of maximum of :raw-html:`
` + absolute values of :raw-html:`
` + forecast and observed :raw-html:`
` + gradients + - MGBAR + - + - Grid-Stat + - GRAD + * - Number of cluster objects + - N_CLUS + - Diagnostic + - MODE + - MODE obj + * - Number of simple :raw-html:`
` + forecast objects + - N_FCST_SIMP + - Diagnostic + - MODE + - MODE obj + * - Number of simple :raw-html:`
` + observation objects + - N_OBS_SIMP + - Diagnostic + - MODE + - MODE obj + * - Number of observed :raw-html:`
` + clusters + - OBS_CLUS + - Diagnostic + - MODE + - MODE obj + * - Number of points used to :raw-html:`
` + define the hull of all of :raw-html:`
` + the cluster observation :raw-html:`
` + objects + - OBS_CLUS :raw-html:`
` + _HULL + - Diagnostic + - MODE + - MODE obj + * - Observation Cluster Convex :raw-html:`
` + Hull Point Latitude + - OBS_CLUS :raw-html:`
` + _HULL_LAT + - Diagnostic + - MODE + - MODE obj + * - Observation Cluster Convex :raw-html:`
` + Hull Point Longitude + - OBS_CLUS :raw-html:`
` + _HULL_LON + - Diagnostic + - MODE + - MODE obj + * - Number of Observation :raw-html:`
` + Cluster Convex Hull Points + - OBS_CLUS :raw-html:`
` + _HULL_NPTS + - Diagnostic + - MODE + - MODE obj + * - Observation Cluster Convex :raw-html:`
` + Hull Starting Index + - OBS_CLUS :raw-html:`
` + _HULL_START + - Diagnostic + - MODE + - MODE obj + * - Observation Cluster Convex :raw-html:`
` + Hull Point X-Coordinate + - OBS_CLUS :raw-html:`
` + _HULL_X + - Diagnostic + - MODE + - MODE obj + * - Observation Cluster Convex :raw-html:`
` + Hull Point Y-Coordinate + - OBS_CLUS :raw-html:`
` + _HULL_Y + - Diagnostic + - MODE + - MODE obj + * - Number of simple :raw-html:`
` + observation objects + - OBS_SIMP + - Diagnostic + - MODE + - MODE obj + * - Number of points used :raw-html:`
` + to define the boundaries :raw-html:`
` + of the simple observation :raw-html:`
` + objects + - OBS_SIMP :raw-html:`
` + _BDY + - Diagnostic + - MODE + - MODE obj + * - Observation Simple :raw-html:`
` + Boundary Point Latitude + - OBS_SIMP :raw-html:`
` + _BDY_LAT + - Diagnostic + - MODE + - MODE obj + * - Observation Simple :raw-html:`
` + Boundary Point Longitude + - OBS_SIMP :raw-html:`
` + _BDY_LON + - Diagnostic + - MODE + - MODE obj + * - Number of Observation :raw-html:`
` + Simple Boundary Points + - OBS_SIMP :raw-html:`
` + _BDY_NPTS + - Diagnostic + - MODE + - MODE obj + * - Number of points used to :raw-html:`
` + define the hull of the :raw-html:`
` + simple observation objects + - OBS_SIMP :raw-html:`
` + _HULL + - Diagnostic + - MODE + - MODE obj + * - Number of Observation :raw-html:`
` + Simple Convex Hull Points + - OBS_SIMP :raw-html:`
` + _HULL_NPTS + - Diagnostic + - MODE + - MODE obj + * - Observed energy squared :raw-html:`
` + for this scale + - OENERGY + - + - Wavelet-Stat + - ISC + * - Mean of absolute value :raw-html:`
` + of observed gradients + - OGBAR + - + - Grid-Stat + - GRAD + * - OLR-based MJO Index + - OMI + - Diagnostic + - METplus :raw-html:`
` + Use Case + - n/a + + +Diagnostics List P-Z +____________________ + +.. role:: raw-html(raw) + :format: html + +.. list-table:: Diagnostics List P-Z + :widths: auto + :header-rows: 1 + + * - Statistics :raw-html:`
` + Long Name + - METplus Name + - Statistic Type + - Tools + - METplus :raw-html:`
` + Line Type + * - Ratio of the nth percentile :raw-html:`
` + (INTENSITY_NN column) of :raw-html:`
` + intensity of the two :raw-html:`
` + objects + - PERCENTILE :raw-html:`
` + _INTENSITY :raw-html:`
` + _RATIO + - Diagnostic + - MODE + - MODE obj + * - Phase Diagram :raw-html:`
` + for RMM and OMI + - Phase :raw-html:`
` + Diagram + - Diagnostic + - METplus :raw-html:`
` + Use Case + - n/a + * - Realtime Multivariate :raw-html:`
` + MJO Index + - RMM + - Diagnostic + - METplus :raw-html:`
` + Use Case + - n/a + * - Spatial distance between :raw-html:`
` + (𝑥,𝑦)(x,y) coordinates of :raw-html:`
` + object spacetime centroid + - SPACE :raw-html:`
` + _CENTROID :raw-html:`
` + _DIST + - Diagnostic + - MTD + - MTD 3D obs + * - Difference in object speeds + - SPEED_DELTA + - Diagnostic + - MTD + - MTD 3D obs + * - Difference in object :raw-html:`
` + starting time steps + - START_TIME :raw-html:`
` + _DELTA + - Diagnostic + - MTD + - MTD 3D obj + * - Symmetric difference of :raw-html:`
` + two objects :raw-html:`
` + (in grid squares) + - SYMMETRIC :raw-html:`
` + _DIFF + - Diagnostic + - MODE + - MODE obj + * - Difference in t index of :raw-html:`
` + object spacetime centroid + - TIME :raw-html:`
` + _CENTROID :raw-html:`
` + _DELTA + - Diagnostic + - MTD + - MTD 3D obj + * - Union area of :raw-html:`
` + two objects :raw-html:`
` + (in grid squares) + - UNION_AREA + - Diagnostic + - MODE + - MODE obj + * - Integer count of the :raw-html:`
` + number of 3D “cells” :raw-html:`
` + in an object + - VOLUME + - Diagnostic + - MTD + - MTD 3D obj + * - Forecast object volume :raw-html:`
` divided by observation :raw-html:`
` object volume - VOLUME :raw-html:`
` @@ -2029,6 +2353,13 @@ METplus Database of Statistics - Diagnostic - MTD - MTD 3D obj + * - Weather Regime Index + - Weather :raw-html:`
` + Regime Index + - Diagnostic + - METplus :raw-html:`
` + Use Case + - n/a * - Width of the enclosing :raw-html:`
` rectangle (in grid units) - WIDTH @@ -2066,6 +2397,12 @@ METplus Database of Statistics - TC-Pairs - PROBRIRW :raw-html:`
` TCMPR + * - Zonal Means + - Zonal Means + - Diagnostic + - METplus :raw-html:`
` + Use Case + - n/a * - Zhu’s Measure from :raw-html:`
` observation to forecast - ZHU_FO @@ -2096,3 +2433,4 @@ METplus Database of Statistics - Diagnostic - Grid-Stat - DMAP + diff --git a/docs/Users_Guide/systemconfiguration.rst b/docs/Users_Guide/systemconfiguration.rst index 999b6ec049..96f5061ca0 100644 --- a/docs/Users_Guide/systemconfiguration.rst +++ b/docs/Users_Guide/systemconfiguration.rst @@ -1387,7 +1387,7 @@ a single run. If the MET tool allows it and METplus Wrappers is configured accordingly, these two comparisons would be configured in a single run. Read explicit time dimension from a NetCDF level -"""""""""""""""""""""""""""""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ If the input NetCDF data contains a time dimension, the time can be specified in the level value. The MET tool will find the data for the time requested:: @@ -1407,6 +1407,40 @@ functionality). The time can be specified based on the current run time, i.e.:: In this example, {valid?fmt=%Y%m%d_%H%M%S} will be substituted with the valid time of the current run. +Substituting Current Level +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +When using Python Embedding to pass in data for a field, one may want to +call the same Python script for each vertical level specifying the level +string for each call. In this case, a list of levels can be specified using +:term:`FCST_VAR_LEVELS` and the value can be substituted into the +corresponding :term:`FCST_VAR_NAME` using {fcst_level}:: + + [config] + FCST_VAR1_NAME = {INPUT_BASE}/myscripts/read_nc2xr.py {INPUT_BASE}/mydata/forecast_file.nc4 TMP {valid?fmt=%Y%m%d_%H%M} {fcst_level} + FCST_VAR1_LEVELS = P1000,P850,P700,P500,P250,P100 + +This will call the Python script 6 times: + +* {INPUT_BASE}/myscripts/read_nc2xr.py {INPUT_BASE}/mydata/forecast_file.nc4 TMP {valid?fmt=%Y%m%d_%H%M} P1000 +* {INPUT_BASE}/myscripts/read_nc2xr.py {INPUT_BASE}/mydata/forecast_file.nc4 TMP {valid?fmt=%Y%m%d_%H%M} P850 +* {INPUT_BASE}/myscripts/read_nc2xr.py {INPUT_BASE}/mydata/forecast_file.nc4 TMP {valid?fmt=%Y%m%d_%H%M} P700 +* {INPUT_BASE}/myscripts/read_nc2xr.py {INPUT_BASE}/mydata/forecast_file.nc4 TMP {valid?fmt=%Y%m%d_%H%M} P500 +* {INPUT_BASE}/myscripts/read_nc2xr.py {INPUT_BASE}/mydata/forecast_file.nc4 TMP {valid?fmt=%Y%m%d_%H%M} P250 +* {INPUT_BASE}/myscripts/read_nc2xr.py {INPUT_BASE}/mydata/forecast_file.nc4 TMP {valid?fmt=%Y%m%d_%H%M} P100 + +This only applies if the wrapper runs once per field name/level combination +such as MODE or if the wrapper is configured to do so, +for example GridStat using :term:`GRID_STAT_ONCE_PER_FIELD`. + +The same logic applies for observation data using +:term:`OBS_VAR_NAME`, :term:`OBS_VAR_LEVELS`, and {obs_level}. + +To reference the current field name and/or level in another configuration +variable such as :term:`MODE_OUTPUT_PREFIX`, use +{CURRENT_FCST_NAME}, {CURRENT_FCST_LEVEL}, {CURRENT_OBS_NAME}, +and/or {CURRENT_OBS_LEVEL}. + :term:`FCST_VAR_THRESH` / :term:`OBS_VAR_THRESH` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1454,6 +1488,56 @@ fcst dictionary item. If this is the case, then use these variables. If it is not set, the values in the corresponding FCST_VAR_[NAME/LEVELS/THRESH/OPTIONS] will be used in the ens dictionary. +Probabilistic Forecast Fields +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If processing probabilistic forecast fields, there are additional configuration +variables that are used to properly format the field info that is passed into +the wrapped MET configuration files. +:term:`FCST_IS_PROB` is used to process probabilistic data:: + + [config] + FCST_IS_PROB = True + FCST_VAR1_NAME = APCP_24_A24_ENS_FREQ_gt0.0 + FCST_VAR1_LEVELS = "(*,*)" + +will add the following to the MET config file:: + + fcst = {field = [{ name="APCP_24_A24_ENS_FREQ_gt0.0"; level="(*,*)"; prob=TRUE; cat_thresh=[ ==0.1 ]; }];} + +The cat_thresh value defaults to ==0.1 and defines the size of the Nx2 probabilistic contingency table. +It is set by :term:`FCST_GRID_STAT_PROB_THRESH` (for GridStat):: + + [config] + FCST_IS_PROB = True + FCST_VAR1_NAME = APCP + FCST_VAR1_LEVELS = "(*,*)" + FCST_GRID_STAT_PROB_THRESH = ==0.2 + +will add the following to the MET config file:: + + fcst = {field = [{ name="APCP"; level="(*,*)"; prob=TRUE; cat_thresh=[ ==0.2 ]; }];} + +Some GRIB files contain probabilistic field information in the +Product Definition Section (PDS). The format of the fcst.field info to read +these data expect the name to be set to "PROB" and the field name/level values +are set inside a prob dictionary. +If this is the case, then :term:`FCST_PROB_IN_GRIB_PDS` should be set to True. +At least 1 threshold must be set with :term:`FCST_VAR_THRESH` in this case. +The threshold value will be formatted in the prob dictionary using +thresh_lo and/or thresh_hi values:: + + [config] + FCST_IS_PROB = True + FCST_PROB_IN_GRIB_PDS = True + FCST_VAR1_NAME = APCP + FCST_VAR1_LEVELS = A03 + FCST_VAR1_THRESH = gt12.7 + +will add the following to the MET config file:: + + fcst = {field = [{ name="PROB"; level="A03"; prob={ name="APCP"; thresh_lo=12.7; } cat_thresh=[ ==0.1 ]; }];} + Wrapper Specific Field Info ^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/_static/marine_and_cryosphere-UserScript_fcstRTOFS_obsAOML_calcTransport.png b/docs/_static/marine_and_cryosphere-UserScript_fcstRTOFS_obsAOML_calcTransport.png new file mode 100644 index 0000000000..68df6c1143 Binary files /dev/null and b/docs/_static/marine_and_cryosphere-UserScript_fcstRTOFS_obsAOML_calcTransport.png differ diff --git a/docs/conf.py b/docs/conf.py index 92504c20b6..e1d3d16a3c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -27,9 +27,9 @@ author = 'UCAR/NCAR, NOAA, CSU/CIRA, and CU/CIRES' # list of contributing authors currently involved with the project -# rotate this list of authors for each release +# To rotate this list of authors for each official release, +# move the first author to the end of the list CURRENT_AUTHORS = [ - 'Minna Win-Gildenmeister', 'George McCabe', 'Julie Prestopnik', 'John Opatz', @@ -41,6 +41,7 @@ 'Hank Fisher', 'Lisa Goodrich', 'Dan Adriaansen', + 'Minna Win-Gildenmeister', ] # list of former contributors who are no longer involved with the project diff --git a/docs/use_cases/model_applications/convection_allowing_models/EnsembleStat_fcstHRRRE_obsHRRRE_Sfc_MultiField.py b/docs/use_cases/model_applications/convection_allowing_models/EnsembleStat_fcstHRRRE_obsHRRRE_Sfc_MultiField.py index fc115a336c..78cc9bd9d3 100644 --- a/docs/use_cases/model_applications/convection_allowing_models/EnsembleStat_fcstHRRRE_obsHRRRE_Sfc_MultiField.py +++ b/docs/use_cases/model_applications/convection_allowing_models/EnsembleStat_fcstHRRRE_obsHRRRE_Sfc_MultiField.py @@ -1,11 +1,10 @@ """ Ensemble-Stat: Ensemble Statistics using Obs Uncertainty -============================================================================= +======================================================== model_applications/ -convection_allowing_model/ -EnsembleStat_fcstHRRRE_obsHRRRE_Sfc -_Sfc_MultiField.conf +convection_allowing_models/ +EnsembleStat_fcstHRRRE_obsHRRRE_Sfc_MultiField.conf """ ############################################################################## diff --git a/docs/use_cases/model_applications/marine_and_cryosphere/UserScript_fcstRTOFS_obsAOML_calcTransport.py b/docs/use_cases/model_applications/marine_and_cryosphere/UserScript_fcstRTOFS_obsAOML_calcTransport.py new file mode 100644 index 0000000000..73709c5a56 --- /dev/null +++ b/docs/use_cases/model_applications/marine_and_cryosphere/UserScript_fcstRTOFS_obsAOML_calcTransport.py @@ -0,0 +1,159 @@ +""" +UserScript: Python Script to compute cable transport +======================================================= + +model_applications/marine_and_cryosphere/UserScript_fcstRTOFS_obsAOML_calcTransport.conf + +""" +############################################################################## +# Scientific Objective +# -------------------- +# The Florida Current flows northward along the eastern Florida coast and feeds to the Gulf Stream. More info can +# be obtained from: https://www.aoml.noaa.gov/phod/floridacurrent/index.php +# +# This use case utilizes a Python script to calculate transport (units Sv) variations of the Florida current +# using a submarine cable and snapshot estimates made by shipboard instruments. The code compares the transport +# using RTOFS data and compare it with the AOML cable transport data and computes BIAS, RMSE, CORRELATION, and +# Scatter Index. The operational code utilizes 21 days of data and computes 7 day statistics. +# For the use case 3 days of data are utilized. The valid date is passed though an argument. The valid date +# is the last processed day i.e. the code grabs 3 previous days of data. + +############################################################################## +# Datasets +# --------------------- +# +# | **Forecast:** RTOFS u(3zuio) amd ,v(3zvio) files via Python Embedding script/file +# +# | **Observations:** AOML Florida Current data via Python Embedding script/file +# +# +# | **Location:** All of the input data required for this use case can be found in the met_test sample data tarball. Click here to the METplus releases page and download sample data for the appropriate release: https://github.com/dtcenter/METplus/releases +# | This tarball should be unpacked into the directory that you will set the value of INPUT_BASE. See `Running METplus`_ section for more information. +# +# | **Data Source:** NOMADS RTOFS Global + Daily mean transport (https://www.aoml.noaa.gov/phod/floridacurrent/data_access.php)+ Eightmilecable (static, provided with the use case) + +############################################################################## +# External Dependencies +# --------------------- +# +# You will need to use a version of Python 3.6+ that has the following packages installed: +# +# * scikit-learn +# * pyproj +# +# If the version of Python used to compile MET did not have these libraries at the time of compilation, you will need to add these packages or create a new Python environment with these packages. +# +# If this is the case, you will need to set the MET_PYTHON_EXE environment variable to the path of the version of Python you want to use. If you want this version of Python to only apply to this use case, set it in the [user_env_vars] section of a METplus configuration file.:: +# +# [user_env_vars] +# MET_PYTHON_EXE = /path/to/python/with/required/packages/bin/python + +############################################################################## +# METplus Components +# ------------------ +# +# This use case utilizes the METplus UserScript wrapper to generate a +# command to run with Python Embedding for the specified valid time. + +############################################################################## +# METplus Workflow +# ---------------- +# +# This use case uses UserScript. All the gridded data being pulled from the files via Python Embedding. +# All of the desired statistics are in the log file. +# It processes the following run time: +# +# | **Valid:** 2021-10-28 +# +# The code grabs the 20211028, 20211027, and 20211026 24 hour RTOFS files. + +############################################################################## +# METplus Configuration +# --------------------- +# +# METplus first loads all of the configuration files found in parm/metplus_config, +# then it loads any configuration files passed to METplus via the command line +# i.e. -c parm/use_cases/model_applications/marine_and_cryosphere/UserScript_fcstRTOFS_obsAOML_calcTransport.conf +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/use_cases/model_applications/marine_and_cryosphere/UserScript_fcstRTOFS_obsAOML_calcTransport.conf + +############################################################################## +# MET Configuration +# --------------------- +# +# None. All of the processing is completed in the UserScript +# + +############################################################################## +# User Script +# ---------------- +# +# This use case uses one Python script to read forecast and observation data +# as well as processing the desired statistics. +# +# parm/use_cases/model_applications/marine_and_cryosphere/UserScript_fcstRTOFS_obsAOML_calcTransport/read_aomlcable_rtofs_transport.py +# +# .. highlight:: python +# .. literalinclude:: ../../../../parm/use_cases/model_applications/marine_and_cryosphere/UserScript_fcstRTOFS_obsAOML_calcTransport/read_aomlcable_rtofs_transport.py +# + +############################################################################## +# Running METplus +# --------------- +# +# This use case can be run two ways: +# +# 1) Passing in UserScript_fcstRTOFS_obsAOML_calcTransport.conf then a user-specific system configuration file:: +# +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/marine_and_cryosphere/UserScript_fcstRTOFS_obsAOML_calcTransport.conf /path/to/user_system.conf +# +# 2) Modifying the configurations in parm/metplus_config, then passing in UserScript_fcstRTOFS_obsAOML_calcTransport.conf:: +# +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/marine_and_cryosphere/UserScript_fcstRTOFS_obsAOML_calcTransport.conf +# +# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: +# +# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases +# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions +# * **MET_INSTALL_DIR** - Path to location where MET is installed locally +# +# Example User Configuration File:: +# +# [config] +# INPUT_BASE = /path/to/sample/input/data +# OUTPUT_BASE = /path/to/output/dir +# MET_INSTALL_DIR = /path/to/met-X.Y +# +# + +############################################################################## +# Expected Output +# --------------- +# +# A successful run will output the following both to the screen and to the logfile:: +# +# INFO: METplus has successfully finished running. +# +# Refer to the value set for **OUTPUT_BASE** to find where the output data was generated. +# Output for use case will be found in calc_transport (relative to **OUTPUT_BASE**) +# and will contain the following files: +# +# * calc_transport.log + +############################################################################## +# Keywords +# -------- +# +# .. note:: +# +# * UserScriptUseCase +# * PythonEmbeddingFileUseCase +# * MarineAndCryosphereAppUseCase +# +# Navigate to the :ref:`quick-search` page to discover other similar use cases. +# +# +# +# sphinx_gallery_thumbnail_path = '_static/marine_and_cryosphere-UserScript_fcstRTOFS_obsAOML_calcTransport.png' + diff --git a/internal_tests/pytests/config_metplus/test_config_metplus.py b/internal_tests/pytests/config_metplus/test_config_metplus.py index 1c55fbc973..f0e669443b 100644 --- a/internal_tests/pytests/config_metplus/test_config_metplus.py +++ b/internal_tests/pytests/config_metplus/test_config_metplus.py @@ -991,4 +991,38 @@ def test_get_process_list_instances(metplus_config, input_list, expected_list): conf = metplus_config() conf.set('config', 'PROCESS_LIST', input_list) output_list = config_metplus.get_process_list(conf) - assert(output_list == expected_list) \ No newline at end of file + assert(output_list == expected_list) + +def test_getraw_sub_and_nosub(metplus_config): + raw_string = '{MODEL}_{CURRENT_FCST_NAME}' + sub_actual = 'FCST_NAME' + + config = metplus_config() + config.set('config', 'MODEL', 'FCST') + config.set('config', 'CURRENT_FCST_NAME', 'NAME') + config.set('config', 'OUTPUT_PREFIX', raw_string) + nosub_value = config.getraw('config', 'OUTPUT_PREFIX', sub_vars=False) + assert nosub_value == raw_string + + sub_value = config.getraw('config', 'OUTPUT_PREFIX', sub_vars=True) + assert sub_value == sub_actual + +def test_getraw_instance_with_unset_var(metplus_config): + """! Replicates bug where CURRENT_FCST_NAME is substituted with + an empty string when copied from an instance section + """ + pytest.skip() + instance = 'my_section' + config = metplus_config() + config.set('config', 'MODEL', 'FCST') + + config.add_section(instance) + config.set('config', 'CURRENT_FCST_NAME', '') + config.set(instance, 'OUTPUT_PREFIX', '{MODEL}_{CURRENT_FCST_NAME}') + new_config = ( + config_metplus.replace_config_from_section(config, + instance, + required=False) + ) + new_config.set('config', 'CURRENT_FCST_NAME', 'NAME') + assert new_config.getraw('config', 'OUTPUT_PREFIX') == 'FCST_NAME' diff --git a/internal_tests/pytests/pb2nc/test_pb2nc_wrapper.py b/internal_tests/pytests/pb2nc/test_pb2nc_wrapper.py index 2b7e9ed0fa..909baa88d4 100644 --- a/internal_tests/pytests/pb2nc/test_pb2nc_wrapper.py +++ b/internal_tests/pytests/pb2nc/test_pb2nc_wrapper.py @@ -189,6 +189,12 @@ def test_find_input_files(metplus_config, offsets, offset_to_find): 'OBS_PB2NC_WINDOW_END': '1800', }, {'METPLUS_OBS_WINDOW_DICT': 'obs_window = {beg = -1800;end = 1800;}'}), + # test legacy PB2NC_WINDOW_[BEGIN/END] + ({'PB2NC_WINDOW_BEGIN': '-1800', }, + {'METPLUS_OBS_WINDOW_DICT': 'obs_window = {beg = -1800;}'}), + + ({'PB2NC_WINDOW_END': '1800', }, + {'METPLUS_OBS_WINDOW_DICT': 'obs_window = {end = 1800;}'}), ({'PB2NC_MASK_GRID': 'FULL', }, {'METPLUS_MASK_DICT': 'mask = {grid = "FULL";}'}), diff --git a/internal_tests/use_cases/all_use_cases.txt b/internal_tests/use_cases/all_use_cases.txt index f4bfbed186..32c4609407 100644 --- a/internal_tests/use_cases/all_use_cases.txt +++ b/internal_tests/use_cases/all_use_cases.txt @@ -93,6 +93,7 @@ Category: marine_and_cryosphere 3::GridStat_fcstRTOFS_obsSMOS_climWOA_sss::model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsSMOS_climWOA_sss.conf:: icecover_env, py_embed 4::GridStat_fcstRTOFS_obsSMAP_climWOA_sss::model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsSMAP_climWOA_sss.conf:: icecover_env, py_embed 5::GridStat_fcstRTOFS_obsAVISO_climHYCOM_ssh::model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsAVISO_climHYCOM_ssh.conf:: icecover_env, py_embed +6::UserScript_fcstRTOFS_obsAOML_calcTransport::model_applications/marine_and_cryosphere/UserScript_fcstRTOFS_obsAOML_calcTransport.conf:: icecover_env, py_embed #X::GridStat_fcstRTOFS_obsGHRSST_climWOA_sst::model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsGHRSST_climWOA_sst.conf, model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsGHRSST_climWOA_sst/ci_overrides.conf:: icecover_env, py_embed diff --git a/metplus/RELEASE_DATE b/metplus/RELEASE_DATE index 2ae3b32e7d..874dd2e688 100644 --- a/metplus/RELEASE_DATE +++ b/metplus/RELEASE_DATE @@ -1 +1 @@ -20210510 \ No newline at end of file +20220314 \ No newline at end of file diff --git a/metplus/VERSION b/metplus/VERSION index c5d7cae081..3d298f67af 100644 --- a/metplus/VERSION +++ b/metplus/VERSION @@ -1 +1 @@ -4.1.0-rc1-dev \ No newline at end of file +5.0.0-beta1-dev diff --git a/metplus/util/config_metplus.py b/metplus/util/config_metplus.py index 353eed0d31..3ce629bdc2 100644 --- a/metplus/util/config_metplus.py +++ b/metplus/util/config_metplus.py @@ -458,14 +458,14 @@ def replace_config_from_section(config, section, required=True): for key in all_configs: new_config.set(section_to_copy, key, - config.getraw(section_to_copy, key)) + config.getraw(section_to_copy, key, sub_vars=False)) # override values in [config] with values from {section} all_configs = config.keys(section) for key in all_configs: new_config.set('config', key, - config.getraw(section, key)) + config.getraw(section, key, sub_vars=False)) return new_config @@ -585,7 +585,7 @@ def remove_current_vars(self): self._conf.remove_option('config', current_var) # override get methods to perform additional error checking - def getraw(self, sec, opt, default='', count=0): + def getraw(self, sec, opt, default='', count=0, sub_vars=True): """ parse parameter and replace any existing parameters referenced with the value (looking in same section, then config, dir, and os environment) @@ -614,6 +614,10 @@ def getraw(self, sec, opt, default='', count=0): self.check_default(sec, opt, default) return default + # if not substituting values of other variables return value + if not sub_vars: + return in_template + # get inner-most tags that could potentially be other variables match_list = re.findall(r'\{([^}{]*)\}', in_template) for var_name in match_list: @@ -1320,14 +1324,6 @@ def check_for_deprecated_met_config(config): sed_cmds = [] all_good = True - # set CURRENT_* METplus variables in case they are referenced in a - # METplus config variable and not already set - for fcst_or_obs in ['FCST', 'OBS']: - for name_or_level in ['NAME', 'LEVEL']: - current_var = f'CURRENT_{fcst_or_obs}_{name_or_level}' - if not config.has_option('config', current_var): - config.set('config', current_var, '') - # check if *_CONFIG_FILE if set in the METplus config file and check for # deprecated environment variables in those files met_config_keys = [key for key in config.keys('config') diff --git a/metplus/util/met_config.py b/metplus/util/met_config.py index 55e753883d..2b99e54101 100644 --- a/metplus/util/met_config.py +++ b/metplus/util/met_config.py @@ -165,6 +165,12 @@ def add_met_config_dict(config, app_name, output_dict, dict_name, items): metplus_configs.append( f"OBS_{app_name}_WINDOW_{suffix}".upper() ) + + # also add support for legacy PB2NC_WINDOW_[BEGIN/END] + metplus_configs.append( + f"{app_name}_WINDOW_{suffix}".upper() + ) + # also add OBS_WINDOW_[BEGIN/END] metplus_configs.append(f"OBS_WINDOW_{suffix}") diff --git a/metplus/wrappers/command_builder.py b/metplus/wrappers/command_builder.py index 29214acfc2..c0c7d0475e 100755 --- a/metplus/wrappers/command_builder.py +++ b/metplus/wrappers/command_builder.py @@ -328,9 +328,11 @@ def handle_obs_window_legacy(self, c_dict): ('END', 5400)] app = self.app_name.upper() + # check {app}_WINDOW_{edge} to support PB2NC_WINDOW_[BEGIN/END] for edge, default_val in edges: input_list = [f'OBS_{app}_WINDOW_{edge}', f'{app}_OBS_WINDOW_{edge}', + f'{app}_WINDOW_{edge}', f'OBS_WINDOW_{edge}', ] output_key = f'OBS_WINDOW_{edge}' diff --git a/parm/use_cases/met_tool_wrapper/Example/Example.conf b/parm/use_cases/met_tool_wrapper/Example/Example.conf index 023c6d5acd..6286e47ae5 100644 --- a/parm/use_cases/met_tool_wrapper/Example/Example.conf +++ b/parm/use_cases/met_tool_wrapper/Example/Example.conf @@ -58,4 +58,4 @@ EXAMPLE_INPUT_DIR = /dir/containing/example/data [filename_templates] # Fake template to use to look for input data. This template is substituted with the time information of each # run time that is executed -EXAMPLE_INPUT_TEMPLATE = {init?fmt=%Y%m%d}/file_{init?fmt=%Y%m%d}_{init?fmt=%2H}_F{lead?fmt=%3H}.{custom?fmt=%s} \ No newline at end of file +EXAMPLE_INPUT_TEMPLATE = {init?fmt=%Y%m%d}/file_{init?fmt=%Y%m%d}_{init?fmt=%H}_F{lead?fmt=%3H}.{custom?fmt=%s} \ No newline at end of file diff --git a/parm/use_cases/met_tool_wrapper/PB2NC/PB2NC.conf b/parm/use_cases/met_tool_wrapper/PB2NC/PB2NC.conf index 591058aa8a..a10a30caee 100644 --- a/parm/use_cases/met_tool_wrapper/PB2NC/PB2NC.conf +++ b/parm/use_cases/met_tool_wrapper/PB2NC/PB2NC.conf @@ -15,7 +15,7 @@ PB2NC_OFFSETS = 12 PB2NC_SKIP_IF_OUTPUT_EXISTS = True PB2NC_INPUT_DIR = {INPUT_BASE}/met_test/data/sample_obs/prepbufr -PB2NC_INPUT_TEMPLATE = ndas.t{da_init?fmt=%2H}z.prepbufr.tm{offset?fmt=%2H}.{da_init?fmt=%Y%m%d}.nr +PB2NC_INPUT_TEMPLATE = ndas.t{da_init?fmt=%H}z.prepbufr.tm{offset?fmt=%2H}.{da_init?fmt=%Y%m%d}.nr PB2NC_OUTPUT_DIR = {OUTPUT_BASE}/pb2nc PB2NC_OUTPUT_TEMPLATE = sample_pb.nc diff --git a/parm/use_cases/met_tool_wrapper/PlotDataPlane/PlotDataPlane_grib1.conf b/parm/use_cases/met_tool_wrapper/PlotDataPlane/PlotDataPlane_grib1.conf index fcb19721f2..5a2a113c74 100644 --- a/parm/use_cases/met_tool_wrapper/PlotDataPlane/PlotDataPlane_grib1.conf +++ b/parm/use_cases/met_tool_wrapper/PlotDataPlane/PlotDataPlane_grib1.conf @@ -56,7 +56,7 @@ PLOT_DATA_PLANE_OUTPUT_DIR = [filename_templates] # Template to look for input to PlotDataPlane relative to PLOT_DATA_PLANE_INPUT_DIR -PLOT_DATA_PLANE_INPUT_TEMPLATE = {INPUT_BASE}/met_test/data/sample_fcst/{valid?fmt=%Y%m%d%H}/nam.t{valid?fmt=%2H}z.awip1236.tm00.{valid?fmt=%Y%m%d}.grb +PLOT_DATA_PLANE_INPUT_TEMPLATE = {INPUT_BASE}/met_test/data/sample_fcst/{valid?fmt=%Y%m%d%H}/nam.t{valid?fmt=%H}z.awip1236.tm00.{valid?fmt=%Y%m%d}.grb # Template to use to write output from PlotDataPlane -PLOT_DATA_PLANE_OUTPUT_TEMPLATE = {OUTPUT_BASE}/met_tool_wrapper/plot_data_plane/nam.t00z.awip1236.tm{valid?fmt=%2H}.{valid?fmt=%Y%m%d}_TMPZ2.ps +PLOT_DATA_PLANE_OUTPUT_TEMPLATE = {OUTPUT_BASE}/met_tool_wrapper/plot_data_plane/nam.t00z.awip1236.tm{valid?fmt=%H}.{valid?fmt=%Y%m%d}_TMPZ2.ps diff --git a/parm/use_cases/model_applications/convection_allowing_models/EnsembleStat_fcstHRRRE_obsHRRRE_Sfc_MultiField.conf b/parm/use_cases/model_applications/convection_allowing_models/EnsembleStat_fcstHRRRE_obsHRRRE_Sfc_MultiField.conf index f3dc96a313..03adfa304b 100644 --- a/parm/use_cases/model_applications/convection_allowing_models/EnsembleStat_fcstHRRRE_obsHRRRE_Sfc_MultiField.conf +++ b/parm/use_cases/model_applications/convection_allowing_models/EnsembleStat_fcstHRRRE_obsHRRRE_Sfc_MultiField.conf @@ -204,7 +204,7 @@ ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_BASE}/model_applications/convection_allowing_ [filename_templates] # input and output templates for pb2nc -PB2NC_INPUT_TEMPLATE = {da_init?fmt=%Y%m%d}/{da_init?fmt=%Y%j%H%M}.rap.t{da_init?fmt=%2H}z.prepbufr.tm{offset?fmt=%2H}.{da_init?fmt=%Y%m%d} +PB2NC_INPUT_TEMPLATE = {da_init?fmt=%Y%m%d}/{da_init?fmt=%Y%j%H%M}.rap.t{da_init?fmt=%H}z.prepbufr.tm{offset?fmt=%2H}.{da_init?fmt=%Y%m%d} PB2NC_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d}/{valid?fmt=%Y%m%d%H}.rap.nc diff --git a/parm/use_cases/model_applications/marine_and_cryosphere/UserScript_fcstRTOFS_obsAOML_calcTransport.conf b/parm/use_cases/model_applications/marine_and_cryosphere/UserScript_fcstRTOFS_obsAOML_calcTransport.conf new file mode 100644 index 0000000000..cd7bfe6489 --- /dev/null +++ b/parm/use_cases/model_applications/marine_and_cryosphere/UserScript_fcstRTOFS_obsAOML_calcTransport.conf @@ -0,0 +1,73 @@ +[config] + +# List of applications to run +PROCESS_LIST = UserScript + +# time looping - options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +LOOP_BY = VALID + +# Format of INIT_BEG and INIT_END using % items +# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. +# see www.strftime.org for more information +# %Y%m%d%H expands to YYYYMMDDHH +VALID_TIME_FMT = %Y%m%d + +# Start time for METplus run - must match VALID_TIME_FMT +VALID_BEG = 20211028 + +# Increment between METplus runs (in seconds if no units are specified) +# Must be >= 60 seconds +VALID_INCREMENT = 24H + +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) +LEAD_SEQ = + +# Order of loops to process data - Options are times, processes +# Not relevant if only one item is in the PROCESS_LIST +# times = run all wrappers in the PROCESS_LIST for a single run time, then +# increment the run time and run all wrappers again until all times have +# been evaluated. +# processes = run the first wrapper in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST until all +# wrappers have been run + +LOOP_ORDER = processes + +USER_SCRIPT_RUNTIME_FREQ = RUN_ONCE + +USER_SCRIPT_OUTPUT_DIR = {OUTPUT_BASE}/model_applications/marine_and_cryosphere/calc_transport + +USER_SCRIPT_INPUT_TEMPLATE = {VALID_BEG} + +# The valid date increments backwards +USER_SCRIPT_COMMAND = {PARM_BASE}/use_cases/model_applications/marine_and_cryosphere/UserScript_fcstRTOFS_obsAOML_calcTransport/read_aomlcable_rtofs_transport.py {USER_SCRIPT_INPUT_TEMPLATE} + + +[user_env_vars] + +# Calc Transport specific variables + +CALC_TRANSPORT_RTOFS_DIRNAME = {INPUT_BASE}/model_applications/marine_and_cryosphere/UserScript_fcstRTOFS_obsAOML_calcTransport/RTOFS + +CALC_TRANSPORT_CABLE_FILENAME = {INPUT_BASE}/model_applications/marine_and_cryosphere/UserScript_fcstRTOFS_obsAOML_calcTransport/FC_cable_transport_2021.dat + +CALC_TRANSPORT_EIGHTMILE_FILENAME = {INPUT_BASE}/model_applications/marine_and_cryosphere/UserScript_fcstRTOFS_obsAOML_calcTransport/eightmilecable.dat + +CALC_TRANSPORT_LEAD_TIME = 24 + +# Calculate stats for number of days. The operational website uses 21 days +# of data and then calculates 7 day stats. For the use case both of them are 3 days each. +# The code calculates the number of subdirectories +# under RTOFS directory, however, CALC_TRANSPORT_STATS_DAY is the number of days the statistics +# will be calculated. +CALC_TRANSPORT_STATS_DAY = 3 + +CALC_TRANSPORT_LOG_FILE = calc_transport.log + +OUTPUT_DIR = {OUTPUT_BASE}/model_applications/marine_and_cryosphere/calc_transport diff --git a/parm/use_cases/model_applications/marine_and_cryosphere/UserScript_fcstRTOFS_obsAOML_calcTransport/read_aomlcable_rtofs_transport.py b/parm/use_cases/model_applications/marine_and_cryosphere/UserScript_fcstRTOFS_obsAOML_calcTransport/read_aomlcable_rtofs_transport.py new file mode 100755 index 0000000000..4d92710c68 --- /dev/null +++ b/parm/use_cases/model_applications/marine_and_cryosphere/UserScript_fcstRTOFS_obsAOML_calcTransport/read_aomlcable_rtofs_transport.py @@ -0,0 +1,188 @@ +#! /usr/bin/env python3 +""" +Florida Cable Transport Class-4 Validation System +Adapted from Todd Spindler's code +""" + +from netCDF4 import Dataset +import numpy as np +from pyproj import Geod +import math +from sklearn.metrics import mean_squared_error +from datetime import datetime, timedelta +import pandas as pd +import sys, os +import logging + +vDate=datetime.strptime(sys.argv[1],'%Y%m%d') +rtofsdir = os.environ.get('CALC_TRANSPORT_RTOFS_DIRNAME') +cablefile = os.environ.get('CALC_TRANSPORT_CABLE_FILENAME') +eightmilefile = os.environ.get('CALC_TRANSPORT_EIGHTMILE_FILENAME') + +print('Starting Cable V&V at',datetime.now(),'for',vDate) + + + +if not os.path.exists(cablefile): + print('missing AOML Cable transport file for',vDate) + +#----------------------------------------------- +# read cable transport data from AOML +#----------------------------------------------- + +# read the AOML dataset +names=['year','month','day','transport'] +cable=pd.read_csv(cablefile,comment='%',names=names,delimiter=' ', + skipinitialspace=True,header=None,usecols=list(range(4))) +cable['date']=pd.to_datetime(cable[['year','month','day']]) +cable.index=cable.date +cable['error']=2.0 +del cable['year'], cable['month'], cable['day'], cable['date'] +print(cable) + +#----------------------------------------------- +# full cross-section transport calculation +#----------------------------------------------- +def calc_transport(dates,fcst): + """ + Calculate the transport of water across the Florida Straits + This extracts the section and integrates the flow through it. + """ + transport=[] + fcst_str='f{:03d}'.format(fcst) + cable_loc=np.loadtxt(eightmilefile,dtype='int',usecols=(0,1)) + eightmile_lat = 26.5167 + eightmile_lon = -78.7833%360 + wpb_lat = 26.7153425 + wpb_lon = -80.0533746%360 + cable_angle = math.atan((eightmile_lat-wpb_lat)/(eightmile_lon-wpb_lon)) + g=Geod(ellps='WGS84') + + for date in dates: + print('DATE :', date, ' DATES :',dates) + print('processing',date.strftime('%Y%m%d'),'fcst',fcst) + rundate=date-timedelta(fcst/24.) # calc rundate from fcst and date + ufile=rtofsdir+'/'+rundate.strftime('%Y%m%d')+'/rtofs_glo_3dz_'+fcst_str+'_daily_3zuio.nc' + vfile=rtofsdir+'/'+rundate.strftime('%Y%m%d')+'/rtofs_glo_3dz_'+fcst_str+'_daily_3zvio.nc' + + print(ufile) + print(vfile) + + udata=Dataset(ufile) + vdata=Dataset(vfile) + + lon=udata['Longitude'][:] + lat=udata['Latitude'][:] + depth=udata['Depth'][:] + + usection=np.zeros((depth.shape[0],cable_loc.shape[0])) + vsection=np.zeros((depth.shape[0],cable_loc.shape[0])) + + udata=udata['u'][:].squeeze() + vdata=vdata['v'][:].squeeze() + + for ncol,(row,col) in enumerate(cable_loc): + usection[:,ncol]=udata[:,row,col].filled(fill_value=0.0) + vsection[:,ncol]=vdata[:,row,col].filled(fill_value=0.0) + + lon=lon[cable_loc[:,0],cable_loc[:,1]] + lat=lat[cable_loc[:,0],cable_loc[:,1]] + + # compute the distances along the track + _,_,dist=g.inv(lon[0:-1],lat[0:-1],lon[1:],lat[1:]) + depth=np.diff(depth) + usection=usection[:-1,:-1] + vsection=vsection[:-1,:-1] + + dist,depth=np.meshgrid(dist,depth) + u,v=rotate(usection,vsection,cable_angle) + trans1=(v*dist*depth).sum()/1e6 + #print(date.strftime('%Y-%m-%d'),' transport:',transport,'Sv') + transport.append(trans1) + + return transport + +#----------------------------------------------- +# retrieve model data +#----------------------------------------------- +def get_model(dates,fcsts): + + transport={'dates':dates} + + + for fcst in fcsts: + transport[fcst]=calc_transport(dates,fcst) + + model=pd.DataFrame(transport) + model.index=model.dates + del model['dates'] + #del model['validDates'] + + print(model) + return model +#----------------------------------------------- +# coordinate rotation +#----------------------------------------------- +def rotate(u,v,phi): + # phi is in radians + u2 = u*math.cos(phi) + v*math.sin(phi) + v2 = -u*math.sin(phi) + v*math.cos(phi) + return u2,v2 + +#----------------------------------------------- +if __name__ == "__main__": + + want_date=vDate + DateSet=True + + fcst = int(os.environ.get('CALC_TRANSPORT_LEAD_TIME')) + no_of_fcst_stat_days = int(os.environ.get('CALC_TRANSPORT_STATS_DAY')) + + fcsts=list(range(fcst,fcst+1,24)) + + start_date=want_date + stop_date=want_date + cable=cable[:stop_date] + + # Count the number in the subdirs RTOFS dir + path, dirs, files = next(os.walk(rtofsdir)) + dir_count = len(dirs) + dir_count + + """ + Setup logging + """ + logfile = os.environ.get('CALC_TRANSPORT_LOG_FILE') + + + for end_date in pd.date_range(start_date,stop_date): + dates=pd.date_range(end=end_date,periods=dir_count) + model=get_model(dates,fcsts) + + both=pd.merge(cable,model,left_index=True,right_index=True,how='inner') + print("both :", both) + both=both[both.index.max()-timedelta(no_of_fcst_stat_days):] + + diff=both[fcst] - both.transport + bias=diff.mean() + rmse=mean_squared_error(both.transport,both[fcst])**0.5 + if both[fcst].mean() != 0.0: + scatter_index=100.0*(((diff**2).mean())**0.5 - bias**2)/both.transport.mean() + else: + scatter_index=np.nan + + corr=both[fcst].corr(both.transport) + +# print("BIAS :",bias, "RMSE :",rmse, "CORR :",corr, "SCATTER INDEX :",scatter_index) + + outdir = os.environ.get('OUTPUT_DIR') + + if not os.path.exists(outdir): + print(f"Creating output directory: {outdir}") + os.makedirs(outdir) + + expected_file = os.path.join(outdir,logfile) + print(expected_file) + + with open(expected_file, 'w') as f: + print("BIAS :",bias, "RMSE :",rmse, "CORR :",corr, "SCATTER INDEX :",scatter_index, file=f) diff --git a/parm/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsNAM_Sfc_MultiField_PrepBufr.conf b/parm/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsNAM_Sfc_MultiField_PrepBufr.conf index bf86801b45..224833c82c 100644 --- a/parm/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsNAM_Sfc_MultiField_PrepBufr.conf +++ b/parm/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsNAM_Sfc_MultiField_PrepBufr.conf @@ -130,7 +130,7 @@ OBS_POINT_STAT_INPUT_DIR = {PB2NC_OUTPUT_DIR} POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE}/{OBTYPE} [filename_templates] -PB2NC_INPUT_TEMPLATE = nam.{da_init?fmt=%Y%m%d}/nam.t{da_init?fmt=%2H}z.prepbufr.tm{offset?fmt=%2H} +PB2NC_INPUT_TEMPLATE = nam.{da_init?fmt=%Y%m%d}/nam.t{da_init?fmt=%H}z.prepbufr.tm{offset?fmt=%2H} PB2NC_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d}/nam.{valid?fmt=%Y%m%d%H}.nc diff --git a/parm/use_cases/model_applications/precipitation/PointStat_fcstMULTI_obsMETAR_PtypeComparisons.conf b/parm/use_cases/model_applications/precipitation/PointStat_fcstMULTI_obsMETAR_PtypeComparisons.conf index aaf0a32504..738d1c8d03 100644 --- a/parm/use_cases/model_applications/precipitation/PointStat_fcstMULTI_obsMETAR_PtypeComparisons.conf +++ b/parm/use_cases/model_applications/precipitation/PointStat_fcstMULTI_obsMETAR_PtypeComparisons.conf @@ -16,10 +16,10 @@ PB2NC_OFFSETS = 0, 12 PB2NC_SKIP_IF_OUTPUT_EXISTS = True PB2NC_INPUT_DIR = {INPUT_BASE}/model_applications/precipitation/PointStat_fcstMULTI_obsMETAR_PtypeComparisons -PB2NC_INPUT_TEMPLATE = nam.{valid?fmt=%Y%m%d}.t{valid?fmt=%2H}z.prepbufr.tm00 +PB2NC_INPUT_TEMPLATE = nam.{valid?fmt=%Y%m%d}.t{valid?fmt=%H}z.prepbufr.tm00 PB2NC_OUTPUT_DIR = {OUTPUT_BASE}/model_applications/precipitation -PB2NC_OUTPUT_TEMPLATE = nam.obsfile_sfc_prwe.{valid?fmt=%m%d%Y}_{valid?fmt=%2H}z.nc +PB2NC_OUTPUT_TEMPLATE = nam.obsfile_sfc_prwe.{valid?fmt=%m%d%Y}_{valid?fmt=%H}z.nc PB2NC_CONFIG_FILE = {PARM_BASE}/met_config/PB2NCConfig_wrapped diff --git a/parm/use_cases/model_applications/s2s/GridStat_SeriesAnalysis_fcstNMME_obsCPC_seasonal_forecast.conf b/parm/use_cases/model_applications/s2s/GridStat_SeriesAnalysis_fcstNMME_obsCPC_seasonal_forecast.conf index 160291463f..6a59ac2da9 100644 --- a/parm/use_cases/model_applications/s2s/GridStat_SeriesAnalysis_fcstNMME_obsCPC_seasonal_forecast.conf +++ b/parm/use_cases/model_applications/s2s/GridStat_SeriesAnalysis_fcstNMME_obsCPC_seasonal_forecast.conf @@ -70,7 +70,7 @@ FCST_IS_PROB = false # Increase verbosity of MET tools #LOG_MET_VERBOSITY=4 -GRID_STAT_OUTPUT_PREFIX = {MODEL}-hindcast_{CURRENT_OBS_NAME}_vs_{OBTYPE}_IC{init?fmt=%Y%b}_V{valid?fmt=%Y%2m%d} +GRID_STAT_OUTPUT_PREFIX = {MODEL}-hindcast_{CURRENT_OBS_NAME}_vs_{OBTYPE}_IC{init?fmt=%Y%b}_V{valid?fmt=%Y%m%d} # sets the desc variable in the SeriesAnalysis config file SERIES_ANALYSIS_DESC = hindcast @@ -137,12 +137,12 @@ SERIES_ANALYSIS_CLIMO_MEAN_INPUT_DIR = # format of filenames # FCST -FCST_GRID_STAT_INPUT_TEMPLATE = nmme_pr_hcst_{init?fmt=%b}IC_{valid?fmt=%2m}_*.nc +FCST_GRID_STAT_INPUT_TEMPLATE = nmme_pr_hcst_{init?fmt=%b}IC_{valid?fmt=%m}_*.nc # ANLYS OBS_GRID_STAT_INPUT_TEMPLATE = obs_cpc_pp.1x1.nc -BOTH_SERIES_ANALYSIS_INPUT_TEMPLATE = grid_stat_{MODEL}-hindcast_precip_vs_{OBTYPE}_IC{init?fmt=%Y%b}_V{valid?fmt=%Y%2m}01_*pairs.nc +BOTH_SERIES_ANALYSIS_INPUT_TEMPLATE = grid_stat_{MODEL}-hindcast_precip_vs_{OBTYPE}_IC{init?fmt=%Y%b}_V{valid?fmt=%Y%m}01_*pairs.nc SERIES_ANALYSIS_OUTPUT_TEMPLATE = series_analysis_{MODEL}_{OBTYPE}_stats_F{lead?fmt=%2m}_{instance?fmt=%s}.nc