From 76a535ed10172cc0fd3b65f9203a88dc1b441c7c Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 3 Aug 2023 09:57:46 -0600 Subject: [PATCH] Make code coverage report available (#2287) --- .coveragerc | 3 ++ .github/actions/run_tests/entrypoint.sh | 42 +++--------------- .github/jobs/get_use_cases_to_run.sh | 17 -------- .github/jobs/set_job_controls.sh | 3 +- .github/workflows/testing.yml | 43 ++++++++++++++++--- docs/Contributors_Guide/basic_components.rst | 6 +-- .../dev_tools/add_met_config_helper.py | 9 ++-- internal/tests/pytests/requirements.txt | 19 ++++++++ .../grid_stat/test_grid_stat_wrapper.py | 1 - metplus/util/__init__.py | 1 - 10 files changed, 74 insertions(+), 70 deletions(-) create mode 100644 .coveragerc rename metplus/util/doc_util.py => internal/scripts/dev_tools/add_met_config_helper.py (97%) create mode 100644 internal/tests/pytests/requirements.txt diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000000..d561722b1f --- /dev/null +++ b/.coveragerc @@ -0,0 +1,3 @@ +[run] +relative_files = True +source = metplus diff --git a/.github/actions/run_tests/entrypoint.sh b/.github/actions/run_tests/entrypoint.sh index 78ce25e086..eecb719a2f 100644 --- a/.github/actions/run_tests/entrypoint.sh +++ b/.github/actions/run_tests/entrypoint.sh @@ -12,7 +12,7 @@ source ${GITHUB_WORKSPACE}/${CI_JOBS_DIR}/bash_functions.sh # get branch name for push or pull request events # add -pull_request if pull request event to keep separated -branch_name=`${GITHUB_WORKSPACE}/${CI_JOBS_DIR}/print_branch_name.py` +branch_name=$(${GITHUB_WORKSPACE}/${CI_JOBS_DIR}/print_branch_name.py) if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then branch_name=${branch_name}-pull_request fi @@ -23,49 +23,17 @@ time_command docker pull $DOCKERHUBTAG # if unsuccessful (i.e. pull request from a fork) # then build image locally -docker inspect --type=image $DOCKERHUBTAG > /dev/null -if [ $? != 0 ]; then +if ! docker inspect --type=image $DOCKERHUBTAG > /dev/null; then # if docker pull fails, build locally echo docker pull failed. Building Docker image locally... ${GITHUB_WORKSPACE}/${CI_JOBS_DIR}/docker_setup.sh fi -# running unit tests (pytests) -if [[ "$INPUT_CATEGORIES" == pytests* ]]; then - export METPLUS_ENV_TAG="test.v5.1" - export METPLUS_IMG_TAG=${branch_name} - echo METPLUS_ENV_TAG=${METPLUS_ENV_TAG} - echo METPLUS_IMG_TAG=${METPLUS_IMG_TAG} - - export RUN_TAG=metplus-run-env - - # use BuildKit to build image - export DOCKER_BUILDKIT=1 - - start_seconds=$SECONDS - - # build an image with the pytest conda env and the METplus branch image - # Note: adding --build-arg without any value tells docker to - # use value from local environment (export METPLUS_IMG_TAG) - time_command docker build -t $RUN_TAG \ - --build-arg METPLUS_IMG_TAG \ - --build-arg METPLUS_ENV_TAG \ - -f .github/actions/run_tests/Dockerfile.run \ - . - - echo Running Pytests - command="export METPLUS_TEST_OUTPUT_BASE=/data/output;" - command+="/usr/local/conda/envs/${METPLUS_ENV_TAG}/bin/pytest internal/tests/pytests -vv --cov=metplus --cov-append --cov-report=term-missing;" - command+="if [ \$? != 0 ]; then echo ERROR: Some pytests failed. Search for FAILED to review; false; fi" - time_command docker run -v $WS_PATH:$GITHUB_WORKSPACE --workdir $GITHUB_WORKSPACE $RUN_TAG bash -c "$command" - exit $? -fi - # running use case tests # split apart use case category and subset list from input -CATEGORIES=`echo $INPUT_CATEGORIES | awk -F: '{print $1}'` -SUBSETLIST=`echo $INPUT_CATEGORIES | awk -F: '{print $2}'` +CATEGORIES=$(echo $INPUT_CATEGORIES | awk -F: '{print $1}') +SUBSETLIST=$(echo $INPUT_CATEGORIES | awk -F: '{print $2}') # run all cases if no subset list specified if [ -z "${SUBSETLIST}" ]; then @@ -73,7 +41,7 @@ if [ -z "${SUBSETLIST}" ]; then fi # get METviewer if used in any use cases -all_requirements=`./${CI_JOBS_DIR}/get_requirements.py ${CATEGORIES} ${SUBSETLIST}` +all_requirements=$(./${CI_JOBS_DIR}/get_requirements.py ${CATEGORIES} ${SUBSETLIST}) echo All requirements: $all_requirements NETWORK_ARG="" if [[ "$all_requirements" =~ .*"metviewer".* ]]; then diff --git a/.github/jobs/get_use_cases_to_run.sh b/.github/jobs/get_use_cases_to_run.sh index 0d06a85df0..d937f1f7a8 100755 --- a/.github/jobs/get_use_cases_to_run.sh +++ b/.github/jobs/get_use_cases_to_run.sh @@ -7,11 +7,9 @@ matrix="[]" run_use_cases=$1 run_all_use_cases=$2 -run_unit_tests=$3 echo Run use cases: $run_use_cases echo Run all use cases: $run_all_use_cases -echo Run unit tests: $run_unit_tests # if running use cases, generate JQ filter to use if [ "$run_use_cases" == "true" ]; then @@ -28,21 +26,6 @@ if [ "$run_use_cases" == "true" ]; then fi -# if unit tests will be run, add "pytests" to beginning of matrix list -if [ "$run_unit_tests" == "true" ]; then - echo Adding unit tests to list to run - - pytests="\"pytests\"," - - # if matrix is empty, set to an array that only includes pytests - if [ "$matrix" == "[]" ]; then - matrix="[${pytests:0: -1}]" - # otherwise prepend item to list - else - matrix="[${pytests}${matrix:1}" - fi -fi - echo Array of groups to run is: $matrix # if matrix is still empty, exit 1 to fail step and skip rest of workflow if [ "$matrix" == "[]" ]; then diff --git a/.github/jobs/set_job_controls.sh b/.github/jobs/set_job_controls.sh index f03c4809ae..032197ff46 100755 --- a/.github/jobs/set_job_controls.sh +++ b/.github/jobs/set_job_controls.sh @@ -87,6 +87,7 @@ fi echo "run_get_image=$run_get_image" >> $GITHUB_OUTPUT echo "run_get_input_data=$run_get_input_data" >> $GITHUB_OUTPUT echo "run_diff=$run_diff" >> $GITHUB_OUTPUT +echo "run_unit_tests=$run_unit_tests" >> $GITHUB_OUTPUT echo "run_save_truth_data=$run_save_truth_data" >> $GITHUB_OUTPUT echo "external_trigger=$external_trigger" >> $GITHUB_OUTPUT @@ -96,7 +97,7 @@ branch_name=`${GITHUB_WORKSPACE}/.github/jobs/print_branch_name.py` echo "branch_name=$branch_name" >> $GITHUB_OUTPUT # get use cases to run -.github/jobs/get_use_cases_to_run.sh $run_use_cases $run_all_use_cases $run_unit_tests +.github/jobs/get_use_cases_to_run.sh $run_use_cases $run_all_use_cases # echo output variables to review in logs echo branch_name: $branch_name diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index aff2d9a496..148d3de5b0 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -79,6 +79,7 @@ jobs: run_get_image: ${{ steps.job_status.outputs.run_get_image }} run_get_input_data: ${{ steps.job_status.outputs.run_get_input_data }} run_diff: ${{ steps.job_status.outputs.run_diff }} + run_unit_tests: ${{ steps.job_status.outputs.run_unit_tests }} run_save_truth_data: ${{ steps.job_status.outputs.run_save_truth_data }} external_trigger: ${{ steps.job_status.outputs.external_trigger }} branch_name: ${{ steps.job_status.outputs.branch_name }} @@ -87,7 +88,7 @@ jobs: name: Docker Setup - Get METplus Image runs-on: ubuntu-latest needs: job_control - if: ${{ needs.job_control.outputs.run_get_image == 'true' }} + if: ${{ needs.job_control.outputs.run_get_image == 'true' && needs.job_control.outputs.run_some_tests == 'true' }} steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 @@ -105,7 +106,7 @@ jobs: name: Docker Setup - Update Data Volumes runs-on: ubuntu-latest needs: job_control - if: ${{ needs.job_control.outputs.run_get_input_data == 'true' }} + if: ${{ needs.job_control.outputs.run_get_input_data == 'true' && needs.job_control.outputs.run_some_tests == 'true' }} continue-on-error: true steps: - uses: dtcenter/metplus-action-data-update@v2 @@ -121,10 +122,38 @@ jobs: use_feature_data: true tag_max_pages: 15 + unit_tests: + name: Unit Tests + runs-on: ubuntu-latest + needs: [job_control] + if: ${{ needs.job_control.outputs.run_unit_tests == 'true' }} + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + cache: 'pip' + - name: Install Python Test Dependencies + run: | + python3 -m pip install --upgrade pip + python3 -m pip install -r internal/tests/pytests/requirements.txt + - name: Run Pytests + run: coverage run -m pytest internal/tests/pytests + env: + METPLUS_TEST_OUTPUT_BASE: ${{ runner.workspace }}/pytest_output + - name: Generate coverage report + run: coverage report -m + if: always() + - name: Run Coveralls + uses: AndreMiras/coveralls-python-action@develop + if: always() + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + use_case_tests: name: Use Case Tests runs-on: ubuntu-latest - needs: [get_image, update_data_volumes, job_control] + needs: [get_image, update_data_volumes, job_control, unit_tests] if: ${{ needs.job_control.outputs.run_some_tests == 'true' }} strategy: fail-fast: false @@ -158,24 +187,24 @@ jobs: # copy logs with errors to error_logs directory to save as artifact - name: Save error logs id: save-errors - if: ${{ always() && steps.run_tests.conclusion == 'failure' && !startsWith(matrix.categories,'pytests') }} + if: ${{ always() && steps.run_tests.conclusion == 'failure' }} run: .github/jobs/save_error_logs.sh # run difference testing - name: Run difference tests id: run-diff - if: ${{ needs.job_control.outputs.run_diff == 'true' && steps.run_tests.conclusion == 'success' && !startsWith(matrix.categories,'pytests') }} + if: ${{ needs.job_control.outputs.run_diff == 'true' && steps.run_tests.conclusion == 'success' }} run: .github/jobs/run_difference_tests.sh ${{ matrix.categories }} ${{ steps.get-artifact-name.outputs.artifact_name }} # copy output data to save as artifact - name: Save output data id: save-output - if: ${{ always() && steps.run_tests.conclusion != 'skipped' && !startsWith(matrix.categories,'pytests') }} + if: ${{ always() && steps.run_tests.conclusion != 'skipped' }} run: .github/jobs/copy_output_to_artifact.sh ${{ steps.get-artifact-name.outputs.artifact_name }} - name: Upload output data artifact uses: actions/upload-artifact@v3 - if: ${{ always() && steps.run_tests.conclusion != 'skipped' && !startsWith(matrix.categories,'pytests') }} + if: ${{ always() && steps.run_tests.conclusion != 'skipped' }} with: name: ${{ steps.get-artifact-name.outputs.artifact_name }} path: artifact/${{ steps.get-artifact-name.outputs.artifact_name }} diff --git a/docs/Contributors_Guide/basic_components.rst b/docs/Contributors_Guide/basic_components.rst index c386c03f79..441bb9320b 100644 --- a/docs/Contributors_Guide/basic_components.rst +++ b/docs/Contributors_Guide/basic_components.rst @@ -252,13 +252,13 @@ used to easily add support for overriding MET configuration variables that were not previously supported in METplus configuration files. There is a utility that can be used to easily see what changes are needed to -add support for a new variable. The doc_util.py script can be run from the +add support for a new variable. The add_met_config_helper.py script can be run from the command line to output a list of instructions to add new support. It can be run from the top level of the METplus repository. The script can be called to add a single MET configuration variable by supplying the MET tool name and the variable name:: - ./metplus/util/doc_util.py point_stat sid_exc + ./internal/scripts/dev_tools/add_met_config_helper.py point_stat sid_exc This command will provide guidance for adding support for the sid_exc variable found in the PointStatConfig file. @@ -266,7 +266,7 @@ found in the PointStatConfig file. The script can also be called with the name of a dictionary and the names of each dictionary variable:: - ./metplus/util/doc_util.py grid_stat distance_map baddeley_p baddeley_max_dist fom_alpha zhu_weight beta_value_n + ./internal/scripts/dev_tools/add_met_config_helper.py grid_stat distance_map baddeley_p baddeley_max_dist fom_alpha zhu_weight beta_value_n This command will provide guidance for adding support for the distance_map dictionary found in the GridStatConfig file. The list of variables found inside diff --git a/metplus/util/doc_util.py b/internal/scripts/dev_tools/add_met_config_helper.py similarity index 97% rename from metplus/util/doc_util.py rename to internal/scripts/dev_tools/add_met_config_helper.py index 05b8269494..6cd8faaef0 100755 --- a/metplus/util/doc_util.py +++ b/internal/scripts/dev_tools/add_met_config_helper.py @@ -11,10 +11,13 @@ import os try: - from .string_manip import get_wrapper_name + from metplus.util.string_manip import get_wrapper_name except ImportError: - sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) - from string_manip import get_wrapper_name + # if metplus package is not installed, find util relative to this script + metplus_home = os.path.join(os.path.dirname(__file__), + os.pardir, os.pardir, os.pardir) + sys.path.insert(0, os.path.abspath(metplus_home)) + from metplus.util.string_manip import get_wrapper_name SCRIPT_INFO_TEXT = ( 'This script is intended to help developers add support for setting ' diff --git a/internal/tests/pytests/requirements.txt b/internal/tests/pytests/requirements.txt new file mode 100644 index 0000000000..0a44495376 --- /dev/null +++ b/internal/tests/pytests/requirements.txt @@ -0,0 +1,19 @@ +certifi==2023.7.22 +cftime==1.6.2 +coverage==7.2.7 +exceptiongroup==1.1.2 +iniconfig==2.0.0 +netCDF4==1.6.4 +numpy==1.25.2 +packaging==23.1 +pandas==2.0.3 +pdf2image==1.16.3 +Pillow==10.0.0 +pluggy==1.2.0 +pytest==7.4.0 +pytest-cov==4.1.0 +python-dateutil==2.8.2 +pytz==2023.3 +six==1.16.0 +tomli==2.0.1 +tzdata==2023.3 diff --git a/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py b/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py index 80dfe61170..c971c44ffa 100644 --- a/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py +++ b/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py @@ -89,7 +89,6 @@ def set_minimum_config_settings(config): ) @pytest.mark.wrapper_b def test_grid_stat_is_prob(metplus_config, config_overrides, expected_values): - config = metplus_config set_minimum_config_settings(config) diff --git a/metplus/util/__init__.py b/metplus/util/__init__.py index 752469810f..b5e7eeb8ac 100644 --- a/metplus/util/__init__.py +++ b/metplus/util/__init__.py @@ -7,7 +7,6 @@ from .config_util import * from .config_metplus import * from .config_validate import * -from .doc_util import * from .run_util import * from .met_config import * from .time_looping import *