Bugfix #2244 develop fix diff tests

.github/actions/run_tests/entrypoint.sh

@@ -8,8 +8,6 @@ WS_PATH=$RUNNER_WORKSPACE/$REPO_NAME
 # set CI jobs directory variable to easily move it
 CI_JOBS_DIR=.github/jobs
 
-PYTESTS_GROUPS_FILEPATH=.github/parm/pytest_groups.txt
-
 source ${GITHUB_WORKSPACE}/${CI_JOBS_DIR}/bash_functions.sh
 
 # get branch name for push or pull request events
@@ -34,7 +32,7 @@ fi
 
 # running unit tests (pytests)
 if [[ "$INPUT_CATEGORIES" == pytests* ]]; then
-  export METPLUS_ENV_TAG="pytest.v5.1"
+  export METPLUS_ENV_TAG="test.v5.1"
   export METPLUS_IMG_TAG=${branch_name}
   echo METPLUS_ENV_TAG=${METPLUS_ENV_TAG}
   echo METPLUS_IMG_TAG=${METPLUS_IMG_TAG}
@@ -56,15 +54,9 @@ if [[ "$INPUT_CATEGORIES" == pytests* ]]; then
 	 .
 
   echo Running Pytests
-  command="export METPLUS_PYTEST_HOST=docker; cd internal/tests/pytests;"
-  command+="status=0;"
-  for x in `cat $PYTESTS_GROUPS_FILEPATH`; do
-    marker="${x//_or_/ or }"
-    marker="${marker//not_/not }"
-    command+="/usr/local/conda/envs/${METPLUS_ENV_TAG}/bin/pytest -vv --cov=../../../metplus --cov-append -m \"$marker\""
-    command+=";if [ \$? != 0 ]; then status=1; fi;"
-  done
-  command+="if [ \$status != 0 ]; then echo ERROR: Some pytests failed. Search for FAILED to review; false; fi"
+  command="export METPLUS_TEST_OUTPUT_BASE=/data/output;"
+  command+="/usr/local/conda/envs/${METPLUS_ENV_TAG}/bin/pytest internal/tests/pytests -vv --cov=metplus --cov-append --cov-report=term-missing;"
+  command+="if [ \$? != 0 ]; then echo ERROR: Some pytests failed. Search for FAILED to review; false; fi"
   time_command docker run -v $WS_PATH:$GITHUB_WORKSPACE --workdir $GITHUB_WORKSPACE $RUN_TAG bash -c "$command"
   exit $?
 fi

.github/jobs/run_diff_docker.py

@@ -24,6 +24,7 @@
 OUTPUT_DIR = '/data/output'
 DIFF_DIR = '/data/diff'
 
+
 def copy_diff_output(diff_files):
     """!  Loop through difference output and copy files
     to directory so it can be made available for comparison.
@@ -45,6 +46,7 @@ def copy_diff_output(diff_files):
             copy_to_diff_dir(diff_file,
                              'diff')
 
+
 def copy_to_diff_dir(file_path, data_type):
     """! Generate output path based on input file path,
     adding text based on data_type to the filename, then
@@ -85,6 +87,7 @@ def copy_to_diff_dir(file_path, data_type):
 
     return True
 
+
 def main():
     print('******************************')
     print("Comparing output to truth data")
@@ -97,5 +100,6 @@ def main():
     if diff_files:
         copy_diff_output(diff_files)
 
+
 if __name__ == '__main__':
     main()

docs/Contributors_Guide/testing.rst

@@ -9,20 +9,67 @@ directory.
 Unit Tests
 ----------
 
-Unit tests are run with pytest. They are found in the *pytests* directory.
+Unit tests are run with pytest.
+They are found in the *internal/tests/pytests* directory under the *wrappers*
+and *util* directories.
 Each tool has its own subdirectory containing its test files.
 
-Unit tests can be run by running the 'pytest' command from the
-internal/tests/pytests directory of the repository.
-The 'pytest' Python package must be available.
+Pytest Requirements
+^^^^^^^^^^^^^^^^^^^
+
+The following Python packages are required to run the tests.
+
+* **pytest**: Runs the tests
+* **python-dateutil**: Required to run METplus wrappers
+* **netCDF4**: Required for some METplus wrapper functionality
+* **pytest-cov** (optional): Only if generating code coverage stats
+* **pillow** (optional): Only used if running diff utility tests
+* **pdf2image** (optional): Only used if running diff utility tests
+
+Running
+^^^^^^^
+
+To run the unit tests, set the environment variable
+**METPLUS_TEST_OUTPUT_BASE** to a path where the user running has write
+permissions, nativate to the METplus directory, then call pytest::
+
+    export METPLUS_TEST_OUTPUT_BASE=/d1/personal/${USER}/pytest
+    cd METplus
+    pytest internal/tests/pytests
+
 A report will be output showing which pytest categories failed.
-When running on a new computer, a **minimum_pytest.<HOST>.sh**
-file must be created to be able to run the script. This file contains
-information about the local environment so that the tests can run.
+To view verbose test output, add the **-vv** argument::
+
+    pytest internal/tests/pytests -vv
+
+Code Coverage
+^^^^^^^^^^^^^
+
+If the *pytest-cov* package is installed, the code coverage report can
+be generated from the tests by running::
+
+    pytest internal/tests/pytests --cov=metplus --cov-report=term-missing
+
+In addition to the pass/fail report, the code coverage information will be
+displayed including line numbers that are not covered by any test.
 
-All unit tests must include one of the custom markers listed in the
+Subsetting Tests by Directory
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+A subset of the unit tests can be run by adjusting the path.
+Be sure to include the *--cov-append* argument so the results of the run
+are appended to the full code coverage results.
+To run only the GridStat unit tests::
+
+    pytest internal/tests/pytests/wrappers/grid_stat --cov=metplus --cov-report=term-missing --cov-append
+
+
+Subsetting Tests by Marker
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+Unit tests can include one of the custom markers listed in the
 internal/tests/pytests/pytest.ini file. Some examples include:
 
+    * diff
     * run_metplus
     * util
     * wrapper_a
@@ -44,47 +91,52 @@ New pytest markers should be added to the pytest.ini file with a brief
 description. If they are not added to the markers list, then a warning will
 be output when running the tests.
 
-There are many unit tests for METplus and false failures can occur if all of
-the are attempted to run at once.
 To run only tests with a given marker, run::
 
-    pytest -m <MARKER-NAME>
+    pytest internal/tests/pytests -m <MARKER-NAME>
 
 To run all tests that do not have a given marker, run::
 
-    pytest -m "not <MARKER-NAME>"
+    pytest internal/tests/pytests -m "not <MARKER-NAME>"
+
+For example, **if you are running on a system that does not have the additional
+dependencies required to run the diff utility tests**, you can run all of the
+tests except those by running::
+
+    pytest internal/tests/pytests -m "not diff"
+
+Multiple marker groups can be run by using the *or* keyword::
+
+    pytest internal/tests/pytests -m "<MARKER-NAME1> or <MARKER-NAME2>"
+
+Writing Unit Tests
+^^^^^^^^^^^^^^^^^^
+
+metplus_config fixture
+""""""""""""""""""""""
 
-Multiple marker groups can be run by using the 'or' keyword::
+Many unit tests utilize a pytest fixture named **metplus_config**.
+This is defined in the **conftest.py** file in internal/tests/pytests.
+This is used to create a METplusConfig object that contains the minimum
+configurations needed to run METplus, like **OUTPUT_BASE**.
+Using this fixture in a pytest will initialize the METplusConfig object to use
+in the tests.
 
-    pytest -m "<MARKER-NAME1> or <MARKER-NAME2>"
+This also creates a unique output directory for each test where
+logs and output files are written. This directory is created under
+**$METPLUS_TEST_OUTPUT_BASE**/test_output and is named with the run ID.
+If the test passes, then the output directory is automatically removed.
+If the test fails, the output directory will not be removed so the content
+can be reviewed to debug the issue.
 
+To use it, add **metplus_config** as an argument to the test function::
 
-Use Case Tests
---------------
+    def test_something(metplus_config)
 
-Use case tests are run via a Python script called **test_use_cases.py**,
-found in the *use_cases* directory.
-Eventually the running of these tests will be automated using an external
-tool, such as GitHub Actions or Travis CI.
-The script contains a list of use cases that are found in the repository.
-For each computer that will run the use cases, a
-**metplus_test_env.<HOST>.sh** file must exist to set local configurations.
-All of the use cases can be run by executing the script
-**run_test_use_cases.sh**. The use case test script will output the results
-into a directory such as */d1/<USER>/test-use-case-b*, defined in the
-environment file.
-If */d1/<USER>/test-use-case-b* already exists, its content will be copied
-over to */d1/<USER>/test-use-case-a*. If data is found in
-the */d1/<USER>/test-use-case-b* directory  already exists, its content
-will be copied
-over to the */d1/<USER>/test-use-case-a* directory, the script will prompt
-the user to remove those files.
-Once the tests have finished running, the output found in the two
-directories can be compared to see what has changed. Suggested commands
-to run to compare the output will be shown on the screen after completion
-of the script.
+then set a variable called **config** using the fixture name::
 
-To see which files and directories are only found in one run::
+    config = metplus_config
 
-    diff -r /d1/mccabe/test-use-case-a /d1/mccabe/test-use-case-b | grep Only
+Additional configuration variables can be set by using the set method::
 
+    config.set('config', key, value)

internal/scripts/docker_env/Dockerfile

@@ -19,3 +19,8 @@ ARG METPLUS_ENV_VERSION
 ARG ENV_NAME
 RUN conda list --name ${ENV_NAME}.${METPLUS_ENV_VERSION} > \
     /usr/local/conda/envs/${ENV_NAME}.${METPLUS_ENV_VERSION}/environments.yml
+
+# remove base environment to free up space
+ARG METPLUS_ENV_VERSION
+ARG BASE_ENV=metplus_base
+RUN conda env remove -y --name ${BASE_ENV}.${METPLUS_ENV_VERSION}

internal/scripts/docker_env/Dockerfile.cartopy

@@ -27,3 +27,8 @@ RUN apt update && apt -y upgrade \
   && rm -f cartopy_feature_download.py \
   && curl https://raw.githubusercontent.com/SciTools/cartopy/master/tools/cartopy_feature_download.py > cartopy_feature_download.py \
   && /usr/local/conda/envs/${ENV_NAME}.${METPLUS_ENV_VERSION}/bin/python3 cartopy_feature_download.py cultural physical
+
+# remove base environment to free up space
+ARG METPLUS_ENV_VERSION
+ARG BASE_ENV=metplus_base
+RUN conda env remove -y --name ${BASE_ENV}.${METPLUS_ENV_VERSION}

internal/scripts/docker_env/README.md

@@ -426,20 +426,20 @@ export METPLUS_ENV_VERSION=v5.1
 
 
 
-## pytest.v5.1 (from metplus_base.v5.1)
+## test.v5.1 (from metplus_base.v5.1)
 
-This environment is used in automation to run the pytests. It requires all of the
+This environment is used in automation to run the pytests and diff tests. It requires all of the
 packages needed to run all of the METplus wrappers, the pytest package and the pytest
 code coverage package.
 
 ### Docker
 
 ```
 export METPLUS_ENV_VERSION=v5.1
-docker build -t dtcenter/metplus-envs:pytest.${METPLUS_ENV_VERSION} \
+docker build -t dtcenter/metplus-envs:test.${METPLUS_ENV_VERSION} \
     --build-arg METPLUS_ENV_VERSION \
-    --build-arg ENV_NAME=pytest .
-docker push dtcenter/metplus-envs:pytest.${METPLUS_ENV_VERSION}
+    --build-arg ENV_NAME=test .
+docker push dtcenter/metplus-envs:test.${METPLUS_ENV_VERSION}
 ```
 
 

internal/scripts/docker_env/scripts/pytest_env.sh → internal/scripts/docker_env/scripts/test_env.sh

@@ -1,16 +1,20 @@
 #! /bin/sh
 
 ################################################################################
-# Environment: pytest.v5.1
-# Last Updated: 2023-01-31 ([email protected])
+# Environment: test.v5.1
+# Last Updated: 2023-07-14 ([email protected])
 # Notes: Adds pytest and pytest coverage packages to run unit tests
 #   Added pandas because plot_util test needs it
 #   Added netcdf4 because SeriesAnalysis test needs it
+#   Added pillow and pdf2image for diff tests
 # Python Packages:
 # TODO: update version numbers
 #   pytest==?
 #   pytest-cov==?
 #   pandas==?
+#   netcdf4==?
+#   pillow==?
+#   pdf2image==?
 #
 # Other Content: None
 ################################################################################
@@ -19,7 +23,7 @@
 METPLUS_VERSION=$1
 
 # Conda environment to create
-ENV_NAME=pytest.${METPLUS_VERSION}
+ENV_NAME=test.${METPLUS_VERSION}
 
 # Conda environment to use as base for new environment
 BASE_ENV=metplus_base.${METPLUS_VERSION}
@@ -29,3 +33,9 @@ conda install -y --name ${ENV_NAME} -c conda-forge pytest
 conda install -y --name ${ENV_NAME} -c conda-forge pytest-cov
 conda install -y --name ${ENV_NAME} -c conda-forge pandas
 conda install -y --name ${ENV_NAME} -c conda-forge netcdf4
+conda install -y --name ${ENV_NAME} -c conda-forge pillow
+
+apt-get update
+apt-get install -y poppler-utils
+
+conda install -y --name ${ENV_NAME} -c conda-forge pdf2image

internal/tests/pytests/conftest.py

@@ -13,48 +13,23 @@
 
 from metplus.util import config_metplus
 
-# get host from either METPLUS_PYTEST_HOST or from actual host name
-# Look for minimum_pytest.<pytest_host>.sh script to source
-# error and exit if not found
-pytest_host = os.environ.get('METPLUS_PYTEST_HOST')
-if pytest_host is None:
-    import socket
-    pytest_host = socket.gethostname()
-    print("No hostname provided with METPLUS_PYTEST_HOST, "
-          f"using {pytest_host}")
-else:
-    print(f"METPLUS_PYTEST_HOST = {pytest_host}")
-
-minimum_pytest_file = os.path.join(os.path.dirname(__file__),
-                                   f'minimum_pytest.{pytest_host}.sh')
-if not os.path.exists(minimum_pytest_file):
-    print(f"ERROR: minimum_pytest.{pytest_host}.sh file must exist in "
-          "pytests directory. Set METPLUS_PYTEST_HOST correctly or "
-          "create file to run pytests on this host.")
-    sys.exit(4)
-
-# source minimum_pytest.<pytest_host>.sh script
-current_user = getpass.getuser()
-command = shlex.split(f"env -i bash -c 'export USER={current_user} && "
-                      f"source {minimum_pytest_file} && env'")
-proc = subprocess.Popen(command, stdout=subprocess.PIPE)
-
-for line in proc.stdout:
-    line = line.decode(encoding='utf-8', errors='strict').strip()
-    key, value = line.split('=')
-    os.environ[key] = value
-
-proc.communicate()
-
-output_base = os.environ['METPLUS_TEST_OUTPUT_BASE']
+output_base = os.environ.get('METPLUS_TEST_OUTPUT_BASE')
 if not output_base:
     print('ERROR: METPLUS_TEST_OUTPUT_BASE must be set to a path to write')
     sys.exit(1)
 
-test_output_dir = os.path.join(output_base, 'test_output')
-if os.path.exists(test_output_dir):
-    print(f'Removing test output dir: {test_output_dir}')
-    shutil.rmtree(test_output_dir)
+try:
+    test_output_dir = os.path.join(output_base, 'test_output')
+    if os.path.exists(test_output_dir):
+        print(f'Removing test output dir: {test_output_dir}')
+        shutil.rmtree(test_output_dir)
+
+    if not os.path.exists(test_output_dir):
+        print(f'Creating test output dir: {test_output_dir}')
+        os.makedirs(test_output_dir)
+except PermissionError:
+    print(f'ERROR: Cannot write to $METPLUS_TEST_OUTPUT_BASE: {output_base}')
+    sys.exit(2)
 
 
 @pytest.hookimpl(tryfirst=True, hookwrapper=True)

internal/tests/pytests/minimum_pytest.conf

@@ -1,8 +1,9 @@
 [config]
-INPUT_BASE = {ENV[METPLUS_TEST_INPUT_BASE]}
+INPUT_BASE = {ENV[METPLUS_TEST_OUTPUT_BASE]}/input
 OUTPUT_BASE = {ENV[METPLUS_TEST_OUTPUT_BASE]}/test_output/{RUN_ID}
-MET_INSTALL_DIR = {ENV[METPLUS_TEST_MET_INSTALL_DIR]}
-TMP_DIR = {ENV[METPLUS_TEST_TMP_DIR]}
+MET_INSTALL_DIR = {ENV[METPLUS_TEST_OUTPUT_BASE]}
+
+DO_NOT_RUN_EXE = True
 
 LOG_LEVEL = DEBUG
 LOG_LEVEL_TERMINAL = WARNING