Description on how to run tests & build docs (#2500)

* readme tests
* add note on make
* add pool skipper
*  update Make docs
* Apply suggestions from code review

---------

Co-authored-by: Jirka Borovec <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.github/workflows/docs-build.yml

@@ -56,18 +56,12 @@ jobs:
           sudo apt-get install -y \
             texlive-latex-extra texlive-pictures texlive-fonts-recommended dvipng cm-super
 
-      - name: Pull sphinx template
-        run: |
-          pip install awscli
-          aws s3 sync --no-sign-request s3://sphinx-packages/ dist/
-          pip install lai-sphinx-theme -U -f dist/
       - name: Install package & dependencies
         run: |
-          pip --version
+          make get-sphinx-template
           pip install . -U -r requirements/_docs.txt \
-            --find-links="dist/" --find-links="${PYPI_CACHE}" --find-links="${TORCH_URL}"
-          pip list
-
+            --find-links="${PYPI_CACHE}" --find-links="${TORCH_URL}"
+      - run: pip list
       - name: Full build for deployment
         if: github.event_name != 'pull_request'
         run: echo "SPHINX_FETCH_ASSETS=1" >> $GITHUB_ENV

Makefile

@@ -1,4 +1,4 @@
-.PHONY: clean test pull-template docs env data
+.PHONY: clean test get-sphinx-template docs env data
 
 export FREEZE_REQUIREMENTS=1
 # assume you have installed need packages
@@ -28,12 +28,13 @@ test: clean env data
 	cd tests && python -m pytest unittests -v --cov=torchmetrics
 	cd tests && python -m coverage report
 
-pull-template:
+get-sphinx-template:
 	pip install -q awscli
 	aws s3 sync --no-sign-request s3://sphinx-packages/ dist/
+	pip install lai-sphinx-theme -q -U -f dist/
 
-docs: clean pull-template
-	pip install -e . --quiet -r requirements/_docs.txt -f dist/
+docs: clean get-sphinx-template
+	pip install -e . --quiet -r requirements/_docs.txt
 	# apt-get install -y texlive-latex-extra dvipng texlive-pictures texlive-fonts-recommended cm-super
 	cd docs && make html --debug --jobs $(nproc) SPHINXOPTS="-W --keep-going"
 

tests/README.md

@@ -0,0 +1,35 @@
+# Running tests locally
+
+To run the tests locally, you need to have the full development environment set up. This can be setup by running
+the following command in the root directory of the project:
+
+```bash
+pip install . -r requirements/_devel.txt
+```
+
+Then for windows users, to execute the tests (unit tests and integration tests) run the following command (will only run non-DDP tests):
+
+```bash
+pytest tests/
+```
+
+For linux/Mac users you will need to provide the `-m` argument to indicate if `ddp` tests should also be executed:
+
+```bash
+pytest -m DDP tests/  # to run only DDP tests
+pytest -m "not DDP" tests/  # to run all tests except DDP tests
+```
+
+## Simply Make
+
+Alternatively, for Unix with `make` installed, simply running `make test` from the root of the project will install
+all requirements and run the full test suit.
+
+## Test particular domain
+
+To run only unittests, point the command only to the `tests/unittests` directory. Similarly, to only run a subset of the
+unittests, like all tests related to the regression domain, run the following command:
+
+```bash
+pytest tests/unittests/regression/
+```

tests/unittests/__init__.py

@@ -14,6 +14,7 @@
     NUM_CLASSES,
     NUM_PROCESSES,
     THRESHOLD,
+    USE_PYTEST_POOL,
     setup_ddp,
 )
 
@@ -60,5 +61,6 @@ class _GroupInput(NamedTuple):
     "NUM_CLASSES",
     "NUM_PROCESSES",
     "THRESHOLD",
+    "USE_PYTEST_POOL",
     "setup_ddp",
 ]

tests/unittests/bases/test_ddp.py

@@ -24,7 +24,7 @@
 from torchmetrics.utilities.exceptions import TorchMetricsUserError
 from torchmetrics.utilities.imports import _TORCH_GREATER_EQUAL_2_1
 
-from unittests import NUM_PROCESSES
+from unittests import NUM_PROCESSES, USE_PYTEST_POOL
 from unittests._helpers import seed_all
 from unittests._helpers.testers import DummyListMetric, DummyMetric, DummyMetricSum
 
@@ -88,6 +88,7 @@ def _test_ddp_compositional_tensor(rank: int, worldsize: int = NUM_PROCESSES) ->
 
 @pytest.mark.DDP()
 @pytest.mark.skipif(sys.platform == "win32", reason="DDP not available on windows")
+@pytest.mark.skipif(not USE_PYTEST_POOL, reason="DDP pool is not available.")
 @pytest.mark.parametrize(
     "process",
     [
@@ -125,6 +126,7 @@ def compute(self):
 
 @pytest.mark.DDP()
 @pytest.mark.skipif(sys.platform == "win32", reason="DDP not available on windows")
+@pytest.mark.skipif(not USE_PYTEST_POOL, reason="DDP pool is not available.")
 def test_non_contiguous_tensors():
     """Test that gather_all operation works for non-contiguous tensors."""
     pytest.pool.map(_test_non_contiguous_tensors, range(NUM_PROCESSES))
@@ -232,6 +234,7 @@ def reload_state_dict(state_dict, expected_x, expected_c):
 
 @pytest.mark.DDP()
 @pytest.mark.skipif(sys.platform == "win32", reason="DDP not available on windows")
+@pytest.mark.skipif(not USE_PYTEST_POOL, reason="DDP pool is not available.")
 def test_state_dict_is_synced(tmpdir):
     """Tests that metrics are synced while creating the state dict but restored after to continue accumulation."""
     pytest.pool.map(partial(_test_state_dict_is_synced, tmpdir=tmpdir), range(NUM_PROCESSES))
@@ -260,6 +263,7 @@ def _test_sync_on_compute_list_state(rank, sync_on_compute):
 
 @pytest.mark.DDP()
 @pytest.mark.skipif(sys.platform == "win32", reason="DDP not available on windows")
+@pytest.mark.skipif(not USE_PYTEST_POOL, reason="DDP pool is not available.")
 @pytest.mark.parametrize("sync_on_compute", [True, False])
 @pytest.mark.parametrize("test_func", [_test_sync_on_compute_list_state, _test_sync_on_compute_tensor_state])
 def test_sync_on_compute(sync_on_compute, test_func):
@@ -276,6 +280,7 @@ def _test_sync_with_empty_lists(rank):
 @pytest.mark.DDP()
 @pytest.mark.skipif(not _TORCH_GREATER_EQUAL_2_1, reason="test only works on newer torch versions")
 @pytest.mark.skipif(sys.platform == "win32", reason="DDP not available on windows")
+@pytest.mark.skipif(not USE_PYTEST_POOL, reason="DDP pool is not available.")
 def test_sync_with_empty_lists():
     """Test that synchronization of states can be enabled and disabled for compute."""
     pytest.pool.map(_test_sync_with_empty_lists, range(NUM_PROCESSES))

tests/unittests/conftest.py

@@ -60,11 +60,7 @@ def setup_ddp(rank, world_size):
 
 
 def pytest_sessionstart():
-    """Global initialization of multiprocessing pool.
-
-    Runs before any test.
-
-    """
+    """Global initialization of multiprocessing pool; runs before any test."""
     if not USE_PYTEST_POOL:
         return
     pool = Pool(processes=NUM_PROCESSES)
@@ -73,11 +69,7 @@ def pytest_sessionstart():
 
 
 def pytest_sessionfinish():
-    """Correctly closes the global multiprocessing pool.
-
-    Runs after all tests.
-
-    """
+    """Correctly closes the global multiprocessing pool; runs after all tests."""
     if not USE_PYTEST_POOL:
         return
     pytest.pool.close()

tests/unittests/text/_helpers.py

@@ -22,7 +22,7 @@
 from torch import Tensor
 from torchmetrics import Metric
 
-from unittests import NUM_PROCESSES, _reference_cachier
+from unittests import NUM_PROCESSES, USE_PYTEST_POOL, _reference_cachier
 from unittests._helpers import seed_all
 from unittests._helpers.testers import (
     MetricTester,
@@ -369,6 +369,8 @@ def run_class_metric_test(
         if ddp:
             if sys.platform == "win32":
                 pytest.skip("DDP not supported on windows")
+            if not USE_PYTEST_POOL:
+                pytest.skip("DDP pool is not available.")
             pytest.pool.starmap(
                 partial(_class_test, **common_kwargs, **kwargs_update),
                 [(rank, NUM_PROCESSES) for rank in range(NUM_PROCESSES)],

tests/unittests/wrappers/test_running.py

@@ -22,7 +22,7 @@
 from torchmetrics.regression import MeanAbsoluteError, MeanSquaredError, PearsonCorrCoef
 from torchmetrics.wrappers import Running
 
-from unittests import NUM_PROCESSES
+from unittests import NUM_PROCESSES, USE_PYTEST_POOL
 
 
 def test_errors_on_wrong_input():
@@ -145,6 +145,7 @@ def _test_ddp_running(rank, dist_sync_on_step, expected):
 
 @pytest.mark.DDP()
 @pytest.mark.skipif(sys.platform == "win32", reason="DDP not available on windows")
+@pytest.mark.skipif(not USE_PYTEST_POOL, reason="DDP pool is not available.")
 @pytest.mark.parametrize(("dist_sync_on_step", "expected"), [(False, 1), (True, 2)])
 def test_ddp_running(dist_sync_on_step, expected):
     """Check that the dist_sync_on_step gets correctly passed to base metric."""