From d65f5d05227d4839c1025aa144e73eb1d824c17d Mon Sep 17 00:00:00 2001 From: w-bonelli Date: Fri, 12 Apr 2024 09:48:32 -0400 Subject: [PATCH] feat(markers): add no_parallel marker, support differing pkg/module names --- autotest/test_markers.py | 16 +++++++++++++++ docs/md/markers.md | 12 ++++++++++++ modflow_devtools/markers.py | 11 +++++++++-- modflow_devtools/misc.py | 39 +++++++++++++++++++++++++------------ 4 files changed, 64 insertions(+), 14 deletions(-) diff --git a/autotest/test_markers.py b/autotest/test_markers.py index 8df2a49..4e1718c 100644 --- a/autotest/test_markers.py +++ b/autotest/test_markers.py @@ -7,6 +7,7 @@ from modflow_devtools.markers import ( excludes_platform, + no_parallel, require_exe, require_package, require_platform, @@ -75,3 +76,18 @@ def test_requires_python(version): if Version(py_ver) >= Version(version): assert requires_python(version) assert require_python(version) + + +@no_parallel +@requires_pkg("pytest-xdist", name_map={"pytest-xdist": "xdist"}) +def test_no_parallel(worker_id): + """ + Should only run with xdist disabled, in which case: + - xdist environment variables are not set + - worker_id is 'master' (assuming xdist is installed) + + See https://pytest-xdist.readthedocs.io/en/stable/how-to.html#identifying-the-worker-process-during-a-test. + """ + + assert environ.get("PYTEST_XDIST_WORKER") is None + assert worker_id == "master" diff --git a/docs/md/markers.md b/docs/md/markers.md index a50aec3..aa963c1 100644 --- a/docs/md/markers.md +++ b/docs/md/markers.md @@ -80,6 +80,18 @@ Markers are also provided to ping network resources and skip if unavailable: - `@requires_github`: skips if `github.com` is unreachable - `@requires_spatial_reference`: skips if `spatialreference.org` is unreachable +A marker is also available to skip tests if `pytest` is running in parallel with [`pytest-xdist`](https://pytest-xdist.readthedocs.io/en/latest/): + +```python +from os import environ +from modflow_devtools.markers import no_parallel + +@no_parallel +def test_only_serially(): + # https://pytest-xdist.readthedocs.io/en/stable/how-to.html#identifying-the-worker-process-during-a-test. + assert environ.get("PYTEST_XDIST_WORKER") is None +``` + ## Aliases All markers are aliased to imperative mood, e.g. `require_github`. Some have other aliases as well: diff --git a/modflow_devtools/markers.py b/modflow_devtools/markers.py index 7bfeaf3..c87fcbe 100644 --- a/modflow_devtools/markers.py +++ b/modflow_devtools/markers.py @@ -3,7 +3,9 @@ Occasionally useful to directly assert environment expectations. """ +from os import environ from platform import python_version, system +from typing import Dict, Optional from packaging.version import Version @@ -46,8 +48,8 @@ def requires_python(version, bound="lower"): ) -def requires_pkg(*pkgs): - missing = {pkg for pkg in pkgs if not has_pkg(pkg, strict=True)} +def requires_pkg(*pkgs, name_map: Optional[Dict[str, str]] = None): + missing = {pkg for pkg in pkgs if not has_pkg(pkg, strict=True, name_map=name_map)} return pytest.mark.skipif( missing, reason=f"missing package{'s' if len(missing) != 1 else ''}: " @@ -81,6 +83,11 @@ def excludes_branch(branch): ) +no_parallel = pytest.mark.skipif( + environ.get("PYTEST_XDIST_WORKER_COUNT"), reason="can't run in parallel" +) + + requires_github = pytest.mark.skipif( not is_connected("github.com"), reason="github.com is required." ) diff --git a/modflow_devtools/misc.py b/modflow_devtools/misc.py index fac9ce9..19931ee 100644 --- a/modflow_devtools/misc.py +++ b/modflow_devtools/misc.py @@ -12,7 +12,7 @@ from shutil import which from subprocess import PIPE, Popen from timeit import timeit -from typing import List, Optional, Tuple +from typing import Dict, List, Optional, Tuple from urllib import request from urllib.error import URLError @@ -359,7 +359,9 @@ def has_exe(exe): return _has_exe_cache[exe] -def has_pkg(pkg: str, strict: bool = False) -> bool: +def has_pkg( + pkg: str, strict: bool = False, name_map: Optional[Dict[str, str]] = None +) -> bool: """ Determines if the given Python package is installed. @@ -368,8 +370,13 @@ def has_pkg(pkg: str, strict: bool = False) -> bool: pkg : str Name of the package to check. strict : bool - If False, only check if package metadata is available. + If False, only check if the package is cached or metadata is available. If True, try to import the package (all dependencies must be present). + name_map : dict, optional + Custom mapping between package names (as provided to `metadata.distribution`) + and module names (as used in import statements or `importlib.import_module`). + Useful for packages whose package names do not match the module name, e.g. + `pytest-xdist` and `xdist`, respectively, or `mfpymake` and `pymake`. Returns ------- @@ -378,12 +385,19 @@ def has_pkg(pkg: str, strict: bool = False) -> bool: Notes ----- + If `strict=True` and a package name differs from its top-level module name, a + `name_map` must be provided, otherwise this function will return False even if + the package is installed. + Originally written by Mike Toews (mwtoews@gmail.com) for FloPy. """ - def try_import(): + def get_module_name() -> str: + return pkg if name_map is None else name_map.get(pkg, pkg) + + def try_import() -> bool: try: # import name, e.g. "import shapefile" - importlib.import_module(pkg) + importlib.import_module(get_module_name()) return True except ModuleNotFoundError: return False @@ -395,14 +409,15 @@ def try_metadata() -> bool: except metadata.PackageNotFoundError: return False - found = False - if not strict: - found = pkg in _has_pkg_cache or try_metadata() - if not found: - found = try_import() + is_cached = pkg in _has_pkg_cache + has_metadata = try_metadata() + can_import = try_import() + if strict: + found = has_metadata and can_import + else: + found = has_metadata or is_cached _has_pkg_cache[pkg] = found - - return _has_pkg_cache[pkg] + return found def timed(f):