Improve testing (#231)

Previously: when running tox, it created wheels directly from the
 source tree and installed

Now: When running tox, always run against a wheel which is created
 from sdist created from the source tree (using python -m build). This
 way (1) also the sdist is tested and (2) the same exact wheel files
 which are uploaded to PyPI will be tested in the unit tests.

In addition, start using the .pkg_external tox environment for building
the wheel -- with a small adjustment: Always only build the wheel once
(makes the tests run faster).

.github/workflows/tests-linux.yml

@@ -31,7 +31,7 @@ concurrency:
 
 # Common environment variables
 env:
-  TOX_REQUIREMENT: 'tox~=4.6.0'
+  TOX_REQUIREMENT: 'tox -c requirements/requirements-test.txt'
 
 jobs:
   test-building-docs:

.github/workflows/tests-mac.yml

@@ -55,7 +55,7 @@ jobs:
       - name: install tox
         run: |
           python -m pip install -U pip wheel && \
-          python -m pip install tox~=4.6.0
+          python -m pip install tox -c requirements/requirements-test.txt
       - name: Run tests with tox
         run: tox -e py310
       - name: Show lines not covered by tests

.github/workflows/tests-win.yml

@@ -53,7 +53,7 @@ jobs:
       - name: install tox
         run: |
           python -m pip install -U pip wheel &&`
-          python -m pip install tox~=4.6.0
+          python -m pip install tox -c requirements/requirements-test.txt
       - name: Run tests with tox
         run: tox -e py310
       - name: Show lines not covered by tests

DEV.md

@@ -40,32 +40,38 @@ invoke docs
 
 # Testing
 
-- wakepy uses pytest for tests and tox for testing the library with multiple python versions.
+Wakepy uses pytest for testing the source tree with one python version and tox for testing the created wheel with multiple python versions.
 
 ## Running tests with single environment
 
-- Requirement: Any one python version `python -m pytest` within the range of supported versions (see README.md or tox.ini)
+- Requirement: Any one python version within the range of supported versions (see README.md or tox.ini)
 - Use pytest to run tests within a single environment:
 
 ```
 invoke test
 ```
+this will (1) run tests in your current python environment against the intalled version
+of wakepy (if editable install, uses the source tree), (2) Check code coverage, (3)
+run code formatting checks.
+
 
 ## Running tests with multiple environments
 
-- Requirement:  All the python versions mentioned in the envlist in tox.ini have to be installed and available for tox.
+- Requirement:  One or more of the python versions mentioned in the envlist in tox.ini have to be installed and available for tox. Missing python versions are going to be simply skipped. If running on UNIX/macOS,
+  you may use [pyenv](https://github.com/pyenv/pyenv) to install multiple versions of python.
 - To run the tests with multiple python versions, use tox:
 
 ```
-python -m tox
+tox
 ```
 
 - To start a debugger on error with a specific python version, select the tox environment with "-e <envname>" and add "-- --pdb" to start the python debugger on error. For example:
 
 ```
-python -m tox -e py310 -- --pdb
+tox -e py310 -- --pdb
 ```
 
+- When using tox within this project, what happens is (1) wakepy is built with `python -m build`. This creates sdist from source tree and then wheel from the sdist. (2) Tests are ran against the created *wheel* (if not `skip_install=True` for that environment).
 
 # Deployment
 

pyproject.toml

@@ -63,6 +63,11 @@ no_implicit_reexport = true
 module = 'jeepney.*'
 ignore_missing_imports = true
 
+[[tool.mypy.overrides]]
+module = 'tox.*'
+ignore_missing_imports = true
+
+
 [tool.pytest.ini_options]
 filterwarnings = "ignore:.*is deprecated in wakepy 0.7.0 and will be removed in a future version of wakepy.*:DeprecationWarning"
 

requirements/requirements-test.txt

@@ -1,12 +1,19 @@
 # requirements for running unit tests
-tox==4.6.0
-pytest==8.1.1; python_version>='3.8'
+
+# Python 3.7 support dropped in tox 4.9.0
+tox==4.14.2; python_version>='3.8'
+tox==4.8.0; python_version<'3.8'
+
 # Python 3.7 support dropped in pytest 8.0.0
+pytest==8.1.1; python_version>='3.8'
 pytest==7.4.4; python_version=='3.7'
+
 pytest-cov==4.1.0
 coverage-conditional-plugin==0.9.0
-time-machine==2.14.0; python_version>='3.8'
+
 # Python 3.7 support dropped in time-machine 2.11.0
+time-machine==2.14.0; python_version>='3.8'
 time-machine==2.10.0; python_version=='3.7'
+
 # Jeepney is used in the integration tests for creating a D-Bus server
 jeepney==0.8.0;sys_platform=='linux'

tasks.py

@@ -54,7 +54,7 @@ def check(c) -> None:
     run("python -m isort --check .")
     run("python -m black --check .")
     run("python -m ruff check --no-fix .")
-    run("python -m .")
+    run("python -m mypy .")
 
 
 @task

tests/tox_build_wakepy.py

@@ -0,0 +1,38 @@
+"""This module is used solely by tox and is meant for the .pkg_external
+environment. See tox.ini for more details.
+"""
+
+import shutil
+import subprocess
+from pathlib import Path
+
+dist_dir = Path(__file__).resolve().parent.parent / "dist"
+tox_asks_rebuild = dist_dir / ".TOX-ASKS-REBUILD"
+
+
+def build():
+    print(f"Checking {tox_asks_rebuild}")
+    if not tox_asks_rebuild.exists():
+        print("Build already done. skipping.")
+        return
+
+    print(f"Removing {dist_dir} and building sdist and wheel into {dist_dir}")
+    # Cleanup. Remove all older builds; the /dist folder and its contents.
+    # Note that tox would crash if there were two files with .whl extension.
+    # This also resets the TOX-ASKS-REBUILD so we build only once.
+    shutil.rmtree(dist_dir, ignore_errors=True)
+
+    # This creates first sdist from the source tree and then wheel from the
+    # sdist. By running tests agains the wheel we test all, the source tree,
+    # the sdist and the wheel.
+    out = subprocess.run(
+        f"python -m build -o {dist_dir}", capture_output=True, shell=True
+    )
+    if out.stderr:
+        raise RuntimeError(out.stderr.decode("utf-8"))
+    print(out.stdout.decode("utf-8"))
+
+
+if __name__ == "__main__":
+
+    build()

tox.ini

@@ -4,7 +4,7 @@ envlist =
     py310
     py312
     check
-minversion = 4.6.0
+minversion = 4.8.0
 
 [testenv]
 description = run the tests with pytest
@@ -18,9 +18,15 @@ commands =
     {envpython} -m pytest -W error {tty:--color=yes} \
         --cov-branch --cov wakepy --cov-fail-under=100 {posargs}
 
+; The following makes the packaging use the external builder defined in
+; [testenv:.pkg_external] instead of using tox to create sdist/wheel.
+; https://tox.wiki/en/latest/config.html#external-package-builder
+package = external
+
 [testenv:show-uncovered-lines]
 description = Show the uncovered lines in the test coverage report
 deps = -r{toxinidir}/requirements/requirements-test.txt
+skip_install = true
 commands =
     {envpython} -m coverage report --show-missing --skip-covered
 
@@ -40,6 +46,28 @@ deps = -r{toxinidir}/requirements/requirements-docs.txt
 commands =
     ; -E: Don’t use a saved environment (the structure caching all cross-references),
     ;     but rebuild it completely.
-    ; -W: Turn warnings into errors. This means that the build stops at the first 
+    ; -W: Turn warnings into errors. This means that the build stops at the first
     ;     warning and sphinx-build exits with exit status 1.
-    sphinx-build -EW docs/source/ docs/build
+    sphinx-build -EW docs{/}source{/} docs{/}build
+
+
+
+[testenv:.pkg_external]
+; This is a special environment which is used to build the sdist and wheel
+; to the dist/ folder automatically *before* any other environments are ran.
+; All of this require the "package = external" setting.
+deps =
+    ; The build package from PyPA. See: https://build.pypa.io/en/stable/
+    build==1.1.1
+commands =
+    ; See also the tox_on_install in toxfile.py which is guaranteed to be
+    ; called before any invocations of this command.
+
+    ; This is called once per each environment (if not skip_istall=True).
+    ; We use the tox_on_install hook to create a dummy file /dist/.TOX-ASKS-REBUILD
+    ; to communicate if a build should be really done or not.
+    python tests/tox_build_wakepy.py
+
+; This determines which files tox may use to install wakepy in the test
+; environments. The .whl is created with the tox_build_wakepy.py
+package_glob = {toxinidir}{/}dist{/}wakepy-*-py3-none-any.whl

toxfile.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+import typing
+from pathlib import Path
+from typing import Any
+
+from tox.plugin import impl
+
+if typing.TYPE_CHECKING:
+    from tox.tox_env.api import ToxEnv
+
+
+dist_dir = Path(__file__).resolve().parent / "dist"
+tox_asks_rebuild = dist_dir / ".TOX-ASKS-REBUILD"
+
+
+@impl
+def tox_on_install(tox_env: ToxEnv, arguments: Any, section: str, of_type: str):
+    """The tox_on_install is once of the available tox hooks[1]. What we are
+    here after is the tox_on_intall hook call of the ".pkg_external"
+    environment, which is called max once per tox invocation, before any
+    commands of other environments are executed. The reason why this is used
+    is to make it possible to build wheel just once and use it in multiple tox
+    environments. See tox #2729[2]
+
+    [1]: https://tox.wiki/en/4.14.2/plugins.html
+    [2]: https://github.com/tox-dev/tox/issues/2729
+    """
+
+    # (1) The tox_env of .pkg_external is passed here only if the package needs
+    # to be built; only if tox is run with at least one environment with
+    # skip_install not set to True. This requires the "package = external"
+    # setting in the [testenv] of tox.ini.
+    # (2) There are two matches for `of_type`: 'requires' and 'deps'. We want
+    # to only match once. (but it does not matter which one of them)
+    print(f"Called tox_on_intall hook ({tox_env.name}, {of_type})")
+    if (tox_env.name != ".pkg_external") or (of_type != "requires"):
+        return
+
+    print(f"Creating {tox_asks_rebuild}")
+    # Create a dummy file which tells to the build script that the package
+    # should be built.
+    tox_asks_rebuild.parent.mkdir(parents=True, exist_ok=True)
+    tox_asks_rebuild.touch()

wakepy/__init__.py

@@ -8,4 +8,4 @@
 from .core import ModeExit as ModeExit
 from .modes import keep as keep
 
-__version__ = "0.8.0dev"
+__version__ = "0.8.0.dev0"