From 43e79e25f7a83e955ffe39efaaad55bb213fcb3e Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 27 Aug 2023 20:41:09 +0100 Subject: [PATCH 01/57] removed original import hook --- maturin/import_hook.py | 180 ----------------------------------------- 1 file changed, 180 deletions(-) delete mode 100644 maturin/import_hook.py diff --git a/maturin/import_hook.py b/maturin/import_hook.py deleted file mode 100644 index f86b4a299..000000000 --- a/maturin/import_hook.py +++ /dev/null @@ -1,180 +0,0 @@ -from __future__ import annotations - -import contextlib -import importlib -import importlib.util -import os -import pathlib -import shutil -import subprocess -import sys -from contextvars import ContextVar -from importlib import abc -from importlib.machinery import ModuleSpec -from types import ModuleType -from typing import Sequence - -try: - import tomllib -except ModuleNotFoundError: - import tomli as tomllib # type: ignore - - -# Track if we have already built the package, so we can avoid infinite -# recursion. -_ALREADY_BUILT = ContextVar("_ALREADY_BUILT", default=False) - - -class Importer(abc.MetaPathFinder): - """A meta-path importer for the maturin based packages""" - - def __init__(self, bindings: str | None = None, release: bool = False): - self.bindings = bindings - self.release = release - - def find_spec( - self, - fullname: str, - path: Sequence[str | bytes] | None = None, - target: ModuleType | None = None, - ) -> ModuleSpec | None: - if fullname in sys.modules: - return None - if _ALREADY_BUILT.get(): - # At this point we'll just import normally. - return None - - mod_parts = fullname.split(".") - module_name = mod_parts[-1] - - cwd = pathlib.Path(os.getcwd()) - # Full Cargo project in cwd - cargo_toml = cwd / "Cargo.toml" - if _is_cargo_project(cargo_toml, module_name): - return self._build_and_load(fullname, cargo_toml) - - # Full Cargo project in subdirectory of cwd - cargo_toml = cwd / module_name / "Cargo.toml" - if _is_cargo_project(cargo_toml, module_name): - return self._build_and_load(fullname, cargo_toml) - # module name with '-' instead of '_' - cargo_toml = cwd / module_name.replace("_", "-") / "Cargo.toml" - if _is_cargo_project(cargo_toml, module_name): - return self._build_and_load(fullname, cargo_toml) - - # Single .rs file - rust_file = cwd / (module_name + ".rs") - if rust_file.exists(): - project_dir = generate_project(rust_file, bindings=self.bindings or "pyo3") - cargo_toml = project_dir / "Cargo.toml" - return self._build_and_load(fullname, cargo_toml) - - return None - - def _build_and_load( - self, fullname: str, cargo_toml: pathlib.Path - ) -> ModuleSpec | None: - build_module(cargo_toml, bindings=self.bindings) - loader = Loader(fullname) - return importlib.util.spec_from_loader(fullname, loader) - - -class Loader(abc.Loader): - def __init__(self, fullname: str): - self.fullname = fullname - - def load_module(self, fullname: str) -> ModuleType: - # By the time we're loading, the package should've already been built - # by the previous step of finding the spec. - old_value = _ALREADY_BUILT.set(True) - try: - return importlib.import_module(self.fullname) - finally: - _ALREADY_BUILT.reset(old_value) - - -def _is_cargo_project(cargo_toml: pathlib.Path, module_name: str) -> bool: - with contextlib.suppress(FileNotFoundError): - with open(cargo_toml, "rb") as f: - cargo = tomllib.load(f) - package_name = cargo.get("package", {}).get("name") - if ( - package_name == module_name - or package_name.replace("-", "_") == module_name - ): - return True - return False - - -def generate_project(rust_file: pathlib.Path, bindings: str = "pyo3") -> pathlib.Path: - build_dir = pathlib.Path(os.getcwd()) / "build" - project_dir = build_dir / rust_file.stem - if project_dir.exists(): - shutil.rmtree(project_dir) - - command: list[str] = ["maturin", "new", "-b", bindings, str(project_dir)] - result = subprocess.run(command, stdout=subprocess.PIPE) - if result.returncode != 0: - sys.stderr.write( - f"Error: command {command} returned non-zero exit status {result.returncode}\n" - ) - raise ImportError("Failed to generate cargo project") - - with open(rust_file) as f: - lib_rs_content = f.read() - lib_rs = project_dir / "src" / "lib.rs" - with open(lib_rs, "w") as f: - f.write(lib_rs_content) - return project_dir - - -def build_module( - manifest_path: pathlib.Path, bindings: str | None = None, release: bool = False -) -> None: - command = ["maturin", "develop", "-m", str(manifest_path)] - if bindings: - command.append("-b") - command.append(bindings) - if release: - command.append("--release") - result = subprocess.run(command, stdout=subprocess.PIPE) - sys.stdout.buffer.write(result.stdout) - sys.stdout.flush() - if result.returncode != 0: - sys.stderr.write( - f"Error: command {command} returned non-zero exit status {result.returncode}\n" - ) - raise ImportError("Failed to build module with maturin") - - -def _have_importer() -> bool: - for importer in sys.meta_path: - if isinstance(importer, Importer): - return True - return False - - -def install(bindings: str | None = None, release: bool = False) -> Importer | None: - """ - Install the import hook. - - :param bindings: Which kind of bindings to use. - Possible values are pyo3, rust-cpython and cffi - - :param release: Build in release mode, otherwise debug mode by default - """ - if _have_importer(): - return None - importer = Importer(bindings=bindings, release=release) - sys.meta_path.insert(0, importer) - return importer - - -def uninstall(importer: Importer) -> None: - """ - Uninstall the import hook. - """ - try: - sys.meta_path.remove(importer) - except ValueError: - pass From c4386c2de8af660e882aa4459b0e5d7d96927443 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 27 Aug 2023 20:41:26 +0100 Subject: [PATCH 02/57] small fixes to test-crates --- test-crates/license-test/check_installed/check_installed.py | 2 +- test-crates/pyo3-mixed-py-subdir/README.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test-crates/license-test/check_installed/check_installed.py b/test-crates/license-test/check_installed/check_installed.py index 286755e90..a49bb636a 100644 --- a/test-crates/license-test/check_installed/check_installed.py +++ b/test-crates/license-test/check_installed/check_installed.py @@ -2,7 +2,7 @@ def main(): - output = check_output(["hello-world"]).decode("utf-8").strip() + output = check_output(["license-test"]).decode("utf-8").strip() if not output == "Hello, world!": raise Exception(output) print("SUCCESS") diff --git a/test-crates/pyo3-mixed-py-subdir/README.md b/test-crates/pyo3-mixed-py-subdir/README.md index 038375070..a148582ec 100644 --- a/test-crates/pyo3-mixed-py-subdir/README.md +++ b/test-crates/pyo3-mixed-py-subdir/README.md @@ -1,6 +1,6 @@ # pyo3-mixed -A package for testing maturin with a mixed pyo3/python project. +A package for testing maturin with a mixed pyo3/python project and a non-default package name. ## Usage @@ -9,8 +9,8 @@ pip install . ``` ```python -import pyo3_mixed -assert pyo3_mixed.get_42() == 42 +import pyo3_mixed_py_subdir +assert pyo3_mixed_py_subdir.get_42() == 42 ``` ## Testing From 43524e7b5f48ee334d3ebfbf404a0bc130a441b8 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 27 Aug 2023 20:43:43 +0100 Subject: [PATCH 03/57] added pyo3-mixed-with-path-dep test crate --- .../pyo3-mixed-with-path-dep/Cargo.lock | 304 ++++++++++++++++++ .../pyo3-mixed-with-path-dep/Cargo.toml | 14 + .../pyo3-mixed-with-path-dep/README.md | 30 ++ .../check_installed/check_installed.py | 9 + .../pyo3_mixed_with_path_dep/__init__.py | 7 + .../pyo3-mixed-with-path-dep/pyproject.toml | 11 + .../pyo3-mixed-with-path-dep/src/lib.rs | 27 ++ .../tests/test_pyo3_mixed_with_path_dep.py | 7 + test-crates/pyo3-mixed-with-path-dep/tox.ini | 7 + 9 files changed, 416 insertions(+) create mode 100644 test-crates/pyo3-mixed-with-path-dep/Cargo.lock create mode 100644 test-crates/pyo3-mixed-with-path-dep/Cargo.toml create mode 100644 test-crates/pyo3-mixed-with-path-dep/README.md create mode 100755 test-crates/pyo3-mixed-with-path-dep/check_installed/check_installed.py create mode 100644 test-crates/pyo3-mixed-with-path-dep/pyo3_mixed_with_path_dep/__init__.py create mode 100644 test-crates/pyo3-mixed-with-path-dep/pyproject.toml create mode 100644 test-crates/pyo3-mixed-with-path-dep/src/lib.rs create mode 100644 test-crates/pyo3-mixed-with-path-dep/tests/test_pyo3_mixed_with_path_dep.py create mode 100644 test-crates/pyo3-mixed-with-path-dep/tox.ini diff --git a/test-crates/pyo3-mixed-with-path-dep/Cargo.lock b/test-crates/pyo3-mixed-with-path-dep/Cargo.lock new file mode 100644 index 000000000..d8e5841db --- /dev/null +++ b/test-crates/pyo3-mixed-with-path-dep/Cargo.lock @@ -0,0 +1,304 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "cc" +version = "1.0.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c6b2562119bf28c3439f7f02db99faf0aa1a8cdfe5772a2ee155d32227239f0" +dependencies = [ + "libc", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "indoc" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa799dd5ed20a7e349f3b4639aa80d74549c81716d9ec4f994c9b5815598306" + +[[package]] +name = "libc" +version = "0.2.147" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" + +[[package]] +name = "lock_api" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets", +] + +[[package]] +name = "proc-macro2" +version = "1.0.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyo3" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e681a6cfdc4adcc93b4d3cf993749a4552018ee0a9b65fc0ccfad74352c72a38" +dependencies = [ + "cfg-if", + "indoc", + "libc", + "memoffset", + "parking_lot", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "076c73d0bc438f7a4ef6fdd0c3bb4732149136abd952b110ac93e4edb13a6ba5" +dependencies = [ + "once_cell", + "python3-dll-a", + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e53cee42e77ebe256066ba8aa77eff722b3bb91f3419177cf4cd0f304d3284d9" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfeb4c99597e136528c6dd7d5e3de5434d1ceaf487436a3f03b2d56b6fc9efd1" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "947dc12175c254889edc0c02e399476c2f652b4b9ebd123aa655c224de259536" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pyo3-mixed-with-path-dep" +version = "2.1.3" +dependencies = [ + "pyo3", + "some_path_dep", +] + +[[package]] +name = "python3-dll-a" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f07cd4412be8fa09a721d40007c483981bbe072cd6a21f2e83e04ec8f8343f" +dependencies = [ + "cc", +] + +[[package]] +name = "quote" +version = "1.0.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "smallvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" + +[[package]] +name = "some_path_dep" +version = "0.1.0" +dependencies = [ + "transitive_path_dep", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "target-lexicon" +version = "0.12.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d0e916b1148c8e263850e1ebcbd046f333e0683c724876bb0da63ea4373dc8a" + +[[package]] +name = "transitive_path_dep" +version = "0.1.0" + +[[package]] +name = "unicode-ident" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" + +[[package]] +name = "unindent" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1766d682d402817b5ac4490b3c3002d91dfa0d22812f341609f97b08757359c" + +[[package]] +name = "windows-targets" +version = "0.48.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" diff --git a/test-crates/pyo3-mixed-with-path-dep/Cargo.toml b/test-crates/pyo3-mixed-with-path-dep/Cargo.toml new file mode 100644 index 000000000..ad022c116 --- /dev/null +++ b/test-crates/pyo3-mixed-with-path-dep/Cargo.toml @@ -0,0 +1,14 @@ +[package] +authors = [] +name = "pyo3-mixed-with-path-dep" +version = "2.1.3" +description = "a library using " +edition = "2021" + +[dependencies] +pyo3 = { version = "0.19.0", features = ["extension-module", "generate-import-lib"] } +some_path_dep = { path = "../some_path_dep" } + +[lib] +name = "pyo3_mixed_with_path_dep" +crate-type = ["cdylib"] diff --git a/test-crates/pyo3-mixed-with-path-dep/README.md b/test-crates/pyo3-mixed-with-path-dep/README.md new file mode 100644 index 000000000..29e76130f --- /dev/null +++ b/test-crates/pyo3-mixed-with-path-dep/README.md @@ -0,0 +1,30 @@ +# pyo3-mixed + +A package for testing maturin with a mixed pyo3/python project. + +## Usage + +```bash +pip install . +``` + +```python +import pyo3_mixed_with_path_dep +assert pyo3_mixed_with_path_dep.get_42() == 42 +``` + +## Testing + +Install tox: + +```bash +pip install tox +``` + +Run it: + +```bash +tox +``` + +The tests are in `tests/test_pyo3_mixed_with_path_dep.py`, while the configuration is in tox.ini diff --git a/test-crates/pyo3-mixed-with-path-dep/check_installed/check_installed.py b/test-crates/pyo3-mixed-with-path-dep/check_installed/check_installed.py new file mode 100755 index 000000000..21d947dfc --- /dev/null +++ b/test-crates/pyo3-mixed-with-path-dep/check_installed/check_installed.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 +import pyo3_mixed_with_path_dep + +assert pyo3_mixed_with_path_dep.get_42() == 42, "get_42 did not return 42" + +assert pyo3_mixed_with_path_dep.is_half(21, 42), "21 is not half of 42" +assert not pyo3_mixed_with_path_dep.is_half(21, 73), "21 is half of 63" + +print("SUCCESS") diff --git a/test-crates/pyo3-mixed-with-path-dep/pyo3_mixed_with_path_dep/__init__.py b/test-crates/pyo3-mixed-with-path-dep/pyo3_mixed_with_path_dep/__init__.py new file mode 100644 index 000000000..54a76f703 --- /dev/null +++ b/test-crates/pyo3-mixed-with-path-dep/pyo3_mixed_with_path_dep/__init__.py @@ -0,0 +1,7 @@ +from .pyo3_mixed_with_path_dep import get_21, add_21, is_half + +__all__ = ["get_21", "add_21", "is_half", "get_42"] + + +def get_42() -> int: + return add_21(get_21()) diff --git a/test-crates/pyo3-mixed-with-path-dep/pyproject.toml b/test-crates/pyo3-mixed-with-path-dep/pyproject.toml new file mode 100644 index 000000000..cc54d0ae3 --- /dev/null +++ b/test-crates/pyo3-mixed-with-path-dep/pyproject.toml @@ -0,0 +1,11 @@ +[build-system] +requires = ["maturin>=1.0,<2.0"] +build-backend = "maturin" + +[project] +name = "pyo3-mixed-with-path-dep" +classifiers = [ + "Programming Language :: Python", + "Programming Language :: Rust" +] +requires-python = ">=3.7" diff --git a/test-crates/pyo3-mixed-with-path-dep/src/lib.rs b/test-crates/pyo3-mixed-with-path-dep/src/lib.rs new file mode 100644 index 000000000..1b7194289 --- /dev/null +++ b/test-crates/pyo3-mixed-with-path-dep/src/lib.rs @@ -0,0 +1,27 @@ +use pyo3::prelude::*; +use some_path_dep::{add, is_sum}; + +#[pyfunction] +fn get_21() -> usize { + 21 +} + +#[pyfunction] +fn add_21(num: usize) -> usize { + add(num, get_21()) +} + +#[pyfunction] +fn is_half(a: usize, b: usize) -> bool { + is_sum(a, a, b) +} + + +#[pymodule] +fn pyo3_mixed_with_path_dep(_py: Python, m: &PyModule) -> PyResult<()> { + m.add_wrapped(wrap_pyfunction!(get_21))?; + m.add_wrapped(wrap_pyfunction!(add_21))?; + m.add_wrapped(wrap_pyfunction!(is_half))?; + + Ok(()) +} diff --git a/test-crates/pyo3-mixed-with-path-dep/tests/test_pyo3_mixed_with_path_dep.py b/test-crates/pyo3-mixed-with-path-dep/tests/test_pyo3_mixed_with_path_dep.py new file mode 100644 index 000000000..d77884fe8 --- /dev/null +++ b/test-crates/pyo3-mixed-with-path-dep/tests/test_pyo3_mixed_with_path_dep.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 + +import pyo3_mixed_with_path_dep + + +def test_get_42(): + assert pyo3_mixed_with_path_dep.get_42() == 42 diff --git a/test-crates/pyo3-mixed-with-path-dep/tox.ini b/test-crates/pyo3-mixed-with-path-dep/tox.ini new file mode 100644 index 000000000..421774193 --- /dev/null +++ b/test-crates/pyo3-mixed-with-path-dep/tox.ini @@ -0,0 +1,7 @@ +[tox] +envlist = py36,py37,py38 +isolated_build = True + +[testenv] +deps = pytest +commands = pytest tests/ From a32a78ecc2ac209f2b33630d193a393e1bc687ed Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 27 Aug 2023 20:42:05 +0100 Subject: [PATCH 04/57] added new import hook --- maturin/import_hook/__init__.py | 73 +++ maturin/import_hook/_building.py | 247 ++++++++++ maturin/import_hook/_file_lock.py | 192 ++++++++ maturin/import_hook/_logging.py | 41 ++ maturin/import_hook/_resolve_project.py | 234 +++++++++ maturin/import_hook/project_importer.py | 573 ++++++++++++++++++++++ maturin/import_hook/rust_file_importer.py | 291 +++++++++++ maturin/import_hook/settings.py | 52 ++ 8 files changed, 1703 insertions(+) create mode 100644 maturin/import_hook/__init__.py create mode 100644 maturin/import_hook/_building.py create mode 100644 maturin/import_hook/_file_lock.py create mode 100644 maturin/import_hook/_logging.py create mode 100644 maturin/import_hook/_resolve_project.py create mode 100644 maturin/import_hook/project_importer.py create mode 100644 maturin/import_hook/rust_file_importer.py create mode 100644 maturin/import_hook/settings.py diff --git a/maturin/import_hook/__init__.py b/maturin/import_hook/__init__.py new file mode 100644 index 000000000..5f55b9100 --- /dev/null +++ b/maturin/import_hook/__init__.py @@ -0,0 +1,73 @@ +from pathlib import Path +from typing import Optional, Set, Union + +from maturin.import_hook import project_importer, rust_file_importer +from maturin.import_hook._logging import reset_logger +from maturin.import_hook.settings import MaturinSettings, MaturinSettingsProvider + +__all__ = ["install", "uninstall", "reset_logger"] + + +def install( + *, + enable_package_importer: bool = True, + enable_rs_file_importer: bool = True, + settings: Optional[Union[MaturinSettings, MaturinSettingsProvider]] = None, + build_dir: Optional[Path] = None, + install_new_packages: bool = True, + force_rebuild: bool = False, + excluded_dir_names: Optional[Set[str]] = None, + lock_timeout_seconds: Optional[float] = 120, + show_warnings: bool = True, +) -> None: + """Install import hooks for automatically rebuilding and importing maturin projects or .rs files. + + :param enable_package_importer: enable the hook for automatically rebuilding editable installed maturin projects + + :param enable_rs_file_importer: enable the hook for importing .rs files as though they were regular python modules + + :param settings: settings corresponding to flags passed to maturin. Pass MaturinSettings to use the same + settings for every project or MaturinSettingsProvider to customize + + :param build_dir: where to put the compiled artifacts. defaults to `$MATURIN_BUILD_DIR`, + `sys.exec_prefix / 'maturin_build_cache'` or + `$HOME/.cache/maturin_build_cache/` in order of preference + + :param install_new_packages: whether to install detected packages using the import hook even if they + are not already installed into the virtual environment or are installed in non-editable mode. + + :param force_rebuild: whether to always rebuild and skip checking whether anything has changed + + :param excluded_dir_names: directory names to exclude when determining whether a project has changed + and so whether the extension module needs to be rebuilt + + :param lock_timeout_seconds: a lock is required to prevent projects from being built concurrently. + If the lock is not released before this timeout is reached the import hook stops waiting and aborts + + :param show_warnings: whether to show compilation warnings + + """ + if enable_rs_file_importer: + rust_file_importer.install( + settings=settings, + build_dir=build_dir, + force_rebuild=force_rebuild, + lock_timeout_seconds=lock_timeout_seconds, + show_warnings=show_warnings, + ) + if enable_package_importer: + project_importer.install( + settings=settings, + build_dir=build_dir, + install_new_packages=install_new_packages, + force_rebuild=force_rebuild, + excluded_dir_names=excluded_dir_names, + lock_timeout_seconds=lock_timeout_seconds, + show_warnings=show_warnings, + ) + + +def uninstall() -> None: + """Remove the import hooks.""" + project_importer.uninstall() + rust_file_importer.uninstall() diff --git a/maturin/import_hook/_building.py b/maturin/import_hook/_building.py new file mode 100644 index 000000000..0d5e9791e --- /dev/null +++ b/maturin/import_hook/_building.py @@ -0,0 +1,247 @@ +import hashlib +import json +import logging +import os +import platform +import re +import shutil +import subprocess +import sys +import zipfile +from dataclasses import dataclass +from pathlib import Path +from typing import List, Optional, Tuple + +from ._file_lock import FileLock +from ._logging import logger +from .settings import MaturinSettings + + +@dataclass +class BuildStatus: + build_mtime: float + source_path: Path + maturin_args: List[str] + maturin_output: str + + def to_json(self) -> dict: + return { + "build_mtime": self.build_mtime, + "source_path": str(self.source_path), + "maturin_args": self.maturin_args, + "maturin_output": self.maturin_output, + } + + @staticmethod + def from_json(json_data: dict) -> Optional["BuildStatus"]: + try: + return BuildStatus( + build_mtime=json_data["build_mtime"], + source_path=Path(json_data["source_path"]), + maturin_args=json_data["maturin_args"], + maturin_output=json_data["maturin_output"], + ) + except KeyError: + logger.debug("failed to parse BuildStatus from %s", json_data) + return None + + +class LockNotHeldError(Exception): + pass + + +class BuildCache: + def __init__( + self, build_dir: Optional[Path], lock_timeout_seconds: Optional[float] + ) -> None: + self._build_dir = ( + build_dir if build_dir is not None else _get_default_build_dir() + ) + self._lock = FileLock.new( + self._build_dir / "lock", timeout_seconds=lock_timeout_seconds + ) + + @property + def lock(self) -> FileLock: + return self._lock + + def _build_status_path(self, source_path: Path) -> Path: + if not self._lock.is_locked: + raise LockNotHeldError + path_hash = hashlib.sha1(bytes(source_path)).hexdigest() + build_status_dir = self._build_dir / "build_status" + build_status_dir.mkdir(parents=True, exist_ok=True) + return build_status_dir / f"{path_hash}.json" + + def store_build_status(self, build_status: BuildStatus) -> None: + with self._build_status_path(build_status.source_path).open("w") as f: + json.dump(build_status.to_json(), f, indent=" ") + + def get_build_status(self, source_path: Path) -> Optional[BuildStatus]: + try: + with self._build_status_path(source_path).open("r") as f: + return BuildStatus.from_json(json.load(f)) + except FileNotFoundError: + return None + + def tmp_project_dir(self, project_path: Path, module_path: str) -> Path: + if not self._lock.is_locked: + raise LockNotHeldError + path_hash = hashlib.sha1(bytes(project_path)).hexdigest() + return self._build_dir / "project" / f"{module_path}_{path_hash}" + + +def _get_default_build_dir() -> Path: + build_dir = os.environ.get("MATURIN_BUILD_DIR", None) + if build_dir and os.access(sys.exec_prefix, os.W_OK): + return Path(build_dir) + elif os.access(sys.exec_prefix, os.W_OK): + return Path(sys.exec_prefix) / "maturin_build_cache" + else: + version_string = sys.version.split()[0] + interpreter_hash = hashlib.sha1(sys.exec_prefix.encode()).hexdigest() + return ( + _get_cache_dir() + / f"maturin_build_cache/{version_string}_{interpreter_hash}" + ) + + +def _get_cache_dir() -> Path: + if os.name == "posix": + if sys.platform == "darwin": + return Path("~/Library/Caches").expanduser() + else: + xdg_cache_dir = os.environ.get("XDG_CACHE_HOME", None) + return ( + Path(xdg_cache_dir) if xdg_cache_dir else Path("~/.cache").expanduser() + ) + elif platform.platform().lower() == "windows": + local_app_data = os.environ.get("LOCALAPPDATA", None) + return ( + Path(local_app_data) + if local_app_data + else Path(r"~\AppData\Local").expanduser() + ) + else: + logger.warning("unknown OS. defaulting to ~/.cache as the cache directory") + return Path("~/.cache").expanduser() + + +def generate_project_for_single_rust_file( + build_dir: Path, + rust_file: Path, + available_features: Optional[list[str]], +) -> Path: + project_dir = build_dir / rust_file.stem + if project_dir.exists(): + shutil.rmtree(project_dir) + + success, output = _run_maturin(["new", "--bindings", "pyo3", str(project_dir)]) + if not success: + msg = "Failed to generate project for rust file" + raise ImportError(msg) + + if available_features is not None: + available_features = [ + feature for feature in available_features if "/" not in feature + ] + cargo_manifest = project_dir / "Cargo.toml" + cargo_manifest.write_text( + "{}\n[features]\n{}".format( + cargo_manifest.read_text(), + "\n".join(f"{feature} = []" for feature in available_features), + ) + ) + + shutil.copy(rust_file, project_dir / "src/lib.rs") + return project_dir + + +def build_wheel( + manifest_path: Path, + output_dir: Path, + settings: MaturinSettings, +) -> str: + success, output = _run_maturin( + [ + "build", + "--manifest-path", + str(manifest_path), + "--out", + str(output_dir), + *settings.to_args(), + ], + ) + if not success: + msg = "Failed to build wheel with maturin" + raise ImportError(msg) + return output + + +def develop_build_project( + manifest_path: Path, + settings: MaturinSettings, + skip_install: bool, +) -> str: + args = ["develop", "--manifest-path", str(manifest_path)] + if skip_install: + args.append("--skip-install") + args.extend(settings.to_args()) + success, output = _run_maturin(args) + if not success: + msg = "Failed to build package with maturin" + raise ImportError(msg) + return output + + +def _run_maturin(args: list[str]) -> Tuple[bool, str]: + maturin_path = shutil.which("maturin") + if maturin_path is None: + msg = "maturin not found in the PATH" + raise ImportError(msg) + logger.debug('using maturin at: "%s"', maturin_path) + + command: List[str] = [maturin_path, *args] + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + output = result.stdout.decode() + if result.returncode != 0: + logger.error( + f'command "{subprocess.list2cmdline(command)}" returned non-zero exit status: {result.returncode}' + ) + logger.error("maturin output:\n%s", output) + return False, output + if logger.isEnabledFor(logging.DEBUG): + logger.debug("maturin output:\n%s", output) + return True, output + + +def build_unpacked_wheel( + manifest_path: Path, output_dir: Path, settings: MaturinSettings +) -> str: + if output_dir.exists(): + shutil.rmtree(output_dir) + output = build_wheel(manifest_path, output_dir, settings) + wheel_path = _find_single_file(output_dir, ".whl") + if wheel_path is None: + msg = "failed to generate wheel" + raise ImportError(msg) + with zipfile.ZipFile(wheel_path, "r") as f: + f.extractall(output_dir) + return output + + +def _find_single_file(dir_path: Path, extension: Optional[str]) -> Optional[Path]: + if dir_path.exists(): + candidate_files = [ + p for p in dir_path.iterdir() if extension is None or p.suffix == extension + ] + else: + candidate_files = [] + return candidate_files[0] if len(candidate_files) == 1 else None + + +def maturin_output_has_warnings(output: str) -> bool: + return ( + re.search(r"warning: `.*` \((lib|bin)\) generated [0-9]+ warnings?", output) + is not None + ) diff --git a/maturin/import_hook/_file_lock.py b/maturin/import_hook/_file_lock.py new file mode 100644 index 000000000..196beb4fd --- /dev/null +++ b/maturin/import_hook/_file_lock.py @@ -0,0 +1,192 @@ +import contextlib +import errno +import os +import platform +import time +from abc import ABC, abstractmethod +from pathlib import Path +from types import ModuleType, TracebackType +from typing import Optional, Type + +from maturin.import_hook._logging import logger + +fcntl: Optional[ModuleType] = None +with contextlib.suppress(ImportError): + import fcntl + + +msvcrt: Optional[ModuleType] = None +with contextlib.suppress(ImportError): + import msvcrt + + +class LockError(Exception): + pass + + +class FileLock(ABC): + def __init__( + self, path: Path, timeout_seconds: Optional[float], poll_interval: float = 0.05 + ) -> None: + self._path = path + self._timeout_seconds = timeout_seconds + self._poll_interval = poll_interval + self._is_locked = False + + @property + def is_locked(self) -> bool: + return self._is_locked + + def acquire(self) -> None: + if self._is_locked: + msg = f"{type(self).__name__} is not reentrant" + raise LockError(msg) + start = time.time() + first_attempt = True + while True: + self.try_acquire() + if self._is_locked: + return + if first_attempt: + logger.info("waiting on lock %s (%s)", self._path, type(self).__name__) + first_attempt = False + + if ( + self._timeout_seconds is not None + and time.time() - start > self._timeout_seconds + ): + msg = f"failed to acquire lock {self._path} in time" + raise TimeoutError(msg) + else: + time.sleep(self._poll_interval) + + @abstractmethod + def try_acquire(self) -> None: + raise NotImplementedError + + @abstractmethod + def release(self) -> None: + raise NotImplementedError + + def __enter__(self) -> None: + self.acquire() + + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> None: + self.release() + + def __del__(self) -> None: + self.release() + + @staticmethod + def new(path: Path, timeout_seconds: Optional[float]) -> "FileLock": + if os.name == "posix": + if fcntl is None: + return AtomicOpenLock(path, timeout_seconds) + else: + return FcntlFileLock(path, timeout_seconds) + elif platform.platform().lower() == "windows": + return WindowsFileLock(path, timeout_seconds) + else: + return AtomicOpenLock(path, timeout_seconds) + + +class FcntlFileLock(FileLock): + def __init__(self, path: Path, timeout_seconds: Optional[float]) -> None: + super().__init__(path, timeout_seconds) + self._path.parent.mkdir(parents=True, exist_ok=True) + self._fd = os.open(self._path, os.O_WRONLY | os.O_CREAT) + + def __del__(self) -> None: + self.release() + os.close(self._fd) + + def try_acquire(self) -> None: + if self._is_locked: + return + assert fcntl is not None + try: + fcntl.flock(self._fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError as e: + if e.errno == errno.ENOSYS: + msg = "flock not supported by filesystem" + raise LockError(msg) + else: + self._is_locked = True + + def release(self) -> None: + if self._is_locked: + assert fcntl is not None + # do not remove the lock file to avoid a potential race condition where another + # process opens the file then the file gets unlinked, leaving that process with + # a handle to a dangling file, leading it to believe it holds the lock when it doesn't + fcntl.flock(self._fd, fcntl.LOCK_UN) + self._is_locked = False + + +class WindowsFileLock(FileLock): + def __init__(self, path: Path, timeout_seconds: Optional[float]) -> None: + super().__init__(path, timeout_seconds) + self._fd = os.open(self._path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC) + + def try_acquire(self) -> None: + if self._is_locked: + return + assert msvcrt is not None + try: + msvcrt.locking(self._fd, msvcrt.LK_NBLCK, 1) + except OSError as e: + if e.errno != errno.EACCES: + msg = f"failed to acquire lock: {e}" + raise LockError(msg) + else: + self._is_locked = True + + def release(self) -> None: + if self._is_locked: + assert msvcrt is not None + msvcrt.locking(self._fd, msvcrt.LK_UNLCK, 1) + self._is_locked = False + + +class AtomicOpenLock(FileLock): + """This lock should be supported on all platforms but is not as reliable as it depends + on the filesystem supporting atomic file creation [1]. + + + - [1] https://man7.org/linux/man-pages/man2/open.2.html + """ + + def __init__(self, path: Path, timeout_seconds: Optional[float]) -> None: + super().__init__(path, timeout_seconds) + self._fd: Optional[int] = None + self._is_windows = platform.platform().lower() == "windows" + + def try_acquire(self) -> None: + if self._is_locked: + return + assert self._fd is None + try: + fd = os.open(self._path, os.O_WRONLY | os.O_CREAT | os.O_EXCL) + except OSError as e: + if not ( + e.errno == errno.EEXIST + or (self._is_windows and e.errno == errno.EACCES) + ): + msg = f"failed to acquire lock: {e}" + raise LockError(msg) + else: + self._fd = fd + self._is_locked = True + + def release(self) -> None: + if self._is_locked: + assert self._fd is not None + os.close(self._fd) + self._fd = None + self._is_locked = False + self._path.unlink(missing_ok=True) diff --git a/maturin/import_hook/_logging.py b/maturin/import_hook/_logging.py new file mode 100644 index 000000000..1df21f8ac --- /dev/null +++ b/maturin/import_hook/_logging.py @@ -0,0 +1,41 @@ +import logging + +logger = logging.getLogger("maturin.import_hook") + + +class _LevelDependentFormatter(logging.Formatter): + def __init__(self) -> None: + super().__init__(fmt="", datefmt=None, style="%") + self._info_fmt = "%(message)s" + self._other_fmt = "%(name)s [%(levelname)s] %(message)s" + + def format(self, record: logging.LogRecord) -> str: + if record.levelno == logging.INFO: + self._style._fmt = self._info_fmt + else: + self._style._fmt = self._other_fmt + return super().format(record) + + +def _init_logger() -> None: + """Configure reasonable defaults for the maturin.import_hook logger.""" + logger.setLevel(logging.INFO) + handler = logging.StreamHandler() + handler.setLevel(logging.INFO) + formatter = _LevelDependentFormatter() + handler.setFormatter(formatter) + logger.addHandler(handler) + logger.propagate = False + + +_init_logger() + + +def reset_logger() -> None: + """Clear the custom configuration on the maturin import hook logger + and have it propagate messages to the root logger instead. + """ + logger.propagate = True + logger.setLevel(logging.NOTSET) + for handler in logger.handlers: + logger.removeHandler(handler) diff --git a/maturin/import_hook/_resolve_project.py b/maturin/import_hook/_resolve_project.py new file mode 100644 index 000000000..342e9ff4c --- /dev/null +++ b/maturin/import_hook/_resolve_project.py @@ -0,0 +1,234 @@ +import itertools +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +from maturin.import_hook._logging import logger + +try: + import tomllib +except ModuleNotFoundError: + import tomli as tomllib # type: ignore + + +def find_cargo_manifest(project_dir: Path) -> Optional[Path]: + manifest_path = project_dir / "Cargo.toml" + if manifest_path.exists(): + return manifest_path + manifest_path = project_dir / "rust/Cargo.toml" + if manifest_path.exists(): + return manifest_path + return None + + +def is_maybe_maturin_project(project_dir: Path) -> bool: + """note: this function does not check if this really is a maturin project for simplicity.""" + return (project_dir / "pyproject.toml").exists() and find_cargo_manifest( + project_dir + ) is not None + + +class ProjectResolver: + def __init__(self) -> None: + self._resolved_project_cache: Dict[Path, MaturinProject] = {} + + def resolve(self, project_dir: Path) -> Optional["MaturinProject"]: + if project_dir not in self._resolved_project_cache: + resolved = None + try: + resolved = _resolve_project(project_dir) + except ProjectResolveError as e: + logger.info('failed to resolve project "%s": %s', project_dir, e) + else: + self._resolved_project_cache[project_dir] = resolved + else: + resolved = self._resolved_project_cache[project_dir] + return resolved + + +@dataclass +class MaturinProject: + cargo_manifest_path: Path + # the name of the compiled extension module without any suffix + # (i.e. "some_package.my_module" instead of "some_package/my_module.cpython-311-x86_64-linux-gnu") + module_full_name: str + # the root of the python part of the project (or the project root if there is none) + python_dir: Path + # the path to the top level python package if the project is mixed + python_module: Optional[Path] + # the location that the compiled extension module is written to when installed in editable/unpacked mode + extension_module_dir: Optional[Path] + # path dependencies listed in the Cargo.toml of the main project + immediate_path_dependencies: List[Path] + # all path dependencies including transitive dependencies + _all_path_dependencies: Optional[List[Path]] = None + + @property + def package_name(self) -> str: + return self.module_full_name.split(".")[0] + + @property + def module_name(self) -> str: + return self.module_full_name.split(".")[-1] + + @property + def is_mixed(self) -> bool: + """Whether the project contains both python and rust code.""" + return self.extension_module_dir is not None + + @property + def all_path_dependencies(self) -> List[Path]: + if self._all_path_dependencies is None: + self._all_path_dependencies = _find_all_path_dependencies( + self.immediate_path_dependencies + ) + return self._all_path_dependencies + + +def _find_all_path_dependencies(immediate_path_dependencies: List[Path]) -> List[Path]: + if not immediate_path_dependencies: + return [] + all_path_dependencies = set() + to_search = immediate_path_dependencies.copy() + while to_search: + project_dir = to_search.pop() + if project_dir in all_path_dependencies: + continue + all_path_dependencies.add(project_dir) + manifest_path = project_dir / "Cargo.toml" + if manifest_path.exists(): + with manifest_path.open("rb") as f: + cargo = tomllib.load(f) + to_search.extend(_get_immediate_path_dependencies(project_dir, cargo)) + return sorted(all_path_dependencies) + + +class ProjectResolveError(Exception): + pass + + +def _resolve_project(project_dir: Path) -> MaturinProject: + """This follows the same logic as project_layout.rs. + + module_writer::write_bindings_module() is the function that copies the extension file to `rust_module / so_filename` + """ + pyproject_path = project_dir / "pyproject.toml" + if not pyproject_path.exists(): + msg = "no pyproject.toml found" + raise ProjectResolveError(msg) + with pyproject_path.open("rb") as f: + pyproject = tomllib.load(f) + + manifest_path = find_cargo_manifest(project_dir) + if manifest_path is None: + msg = "no Cargo.toml found" + raise ProjectResolveError(msg) + with manifest_path.open("rb") as f: + cargo = tomllib.load(f) + + module_full_name = _resolve_module_name(pyproject, cargo) + if module_full_name is None: + msg = "could not resolve module_full_name" + raise ProjectResolveError(msg) + + python_dir = _resolve_py_root(project_dir, pyproject) + + extension_module_dir: Optional[Path] + python_module: Optional[Path] + python_module, extension_module_dir, extension_module_name = _resolve_rust_module( + python_dir, module_full_name + ) + immediate_path_dependencies = _get_immediate_path_dependencies(project_dir, cargo) + + if not python_module.exists(): + extension_module_dir = None + python_module = None + + return MaturinProject( + cargo_manifest_path=manifest_path, + module_full_name=module_full_name, + python_dir=python_dir, + python_module=python_module, + extension_module_dir=extension_module_dir, + immediate_path_dependencies=immediate_path_dependencies, + ) + + +def _resolve_rust_module(python_dir: Path, module_name: str) -> Tuple[Path, Path, str]: + """This follows the same logic as project_layout.rs (ProjectLayout::determine). + + rust_module is the directory that the extension library gets written to when the package is + installed in editable mode + """ + parts = module_name.split(".") + if len(parts) > 1: + python_module = python_dir / parts[0] + extension_module_dir = python_dir / Path(*parts[:-1]) + extension_module_name = parts[-1] + else: + python_module = python_dir / module_name + extension_module_dir = python_dir / module_name + extension_module_name = module_name + return python_module, extension_module_dir, extension_module_name + + +def _resolve_module_name( + pyproject: Dict[str, Any], cargo: Dict[str, Any] +) -> Optional[str]: + """This follows the same logic as project_layout.rs (ProjectResolver::resolve). + + Precedence: + * Explicitly declared pyproject.toml `tool.maturin.module-name` + * Cargo.toml `lib.name` + * pyproject.toml `project.name` + * Cargo.toml `package.name` + + """ + module_name = pyproject.get("tool", {}).get("maturin", {}).get("module-name", None) + if module_name is not None: + return module_name + module_name = cargo.get("lib", {}).get("name", None) + if module_name is not None: + return module_name + module_name = pyproject.get("project", {}).get("name") + if module_name is not None: + return module_name + return cargo.get("package", {}).get("name", None) + + +def _get_immediate_path_dependencies( + project_dir: Path, cargo: Dict[str, Any] +) -> list[Path]: + path_dependencies = [] + for dependency in cargo.get("dependencies", {}).values(): + if isinstance(dependency, dict): + relative_path = dependency.get("path", None) + if relative_path is not None: + path_dependencies.append((project_dir / relative_path).resolve()) + return path_dependencies + + +def _resolve_py_root(project_dir: Path, pyproject: Dict[str, Any]) -> Path: + """This follows the same logic as project_layout.rs.""" + py_root = pyproject.get("tool", {}).get("maturin", {}).get("python-source", None) + if py_root is not None: + return project_dir / py_root + project_name = pyproject.get("project", {}).get("name", None) + if project_name is None: + return project_dir + + rust_cargo_toml_found = (project_dir / "rust/Cargo.toml").exists() + + python_packages = ( + pyproject.get("tool", {}).get("maturin", {}).get("python-packages", []) + ) + + import_name = project_name.replace("-", "_") + python_src_found = any( + (project_dir / p / "__init__.py").is_file() + for p in itertools.chain((f"src/{import_name}/",), python_packages) + ) + if rust_cargo_toml_found and python_src_found: + return project_dir / "src" + else: + return project_dir diff --git a/maturin/import_hook/project_importer.py b/maturin/import_hook/project_importer.py new file mode 100644 index 000000000..6dd3825f8 --- /dev/null +++ b/maturin/import_hook/project_importer.py @@ -0,0 +1,573 @@ +import contextlib +import importlib.abc +import itertools +import json +import logging +import math +import site +import sys +import time +import urllib.parse +from importlib.machinery import ModuleSpec, PathFinder +from pathlib import Path +from types import ModuleType +from typing import Iterable, Optional, Sequence, Set, Tuple, Union + +from maturin.import_hook._building import ( + BuildCache, + BuildStatus, + develop_build_project, + maturin_output_has_warnings, +) +from maturin.import_hook._logging import logger +from maturin.import_hook._resolve_project import ( + MaturinProject, + ProjectResolver, + is_maybe_maturin_project, +) +from maturin.import_hook.settings import MaturinSettings, MaturinSettingsProvider + +__all__ = [ + "MaturinProjectImporter", + "install", + "uninstall", + "IMPORTER", + "DEFAULT_EXCLUDED_DIR_NAMES", +] + + +DEFAULT_EXCLUDED_DIR_NAMES = { + "__pycache__", + "target", + "dist", + ".git", + "venv", + ".venv", + ".pytest_cache", +} + + +class MaturinProjectImporter(importlib.abc.MetaPathFinder): + """An import hook for automatically rebuilding editable installed maturin projects.""" + + def __init__( + self, + *, + settings: Optional[Union[MaturinSettings, MaturinSettingsProvider]] = None, + build_dir: Optional[Path] = None, + lock_timeout_seconds: Optional[float] = 120, + install_new_packages: bool = True, + force_rebuild: bool = False, + excluded_dir_names: Optional[Set[str]] = None, + show_warnings: bool = True, + ) -> None: + self._resolver = ProjectResolver() + self._settings = settings + self._build_cache = BuildCache(build_dir, lock_timeout_seconds) + self._install_new_packages = install_new_packages + self._force_rebuild = force_rebuild + self._show_warnings = show_warnings + self._excluded_dir_names = ( + DEFAULT_EXCLUDED_DIR_NAMES + if excluded_dir_names is None + else excluded_dir_names + ) + + def _get_settings(self, module_path: str, source_path: Path) -> MaturinSettings: + if isinstance(self._settings, MaturinSettings): + return self._settings + elif isinstance(self._settings, MaturinSettingsProvider): + return self._settings.get_settings(module_path, source_path) + else: + return MaturinSettings() + + def find_spec( + self, + fullname: str, + path: Optional[Sequence[Union[str, bytes]]] = None, + target: Optional[ModuleType] = None, + ) -> Optional[ModuleSpec]: + if fullname in sys.modules: + return None + + is_top_level_import = path is None + if not is_top_level_import: + return None + assert "." not in fullname + package_name = fullname + + start = time.perf_counter() + + # sys.path includes site-packages and search roots for editable installed packages + search_paths = [Path(p) for p in sys.path] + + if logger.isEnabledFor(logging.DEBUG): + logger.debug('%s searching for "%s"', type(self).__name__, package_name) + + spec = None + rebuilt = False + for search_path in search_paths: + project_dir, is_editable = _load_dist_info(search_path, package_name) + if project_dir is not None: + logger.debug('found project linked by dist-info: "%s"', project_dir) + if not is_editable and not self._install_new_packages: + logger.debug( + "package not installed in editable-mode " + "and install_new_packages=False. not rebuilding" + ) + else: + spec, rebuilt = self._rebuild_project(package_name, project_dir) + if spec is not None: + break + + project_dir = _find_maturin_project_above(search_path) + if project_dir is not None: + logger.debug( + 'found project above the search path: "%s" ("%s")', + project_dir, + search_path, + ) + spec, rebuilt = self._rebuild_project(package_name, project_dir) + if spec is not None: + break + + if spec is not None: + duration = time.perf_counter() - start + if rebuilt: + logger.info( + 'rebuilt and loaded package "%s" in %.3fs', package_name, duration + ) + else: + logger.debug('loaded package "%s" in %.3fs', package_name, duration) + return spec + + def _rebuild_project( + self, + package_name: str, + project_dir: Path, + ) -> Tuple[Optional[ModuleSpec], bool]: + resolved = self._resolver.resolve(project_dir) + if resolved is None: + return None, False + logger.debug("module name %s", resolved.module_full_name) + if package_name != resolved.package_name: + logger.debug( + 'package name "%s" of project does not match "%s". Not importing', + resolved.package_name, + package_name, + ) + return None, False + + if not self._install_new_packages and not _is_editable_installed_package( + project_dir, package_name + ): + logger.debug( + 'package "%s" is not already installed and ' + "install_new_packages=False. Not importing", + package_name, + ) + return None, False + + logger.debug('importing project "%s" as "%s"', project_dir, package_name) + + with self._build_cache.lock: + settings = self._get_settings(package_name, project_dir) + spec, reason = self._get_spec_for_up_to_date_package( + package_name, project_dir, resolved, settings + ) + if spec is not None: + return spec, False + logger.debug( + 'package "%s" will be rebuilt because: %s', package_name, reason + ) + + logger.info('building "%s"', package_name) + start = time.perf_counter() + maturin_output = develop_build_project( + resolved.cargo_manifest_path, settings, skip_install=False + ) + _fix_direct_url(project_dir, package_name) + logger.debug( + 'compiled project "%s" in %.3fs', + package_name, + time.perf_counter() - start, + ) + + if self._show_warnings and maturin_output_has_warnings(maturin_output): + self._log_build_warnings(package_name, maturin_output, is_fresh=True) + + spec = _find_spec_for_package(package_name) + if spec is None: + msg = f'cannot find package "{package_name}" after installation' + raise ImportError(msg) + + installed_package_root = _find_installed_package_root(resolved, spec) + if installed_package_root is None: + logger.error("could not get installed package root") + else: + mtime = _get_installed_package_mtime( + installed_package_root, self._excluded_dir_names + ) + if mtime is None: + logger.error("could not get installed package mtime") + else: + build_status = BuildStatus( + mtime, project_dir, settings.to_args(), maturin_output + ) + self._build_cache.store_build_status(build_status) + + return spec, True + + def _get_spec_for_up_to_date_package( + self, + package_name: str, + project_dir: Path, + resolved: MaturinProject, + settings: MaturinSettings, + ) -> Tuple[Optional[ModuleSpec], Optional[str]]: + """Return a spec for the given module at the given search_dir if it exists and is newer than the source + code that it is derived from. + """ + logger.debug('checking whether the package "%s" is up to date', package_name) + + if self._force_rebuild: + return None, "forcing rebuild" + + spec = _find_spec_for_package(package_name) + if spec is None: + return None, "package not already installed" + + installed_package_root = _find_installed_package_root(resolved, spec) + if installed_package_root is None: + return None, "could not find installed package root" + + installed_package_mtime = _get_installed_package_mtime( + installed_package_root, self._excluded_dir_names + ) + if installed_package_mtime is None: + return None, "could not get installed package mtime" + + if not _package_is_up_to_date( + project_dir, + resolved.all_path_dependencies, + installed_package_root, + installed_package_mtime, + self._excluded_dir_names, + ): + return None, "package is out of date" + + build_status = self._build_cache.get_build_status(project_dir) + if build_status is None: + return None, "no build status found" + if build_status.source_path != project_dir: + return None, "source path in build status does not match the project dir" + if not math.isclose(build_status.build_mtime, installed_package_mtime): + return None, "installed package mtime does not match build status mtime" + if build_status.maturin_args != settings.to_args(): + return None, "current maturin args do not match the previous build" + + logger.debug('package up to date: "%s" ("%s")', package_name, spec.origin) + + if self._show_warnings and maturin_output_has_warnings( + build_status.maturin_output + ): + self._log_build_warnings( + package_name, build_status.maturin_output, is_fresh=False + ) + + return spec, None + + def _log_build_warnings( + self, module_path: str, maturin_output: str, is_fresh: bool + ) -> None: + prefix = "" if is_fresh else "the last " + message = '%sbuild of "%s" succeeded with warnings:\n%s' + if self._show_warnings: + logger.warning(message, prefix, module_path, maturin_output) + else: + logger.debug(message, prefix, module_path, maturin_output) + + +def _find_spec_for_package(package_name: str) -> Optional[ModuleSpec]: + path_finder = PathFinder() + spec = path_finder.find_spec(package_name) + if spec is not None: + return spec + logger.debug('spec for package "%s" not found', package_name) + if _is_installed_package(package_name): + logger.debug( + 'package "%s" appears to be installed. Refreshing packages and trying again', + package_name, + ) + site.addsitepackages(None) + return path_finder.find_spec(package_name) + else: + return None + + +def _is_installed_package(package_name: str) -> bool: + for path_str in site.getsitepackages(): + path = Path(path_str) + if (path / package_name).is_dir() or (path / f"{package_name}.pth").is_file(): + return True + return False + + +def _is_editable_installed_package(project_dir: Path, package_name: str) -> bool: + for path_str in site.getsitepackages(): + path = Path(path_str) + pth_file = path / f"{package_name}.pth" + if pth_file.is_file(): + pth_link = Path(pth_file.read_text().strip()) + if project_dir == pth_link or project_dir in pth_link.parents: + return True + + if (path / package_name).is_dir(): + linked_package_dir, is_editable = _load_dist_info(path, package_name) + return linked_package_dir == project_dir and is_editable + return False + + +def _find_maturin_project_above(path: Path) -> Optional[Path]: + for search_path in itertools.chain((path,), path.parents): + if is_maybe_maturin_project(search_path): + return search_path + return None + + +def _load_dist_info(path: Path, package_name: str) -> Tuple[Optional[Path], bool]: + dist_info_path = next(path.glob(f"{package_name}-*.dist-info"), None) + if dist_info_path is None: + return None, False + try: + with open(dist_info_path / "direct_url.json") as f: + dist_info_data = json.load(f) + except OSError: + return None, False + else: + is_editable = dist_info_data.get("dir_info", {}).get("editable", False) + url = dist_info_data.get("url") + if url is None: + return None, is_editable + prefix = "file://" + if not url.startswith(prefix): + return None, is_editable + linked_path = Path(url[len(prefix) :]) + if is_maybe_maturin_project(linked_path): + return linked_path, is_editable + else: + return None, is_editable + + +def _fix_direct_url(project_dir: Path, package_name: str) -> None: + """Seemingly due to a bug, installing with `pip install -e` will write the correct entry into `direct_url.json` to + point at the project directory, but calling `maturin develop` does not currently write this value correctly. + """ + logger.debug("fixing direct_url for %s", package_name) + for path in site.getsitepackages(): + dist_info = next(Path(path).glob(f"{package_name}-*.dist-info"), None) + if dist_info is None: + continue + direct_url_path = dist_info / "direct_url.json" + try: + with open(direct_url_path) as f: + direct_url = json.load(f) + except OSError: + continue + url = f"file://{urllib.parse.quote(str(project_dir))}" + if direct_url.get("url") != url: + logger.debug("fixing direct_url.json for package %s", package_name) + logger.debug('"%s" -> "%s"', direct_url.get("url"), url) + direct_url = {"dir_info": {"editable": True}, "url": url} + try: + with open(direct_url_path, "w") as f: + json.dump(direct_url, f) + except OSError: + return + + +def _find_installed_package_root( + resolved: MaturinProject, package_spec: ModuleSpec +) -> Optional[Path]: + """Find the root of the files that change each time the project is rebuilt: + - for mixed projects: the root directory or file of the extension module inside the source tree + - for pure projects: the root directory of the installed package. + """ + if resolved.extension_module_dir is not None: + installed_package_root = _find_extension_module( + resolved.extension_module_dir, resolved.module_name, require=False + ) + if installed_package_root is None: + logger.debug( + 'no extension module found in "%s"', resolved.extension_module_dir + ) + return installed_package_root + elif package_spec.origin is not None: + return Path(package_spec.origin).parent + else: + logger.debug("could not find installation location for pure package") + return None + + +def _get_installed_package_mtime( + installed_package_root: Path, excluded_dir_names: Set[str] +) -> Optional[float]: + if installed_package_root.is_dir(): + try: + return min( + path.stat().st_mtime + for path in _get_files_in_dirs( + (installed_package_root,), excluded_dir_names, set() + ) + ) + except ValueError: + logger.debug('no installed files found in "%s"', installed_package_root) + return None + else: + try: + return installed_package_root.stat().st_mtime + except FileNotFoundError: + logger.debug('extension module not found: "%s"', installed_package_root) + return None + + +def _get_project_mtime( + project_dir: Path, + all_path_dependencies: list[Path], + installed_package_root: Path, + excluded_dir_names: Set[str], +) -> Optional[float]: + excluded_dirs = set() + if installed_package_root.is_dir(): + excluded_dirs.add(installed_package_root) + + try: + return max( + path.stat().st_mtime + for path in _get_files_in_dirs( + [project_dir, *all_path_dependencies], excluded_dir_names, excluded_dirs + ) + ) + except (FileNotFoundError, ValueError): + logger.debug("error getting project mtime") + return None + + +def _package_is_up_to_date( + project_dir: Path, + all_path_dependencies: list[Path], + installed_package_root: Path, + installed_package_mtime: float, + excluded_dir_names: Set[str], +) -> bool: + project_mtime = _get_project_mtime( + project_dir, all_path_dependencies, installed_package_root, excluded_dir_names + ) + if project_mtime is None: + return False + + logger.debug( + "extension mtime: %f %s project mtime: %f", + installed_package_mtime, + ">=" if installed_package_mtime >= project_mtime else "<", + project_mtime, + ) + return installed_package_mtime >= project_mtime + + +def _find_extension_module( + dir_path: Path, module_name: str, *, require: bool = False +) -> Optional[Path]: + if (dir_path / module_name / "__init__.py").exists(): + return dir_path / module_name + + # the suffixes include the platform tag and file extension eg '.cpython-311-x86_64-linux-gnu.so' + for suffix in importlib.machinery.EXTENSION_SUFFIXES: + extension_path = dir_path / f"{module_name}{suffix}" + if extension_path.exists(): + return extension_path + if require: + msg = f'could not find module "{module_name}" in "{dir_path}"' + raise ImportError(msg) + return None + + +def _get_files_in_dirs( + dir_paths: Iterable[Path], + excluded_dir_names: Set[str], + excluded_dir_paths: Set[Path], +) -> Iterable[Path]: + for dir_path in dir_paths: + for path in dir_path.iterdir(): + if path.is_dir(): + if ( + path.name not in excluded_dir_names + and path not in excluded_dir_paths + ): + yield from _get_files_in_dirs( + (path,), excluded_dir_names, excluded_dir_paths + ) + else: + yield path + + +IMPORTER: Optional[MaturinProjectImporter] = None + + +def install( + *, + settings: Optional[Union[MaturinSettings, MaturinSettingsProvider]] = None, + build_dir: Optional[Path] = None, + install_new_packages: bool = True, + force_rebuild: bool = False, + excluded_dir_names: Optional[Set[str]] = None, + lock_timeout_seconds: Optional[float] = 120, + show_warnings: bool = True, +) -> MaturinProjectImporter: + """Install an import hook for automatically rebuilding editable installed maturin projects. + + :param settings: settings corresponding to flags passed to maturin. Pass MaturinSettings to use the same + settings for every project or MaturinSettingsProvider to customize + + :param build_dir: where to put the compiled artifacts. defaults to `$MATURIN_BUILD_DIR`, + `sys.exec_prefix / 'maturin_build_cache'` or + `$HOME/.cache/maturin_build_cache/` in order of preference + + :param install_new_packages: whether to install detected packages using the import hook even if they + are not already installed into the virtual environment or are installed in non-editable mode. + + :param force_rebuild: whether to always rebuild and skip checking whether anything has changed + + :param excluded_dir_names: directory names to exclude when determining whether a project has changed + and so whether the extension module needs to be rebuilt + + :param lock_timeout_seconds: a lock is required to prevent projects from being built concurrently. + If the lock is not released before this timeout is reached the import hook stops waiting and aborts + + :param show_warnings: whether to show compilation warnings + + """ + global IMPORTER + if IMPORTER is not None: + with contextlib.suppress(ValueError): + sys.meta_path.remove(IMPORTER) + IMPORTER = MaturinProjectImporter( + settings=settings, + build_dir=build_dir, + install_new_packages=install_new_packages, + force_rebuild=force_rebuild, + excluded_dir_names=excluded_dir_names, + lock_timeout_seconds=lock_timeout_seconds, + show_warnings=show_warnings, + ) + sys.meta_path.insert(0, IMPORTER) + return IMPORTER + + +def uninstall() -> None: + """Uninstall the project importer import hook.""" + global IMPORTER + if IMPORTER is not None: + with contextlib.suppress(ValueError): + sys.meta_path.remove(IMPORTER) + IMPORTER = None diff --git a/maturin/import_hook/rust_file_importer.py b/maturin/import_hook/rust_file_importer.py new file mode 100644 index 000000000..8600f5102 --- /dev/null +++ b/maturin/import_hook/rust_file_importer.py @@ -0,0 +1,291 @@ +import contextlib +import importlib +import importlib.util +import logging +import math +import os +import sys +import time +from importlib.machinery import ExtensionFileLoader, ModuleSpec +from pathlib import Path +from types import ModuleType +from typing import Optional, Sequence, Tuple, Union + +from maturin.import_hook._building import ( + BuildCache, + BuildStatus, + build_unpacked_wheel, + generate_project_for_single_rust_file, + maturin_output_has_warnings, +) +from maturin.import_hook._logging import logger +from maturin.import_hook._resolve_project import ProjectResolver +from maturin.import_hook.settings import MaturinSettings, MaturinSettingsProvider + +__all__ = ["MaturinRustFileImporter", "install", "uninstall", "IMPORTER"] + + +class MaturinRustFileImporter(importlib.abc.MetaPathFinder): + """An import hook for loading .rs files as though they were regular python modules.""" + + def __init__( + self, + *, + settings: Optional[Union[MaturinSettings, MaturinSettingsProvider]] = None, + build_dir: Optional[Path] = None, + force_rebuild: bool = False, + lock_timeout_seconds: Optional[float] = 120, + show_warnings: bool = True, + ) -> None: + self._force_rebuild = force_rebuild + self._resolver = ProjectResolver() + self._settings = settings + self._build_cache = BuildCache(build_dir, lock_timeout_seconds) + self._show_warnings = show_warnings + + def _get_settings(self, module_path: str, source_path: Path) -> MaturinSettings: + if isinstance(self._settings, MaturinSettings): + return self._settings + elif isinstance(self._settings, MaturinSettingsProvider): + return self._settings.get_settings(module_path, source_path) + else: + return MaturinSettings() + + def find_spec( + self, + fullname: str, + path: Optional[Sequence[Union[str, bytes]]] = None, + target: Optional[ModuleType] = None, + ) -> Optional[ModuleSpec]: + if fullname in sys.modules: + return None + + start = time.perf_counter() + + if logger.isEnabledFor(logging.DEBUG): + logger.debug('%s searching for "%s"', type(self).__name__, fullname) + + is_top_level_import = path is None + if is_top_level_import: + search_paths = [Path(p) for p in sys.path] + else: + assert path is not None + search_paths = [Path(os.fsdecode(p)) for p in path] + + module_name = fullname.split(".")[-1] + + spec = None + rebuilt = False + for search_path in search_paths: + single_rust_file_path = search_path / f"{module_name}.rs" + if single_rust_file_path.is_file(): + spec, rebuilt = self._import_rust_file( + fullname, module_name, single_rust_file_path + ) + if spec is not None: + break + + if spec is not None: + duration = time.perf_counter() - start + if rebuilt: + logger.info( + 'rebuilt and loaded module "%s" in %.3fs', fullname, duration + ) + else: + logger.debug('loaded module "%s" in %.3fs', fullname, duration) + return spec + + def _import_rust_file( + self, module_path: str, module_name: str, file_path: Path + ) -> Tuple[Optional[ModuleSpec], bool]: + logger.debug('importing rust file "%s" as "%s"', file_path, module_path) + + with self._build_cache.lock: + output_dir = self._build_cache.tmp_project_dir(file_path, module_name) + logger.debug("output dir: %s", output_dir) + settings = self._get_settings(module_path, file_path) + dist_dir = output_dir / "dist" + package_dir = dist_dir / module_name + + spec, reason = self._get_spec_for_up_to_date_extension_module( + package_dir, + module_path, + module_name, + file_path, + settings, + ) + if spec is not None: + return spec, False + logger.debug('module "%s" will be rebuilt because: %s', module_path, reason) + + logger.info('building "%s"', module_path) + logger.debug('creating project for "%s" and compiling', file_path) + start = time.perf_counter() + output_dir = generate_project_for_single_rust_file( + output_dir, file_path, settings.features + ) + maturin_output = build_unpacked_wheel( + output_dir / "Cargo.toml", dist_dir, settings + ) + logger.debug( + 'compiled "%s" in %.3fs', + file_path, + time.perf_counter() - start, + ) + + if self._show_warnings and maturin_output_has_warnings(maturin_output): + self._log_build_warnings(module_path, maturin_output, is_fresh=True) + extension_module_path = _find_extension_module( + dist_dir / module_name, module_name, require=True + ) + if extension_module_path is None: + logger.error( + 'cannot find extension module for "%s" after rebuild', module_path + ) + return None, True + build_status = BuildStatus( + extension_module_path.stat().st_mtime, + file_path, + settings.to_args(), + maturin_output, + ) + self._build_cache.store_build_status(build_status) + return ( + _get_spec_for_extension_module(module_path, extension_module_path), + True, + ) + + def _get_spec_for_up_to_date_extension_module( + self, + search_dir: Path, + module_path: str, + module_name: str, + source_path: Path, + settings: MaturinSettings, + ) -> Tuple[Optional[ModuleSpec], Optional[str]]: + """Return a spec for the given module at the given search_dir if it exists and is newer than the source + code that it is derived from. + """ + logger.debug('checking whether the module "%s" is up to date', module_path) + + if self._force_rebuild: + return None, "forcing rebuild" + extension_module_path = _find_extension_module( + search_dir, module_name, require=False + ) + if extension_module_path is None: + return None, "already built module not found" + + extension_module_mtime = extension_module_path.stat().st_mtime + if extension_module_mtime < source_path.stat().st_mtime: + return None, "module is out of date" + + build_status = self._build_cache.get_build_status(source_path) + if build_status is None: + return None, "no build status found" + if build_status.source_path != source_path: + return None, "source path in build status does not match the project dir" + if not math.isclose(build_status.build_mtime, extension_module_mtime): + return None, "installed package mtime does not match build status mtime" + if build_status.maturin_args != settings.to_args(): + return None, "current maturin args do not match the previous build" + + spec = _get_spec_for_extension_module(module_path, extension_module_path) + if spec is None: + return None, "module not found" + + logger.debug('module up to date: "%s" (%s)', module_path, spec.origin) + + if self._show_warnings and maturin_output_has_warnings( + build_status.maturin_output + ): + self._log_build_warnings( + module_path, build_status.maturin_output, is_fresh=False + ) + + return spec, None + + def _log_build_warnings( + self, module_path: str, maturin_output: str, is_fresh: bool + ) -> None: + prefix = "" if is_fresh else "the last " + message = '%sbuild of "%s" succeeded with warnings:\n%s' + if self._show_warnings: + logger.warning(message, prefix, module_path, maturin_output) + else: + logger.debug(message, prefix, module_path, maturin_output) + + +def _find_extension_module( + dir_path: Path, module_name: str, *, require: bool = False +) -> Optional[Path]: + # the suffixes include the platform tag and file extension eg '.cpython-311-x86_64-linux-gnu.so' + for suffix in importlib.machinery.EXTENSION_SUFFIXES: + extension_path = dir_path / f"{module_name}{suffix}" + if extension_path.exists(): + return extension_path + if require: + msg = f'could not find module "{module_name}" in "{dir_path}"' + raise ImportError(msg) + return None + + +def _get_spec_for_extension_module( + module_path: str, extension_module_path: Path +) -> Optional[ModuleSpec]: + return importlib.util.spec_from_loader( + module_path, ExtensionFileLoader(module_path, str(extension_module_path)) + ) + + +IMPORTER: Optional[MaturinRustFileImporter] = None + + +def install( + *, + settings: Optional[Union[MaturinSettings, MaturinSettingsProvider]] = None, + build_dir: Optional[Path] = None, + force_rebuild: bool = False, + lock_timeout_seconds: Optional[float] = 120, + show_warnings: bool = True, +) -> MaturinRustFileImporter: + """Install the 'rust file' importer to import .rs files as though + they were regular python modules. + + :param settings: settings corresponding to flags passed to maturin. Pass MaturinSettings to use the same + settings for every project or MaturinSettingsProvider to customize + + :param build_dir: where to put the compiled artifacts. defaults to `$MATURIN_BUILD_DIR`, + `sys.exec_prefix / 'maturin_build_cache'` or + `$HOME/.cache/maturin_build_cache/` in order of preference + + :param force_rebuild: whether to always rebuild and skip checking whether anything has changed + + :param lock_timeout_seconds: a lock is required to prevent projects from being built concurrently. + If the lock is not released before this timeout is reached the import hook stops waiting and aborts + + :param show_warnings: whether to show compilation warnings + + """ + global IMPORTER + if IMPORTER is not None: + with contextlib.suppress(ValueError): + sys.meta_path.remove(IMPORTER) + IMPORTER = MaturinRustFileImporter( + settings=settings, + build_dir=build_dir, + force_rebuild=force_rebuild, + lock_timeout_seconds=lock_timeout_seconds, + show_warnings=show_warnings, + ) + sys.meta_path.insert(0, IMPORTER) + return IMPORTER + + +def uninstall() -> None: + """Uninstall the rust file importer import hook.""" + global IMPORTER + if IMPORTER is not None: + with contextlib.suppress(ValueError): + sys.meta_path.remove(IMPORTER) + IMPORTER = None diff --git a/maturin/import_hook/settings.py b/maturin/import_hook/settings.py new file mode 100644 index 000000000..d94b031e0 --- /dev/null +++ b/maturin/import_hook/settings.py @@ -0,0 +1,52 @@ +from abc import ABC, abstractmethod +from dataclasses import dataclass +from pathlib import Path +from typing import List, Optional + +__all__ = ["MaturinSettings", "MaturinSettingsProvider"] + + +@dataclass +class MaturinSettings: + release: bool = False + strip: bool = False + quiet: bool = False + jobs: Optional[int] = None + features: Optional[List[str]] = None + all_features: bool = False + no_default_features: bool = False + frozen: bool = False + locked: bool = False + offline: bool = False + + def to_args(self) -> List[str]: + args = [] + if self.release: + args.append("--release") + if self.strip: + args.append("--strip") + if self.quiet: + args.append("--quiet") + if self.jobs is not None: + args.append("--jobs") + args.append(str(self.jobs)) + if self.features: + args.append("--features") + args.append(",".join(self.features)) + if self.all_features: + args.append("--all-features") + if self.no_default_features: + args.append("--no-default-features") + if self.frozen: + args.append("--frozen") + if self.locked: + args.append("--locked") + if self.offline: + args.append("--offline") + return args + + +class MaturinSettingsProvider(ABC): + @abstractmethod + def get_settings(self, module_path: str, source_path: Path) -> MaturinSettings: + raise NotImplementedError From a7282180abf008accaf1be7e8e849790a54f3487 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 27 Aug 2023 20:42:24 +0100 Subject: [PATCH 05/57] added new import hook instructions to guide --- guide/src/develop.md | 67 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 62 insertions(+), 5 deletions(-) diff --git a/guide/src/develop.md b/guide/src/develop.md index 414761d93..fe977d232 100644 --- a/guide/src/develop.md +++ b/guide/src/develop.md @@ -131,11 +131,68 @@ from maturin import import_hook # install the import hook with default settings import_hook.install() -# or you can specify bindings -import_hook.install(bindings="pyo3") -# and build in release mode instead of the default debug mode -import_hook.install(release=True) -# now you can start importing your Rust module +# when a rust package that is installed in editable mode is imported, +# that package will be automatically recompiled if necessary. import pyo3_pure + +# when a .rs file is imported a project will be created for it in the +# maturin build cache and the resulting library will be loaded +import subpackage.my_rust_script +``` + +The maturin project importer and the rust file importer can be used separately +```python +from maturin.import_hook import rust_file_importer +rust_file_importer.install() +from maturin.import_hook import package_importer +package_importer.install() +``` + +The import hook can be configured to control its behaviour +```python +from pathlib import Path +from maturin import import_hook +from maturin.import_hook.settings import MaturinSettings, MaturinSettingsProvider + +import_hook.install( + enable_package_importer=True, + enable_rs_file_importer=True, + settings=MaturinSettings( + release=True, + strip=True, + # ... + ), + show_warnings=True, + # ... +) + +# custom settings providers can be used to override settings of particular projects +# or implement custom logic such as loading settings from configuration files +class CustomSettings(MaturinSettingsProvider): + def get_settings(self, module_path: str, source_path: Path) -> MaturinSettings: + return MaturinSettings( + release=True, + strip=True, + # ... + ) + +import_hook.install( + enable_package_importer=True, + enable_rs_file_importer=True, + settings=CustomSettings(), + show_warnings=True, + # ... +) +``` + +The import hook internals can be examined by configuring the root logger and +calling `reset_logger` to propagate messages from the `maturin.import_hook` logger +to the root logger. +```python +import logging +logging.basicConfig(format='%(name)s [%(levelname)s] %(message)s', level=logging.DEBUG) +from maturin import import_hook +import_hook.reset_logger() +import_hook.install() ``` From 1b9ccecad2489b63aab9e851008da5d42586367a Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 27 Aug 2023 20:44:24 +0100 Subject: [PATCH 06/57] added import hook tests --- tests/common/import_hook.rs | 116 +++ tests/common/mod.rs | 1 + tests/import_hook/__init__.py | 0 tests/import_hook/blank-project/Cargo.lock | 273 +++++++ tests/import_hook/blank-project/Cargo.toml | 12 + .../import_hook/blank-project/pyproject.toml | 7 + tests/import_hook/blank-project/src/lib.rs | 6 + tests/import_hook/common.py | 232 ++++++ .../import_hook/rust_file_import/__init__.py | 0 .../absolute_import_helper.py | 29 + .../concurrent_import_helper.py | 15 + .../multiple_import_helper.py | 20 + .../rust_file_import/my_script_1.rs | 10 + .../rust_file_import/my_script_2.rs | 14 + .../rust_file_import/my_script_3.rs | 16 + .../rust_file_import/packages/__init__.py | 0 .../packages/multiple_import_helper.py | 5 + .../rust_file_import/packages/my_py_module.py | 2 + .../packages/my_rust_module.pyi | 1 + .../packages/my_rust_module.rs | 13 + .../packages/subpackage/__init__.py | 0 .../packages/subpackage/my_rust_module.pyi | 1 + .../packages/subpackage/my_rust_module.rs | 13 + .../packages/top_level_import_helper.py | 23 + .../rebuild_on_change_helper.py | 22 + .../rebuild_on_settings_change_helper.py | 30 + .../relative_import_helper.py | 29 + tests/import_hook/test_project_importer.py | 727 ++++++++++++++++++ tests/import_hook/test_rust_file_importer.py | 340 ++++++++ tests/import_hook/test_utilities.py | 334 ++++++++ tests/run.rs | 49 +- 31 files changed, 2338 insertions(+), 2 deletions(-) create mode 100644 tests/common/import_hook.rs create mode 100644 tests/import_hook/__init__.py create mode 100644 tests/import_hook/blank-project/Cargo.lock create mode 100644 tests/import_hook/blank-project/Cargo.toml create mode 100644 tests/import_hook/blank-project/pyproject.toml create mode 100644 tests/import_hook/blank-project/src/lib.rs create mode 100644 tests/import_hook/common.py create mode 100644 tests/import_hook/rust_file_import/__init__.py create mode 100644 tests/import_hook/rust_file_import/absolute_import_helper.py create mode 100644 tests/import_hook/rust_file_import/concurrent_import_helper.py create mode 100644 tests/import_hook/rust_file_import/multiple_import_helper.py create mode 100644 tests/import_hook/rust_file_import/my_script_1.rs create mode 100644 tests/import_hook/rust_file_import/my_script_2.rs create mode 100644 tests/import_hook/rust_file_import/my_script_3.rs create mode 100644 tests/import_hook/rust_file_import/packages/__init__.py create mode 100644 tests/import_hook/rust_file_import/packages/multiple_import_helper.py create mode 100644 tests/import_hook/rust_file_import/packages/my_py_module.py create mode 100644 tests/import_hook/rust_file_import/packages/my_rust_module.pyi create mode 100644 tests/import_hook/rust_file_import/packages/my_rust_module.rs create mode 100644 tests/import_hook/rust_file_import/packages/subpackage/__init__.py create mode 100644 tests/import_hook/rust_file_import/packages/subpackage/my_rust_module.pyi create mode 100644 tests/import_hook/rust_file_import/packages/subpackage/my_rust_module.rs create mode 100644 tests/import_hook/rust_file_import/packages/top_level_import_helper.py create mode 100644 tests/import_hook/rust_file_import/rebuild_on_change_helper.py create mode 100644 tests/import_hook/rust_file_import/rebuild_on_settings_change_helper.py create mode 100644 tests/import_hook/rust_file_import/relative_import_helper.py create mode 100644 tests/import_hook/test_project_importer.py create mode 100644 tests/import_hook/test_rust_file_importer.py create mode 100644 tests/import_hook/test_utilities.py diff --git a/tests/common/import_hook.rs b/tests/common/import_hook.rs new file mode 100644 index 000000000..1eaf0cea0 --- /dev/null +++ b/tests/common/import_hook.rs @@ -0,0 +1,116 @@ +use crate::common::{create_virtualenv, test_python_path}; +use anyhow::{bail, Result}; +use maturin::{BuildOptions, CargoOptions, Target}; +use serde_json; +use serde_json::{json, Value}; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::{env, fs, str}; + +pub fn test_import_hook( + virtualenv_name: &str, + test_script_path: &Path, + extra_packages: Vec<&str>, + extra_envs: BTreeMap<&str, &str>, + verbose: bool, +) -> Result<()> { + let python = test_python_path().map(PathBuf::from).unwrap_or_else(|| { + let target = Target::from_target_triple(None).unwrap(); + target.get_python() + }); + + let (venv_dir, python) = create_virtualenv(virtualenv_name, Some(python)).unwrap(); + + let pytest_args = vec![ + vec!["pytest"], + vec!["uniffi-bindgen"], + vec!["cffi"], + vec!["-e", "."], + ]; + let extras: Vec> = extra_packages.into_iter().map(|name| vec![name]).collect(); + for args in pytest_args.iter().chain(&extras) { + if verbose { + println!("installing {:?}", &args); + } + let status = Command::new(&python) + .args(["-m", "pip", "install", "--disable-pip-version-check"]) + .args(args) + .status() + .unwrap(); + if !status.success() { + bail!("failed to install: {:?}", &args); + } + } + + let path = env::var_os("PATH").unwrap(); + let mut paths = env::split_paths(&path).collect::>(); + paths.insert(0, venv_dir.join("bin")); + let path = env::join_paths(paths).unwrap(); + + let output = Command::new(&python) + .args(["-m", "pytest", test_script_path.to_str().unwrap()]) + .env("PATH", path) + .env("VIRTUAL_ENV", venv_dir) + .envs(extra_envs) + .output() + .unwrap(); + + if !output.status.success() { + bail!( + "import hook test failed: {}\n--- Stdout:\n{}\n--- Stderr:\n{}\n---\n", + output.status, + str::from_utf8(&output.stdout)?.trim(), + str::from_utf8(&output.stderr)?.trim(), + ); + } else if verbose { + println!( + "import hook test finished:\n--- Stdout:\n{}\n--- Stderr:\n{}\n---\n", + str::from_utf8(&output.stdout)?.trim(), + str::from_utf8(&output.stderr)?.trim(), + ) + } + Ok(()) +} + +pub fn resolve_all_packages() -> Result { + let mut resolved_packages = serde_json::Map::new(); + for path in fs::read_dir("test-crates")? { + let path = path?.path(); + if path.join("pyproject.toml").exists() { + let project_name = path.file_name().unwrap().to_str().unwrap().to_owned(); + resolved_packages.insert(project_name, resolve_package(&path).unwrap_or(Value::Null)); + } + } + Ok(serde_json::to_string(&Value::Object(resolved_packages))?) +} + +fn resolve_package(project_root: &Path) -> Result { + let manifest_path = if project_root.join("Cargo.toml").exists() { + project_root.join("Cargo.toml") + } else { + project_root.join("rust").join("Cargo.toml") + }; + + let build_options = BuildOptions { + cargo: CargoOptions { + manifest_path: Some(manifest_path.to_owned()), + ..Default::default() + }, + ..Default::default() + }; + let build_context = build_options.into_build_context(false, false, false)?; + let extension_module_dir = if build_context.project_layout.python_module.is_some() { + Some(build_context.project_layout.rust_module) + } else { + None + }; + + Ok(json!({ + "cargo_manifest_path": build_context.manifest_path, + "python_dir": build_context.project_layout.python_dir, + "python_module": build_context.project_layout.python_module, + "module_full_name": build_context.module_name, + "extension_module_dir": extension_module_dir, + })) +} diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 431ddcff9..aa2bafa46 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -9,6 +9,7 @@ use std::{env, io, str}; pub mod develop; pub mod errors; +pub mod import_hook; pub mod integration; pub mod other; diff --git a/tests/import_hook/__init__.py b/tests/import_hook/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/import_hook/blank-project/Cargo.lock b/tests/import_hook/blank-project/Cargo.lock new file mode 100644 index 000000000..64c0ac661 --- /dev/null +++ b/tests/import_hook/blank-project/Cargo.lock @@ -0,0 +1,273 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "blank-project" +version = "0.1.0" +dependencies = [ + "pyo3", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "indoc" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa799dd5ed20a7e349f3b4639aa80d74549c81716d9ec4f994c9b5815598306" + +[[package]] +name = "libc" +version = "0.2.147" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" + +[[package]] +name = "lock_api" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets", +] + +[[package]] +name = "proc-macro2" +version = "1.0.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyo3" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e681a6cfdc4adcc93b4d3cf993749a4552018ee0a9b65fc0ccfad74352c72a38" +dependencies = [ + "cfg-if", + "indoc", + "libc", + "memoffset", + "parking_lot", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "076c73d0bc438f7a4ef6fdd0c3bb4732149136abd952b110ac93e4edb13a6ba5" +dependencies = [ + "once_cell", + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e53cee42e77ebe256066ba8aa77eff722b3bb91f3419177cf4cd0f304d3284d9" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfeb4c99597e136528c6dd7d5e3de5434d1ceaf487436a3f03b2d56b6fc9efd1" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "947dc12175c254889edc0c02e399476c2f652b4b9ebd123aa655c224de259536" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "quote" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "smallvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "target-lexicon" +version = "0.12.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d0e916b1148c8e263850e1ebcbd046f333e0683c724876bb0da63ea4373dc8a" + +[[package]] +name = "unicode-ident" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" + +[[package]] +name = "unindent" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1766d682d402817b5ac4490b3c3002d91dfa0d22812f341609f97b08757359c" + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" diff --git a/tests/import_hook/blank-project/Cargo.toml b/tests/import_hook/blank-project/Cargo.toml new file mode 100644 index 000000000..d2c0c3322 --- /dev/null +++ b/tests/import_hook/blank-project/Cargo.toml @@ -0,0 +1,12 @@ +[package] +authors = [] +name = "blank-project" +version = "0.1.0" +edition = "2021" +description = "" + +[dependencies] +pyo3 = { version = "0.19.0", features = ["extension-module"] } + +[lib] +crate-type = ["cdylib"] diff --git a/tests/import_hook/blank-project/pyproject.toml b/tests/import_hook/blank-project/pyproject.toml new file mode 100644 index 000000000..34bfa875c --- /dev/null +++ b/tests/import_hook/blank-project/pyproject.toml @@ -0,0 +1,7 @@ +[build-system] +requires = ["maturin>=1.0,<2.0"] +build-backend = "maturin" + +[project] +name = "blank_project" +version = "0.1.0" diff --git a/tests/import_hook/blank-project/src/lib.rs b/tests/import_hook/blank-project/src/lib.rs new file mode 100644 index 000000000..5bd3c6b53 --- /dev/null +++ b/tests/import_hook/blank-project/src/lib.rs @@ -0,0 +1,6 @@ +use pyo3::prelude::*; + +#[pymodule] +fn blank_project(_py: Python, _m: &PyModule) -> PyResult<()> { + Ok(()) +} diff --git a/tests/import_hook/common.py b/tests/import_hook/common.py new file mode 100644 index 000000000..8b540ec46 --- /dev/null +++ b/tests/import_hook/common.py @@ -0,0 +1,232 @@ +import os +import shutil +import site +import subprocess +import sys +import tempfile +import time +from pathlib import Path +from typing import List, Optional, Tuple + +from maturin.import_hook.project_importer import _fix_direct_url, _load_dist_info + +verbose = True + + +script_dir = Path(__file__).resolve().parent +maturin_dir = script_dir.parent.parent +test_crates = maturin_dir / "test-crates" + + +IMPORT_HOOK_HEADER = """ +import logging +logging.basicConfig(format='%(name)s [%(levelname)s] %(message)s', level=logging.DEBUG) + +from maturin import import_hook +import_hook.reset_logger() +import_hook.install() +""" + + +EXCLUDED_PROJECTS = { + "hello-world", # not imported as a python module (subprocess only) + "license-test", # not imported as a python module (subprocess only) + "pyo3-bin", # not imported as a python module (subprocess only) +} + + +def with_underscores(project_name: str) -> str: + return project_name.replace("-", "_") + + +def all_test_crate_names() -> list[str]: + return sorted( + p.name + for p in test_crates.iterdir() + if (p / "check_installed/check_installed.py").exists() + and (p / "pyproject.toml").exists() + if p.name not in EXCLUDED_PROJECTS + ) + + +def mixed_test_crate_names() -> list[str]: + return [name for name in all_test_crate_names() if "mixed" in name] + + +def run_python( + args: List[str], + cwd: Path, + *, + python_path: Optional[List[Path]] = None, + quiet: bool = False, + expect_error: bool = False, +) -> Tuple[str, float]: + start = time.perf_counter() + + env = os.environ + if python_path is not None: + env["PYTHONPATH"] = ":".join(str(p) for p in python_path) + + cmd = [sys.executable, *args] + try: + proc = subprocess.run( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=True, + cwd=cwd, + env=env, + ) + output = proc.stdout.decode() + except subprocess.CalledProcessError as e: + output = e.stdout.decode() + if verbose and not quiet and not expect_error: + message = "\n".join( + [ + "-" * 40, + "ERROR:", + subprocess.list2cmdline(cmd), + "", + output, + "-" * 40, + ] + ) + print(message, file=sys.stderr) + if not expect_error: + raise + duration = time.perf_counter() - start + + if verbose and not quiet: + print("-" * 40) + print(subprocess.list2cmdline(cmd)) + print(output) + print("-" * 40) + + return output, duration + + +def run_python_code( + python_script: str, + *, + args: Optional[List[str]] = None, + cwd: Optional[Path] = None, + python_path: Optional[list[Path]] = None, + quiet: bool = False, + expect_error: bool = False, +) -> Tuple[str, float]: + with tempfile.TemporaryDirectory("run_python_code") as tmpdir_str: + tmpdir = Path(tmpdir_str) + tmp_script_path = tmpdir / "script.py" + tmp_script_path.write_text(python_script) + + python_args = [str(tmp_script_path)] + if args is not None: + python_args.extend(args) + + return run_python( + python_args, + cwd=cwd or tmpdir, + python_path=python_path, + quiet=quiet, + expect_error=expect_error, + ) + + +def log(message: str) -> None: + if verbose: + print(message) + + +def uninstall(project_name: str) -> None: + log(f"uninstalling {project_name}") + subprocess.check_call( + [sys.executable, "-m", "pip", "uninstall", "-y", project_name] + ) + + +def install_editable(project_dir: Path) -> None: + """Install the given project to the virtualenv in editable mode.""" + log(f"installing {project_dir.name} in editable/unpacked mode") + env = os.environ.copy() + env["VIRTUAL_ENV"] = sys.exec_prefix + subprocess.check_call(["maturin", "develop"], cwd=project_dir, env=env) + package_name = with_underscores(project_dir.name) + _fix_direct_url(project_dir, package_name) + + +def install_non_editable(project_dir: Path) -> None: + log(f"installing {project_dir.name} in non-editable mode") + subprocess.check_call([sys.executable, "-m", "pip", "install", str(project_dir)]) + + +def _is_installed_as_pth(project_name: str) -> bool: + package_name = with_underscores(project_name) + return any( + (Path(path) / f"{package_name}.pth").exists() for path in site.getsitepackages() + ) + + +def _is_installed_editable_with_direct_url( + project_name: str, project_dir: Path +) -> bool: + package_name = with_underscores(project_name) + for path in site.getsitepackages(): + linked_path, is_editable = _load_dist_info(Path(path), package_name) + if linked_path == project_dir: + if not is_editable: + log(f'project "{project_name}" is installed but not in editable mode') + return is_editable + else: + log( + f'found linked path "{linked_path}" for project "{project_name}". Expected "{project_dir}"' + ) + return False + + +def is_installed_correctly( + project_name: str, project_dir: Path, is_mixed: bool +) -> bool: + installed_as_pth = _is_installed_as_pth(project_name) + installed_editable_with_direct_url = _is_installed_editable_with_direct_url( + project_name, project_dir + ) + log( + f"checking if {project_name} is installed correctly. " + f"{is_mixed=}, {installed_as_pth=} {installed_editable_with_direct_url=}" + ) + return installed_editable_with_direct_url and (installed_as_pth == is_mixed) + + +def get_project_copy(project_dir: Path, output_path: Path) -> Path: + # using shutil.copy instead of the default shutil.copy2 because we want mtimes to be updated on copy + project_copy_dir = Path( + shutil.copytree(project_dir, output_path, copy_function=shutil.copy) + ) + assert ( + next(project_copy_dir.rglob("*.so"), None) is None + ), f"project {project_dir.name} is not clean" + return project_copy_dir + + +def create_project_from_blank_template( + project_name: str, output_path: Path, *, mixed: bool +) -> Path: + project_dir = get_project_copy(script_dir / "blank-project", output_path) + project_name = project_name.replace("_", "-") + package_name = project_name.replace("-", "_") + for path in [ + project_dir / "pyproject.toml", + project_dir / "Cargo.toml", + project_dir / "src/lib.rs", + ]: + path.write_text( + path.read_text() + .replace("blank-project", project_name) + .replace("blank_project", package_name) + ) + if mixed: + (project_dir / package_name).mkdir() + (project_dir / package_name / "__init__.py").write_text( + f"from .{package_name} import *" + ) + return project_dir diff --git a/tests/import_hook/rust_file_import/__init__.py b/tests/import_hook/rust_file_import/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/import_hook/rust_file_import/absolute_import_helper.py b/tests/import_hook/rust_file_import/absolute_import_helper.py new file mode 100644 index 000000000..b9d8ab805 --- /dev/null +++ b/tests/import_hook/rust_file_import/absolute_import_helper.py @@ -0,0 +1,29 @@ +# ruff: noqa: E402 +import logging + +logging.basicConfig(format="%(name)s [%(levelname)s] %(message)s", level=logging.DEBUG) + +from maturin import import_hook + +import_hook.reset_logger() +import_hook.install() + +import packages.my_py_module + +assert packages.my_py_module.do_something_py(1, 2) == 3 + +import packages.my_rust_module + +assert packages.my_rust_module.do_something(1, 2) == 3 + +from packages import my_rust_module + +assert my_rust_module.do_something(1, 2) == 3 + + +# modules with the same name do not clash +import packages.subpackage.my_rust_module + +assert packages.subpackage.my_rust_module.get_num() == 42 + +print("SUCCESS") diff --git a/tests/import_hook/rust_file_import/concurrent_import_helper.py b/tests/import_hook/rust_file_import/concurrent_import_helper.py new file mode 100644 index 000000000..8c8043209 --- /dev/null +++ b/tests/import_hook/rust_file_import/concurrent_import_helper.py @@ -0,0 +1,15 @@ +# ruff: noqa: E402 +import logging + +logging.basicConfig(format="%(name)s [%(levelname)s] %(message)s", level=logging.DEBUG) + +from maturin import import_hook + +import_hook.reset_logger() +import_hook.install() + +import packages.my_rust_module + +assert packages.my_rust_module.do_something(1, 2) == 3 + +print("SUCCESS") diff --git a/tests/import_hook/rust_file_import/multiple_import_helper.py b/tests/import_hook/rust_file_import/multiple_import_helper.py new file mode 100644 index 000000000..7e1e3747c --- /dev/null +++ b/tests/import_hook/rust_file_import/multiple_import_helper.py @@ -0,0 +1,20 @@ +# ruff: noqa: E402 +import logging + +logging.basicConfig(format="%(name)s [%(levelname)s] %(message)s", level=logging.DEBUG) +logging.getLogger("maturin.import_hook").setLevel(logging.DEBUG) + +from maturin import import_hook + +import_hook.reset_logger() +import_hook.install() + +import packages.subpackage.my_rust_module + +assert packages.subpackage.my_rust_module.get_num() == 42 + +import packages.multiple_import_helper + +assert packages.multiple_import_helper.foo() == 142 + +print("SUCCESS") diff --git a/tests/import_hook/rust_file_import/my_script_1.rs b/tests/import_hook/rust_file_import/my_script_1.rs new file mode 100644 index 000000000..0e57895c1 --- /dev/null +++ b/tests/import_hook/rust_file_import/my_script_1.rs @@ -0,0 +1,10 @@ +use pyo3::prelude::*; + +#[pyfunction] +fn get_num() -> usize { 10 } + +#[pymodule] +fn my_script(_py: Python, m: &PyModule) -> PyResult<()> { + m.add_wrapped(wrap_pyfunction!(get_num))?; + Ok(()) +} diff --git a/tests/import_hook/rust_file_import/my_script_2.rs b/tests/import_hook/rust_file_import/my_script_2.rs new file mode 100644 index 000000000..8e72d25e3 --- /dev/null +++ b/tests/import_hook/rust_file_import/my_script_2.rs @@ -0,0 +1,14 @@ +use pyo3::prelude::*; + +#[pyfunction] +fn get_num() -> usize { 20 } + +#[pyfunction] +fn get_other_num() -> usize { 100 } + +#[pymodule] +fn my_script(_py: Python, m: &PyModule) -> PyResult<()> { + m.add_wrapped(wrap_pyfunction!(get_num))?; + m.add_wrapped(wrap_pyfunction!(get_other_num))?; + Ok(()) +} diff --git a/tests/import_hook/rust_file_import/my_script_3.rs b/tests/import_hook/rust_file_import/my_script_3.rs new file mode 100644 index 000000000..faa2e5575 --- /dev/null +++ b/tests/import_hook/rust_file_import/my_script_3.rs @@ -0,0 +1,16 @@ +use pyo3::prelude::*; + +#[pyfunction] +fn get_num() -> usize { + if cfg!(feature = "large_number") { + 100 + } else { + 10 + } +} + +#[pymodule] +fn my_script(_py: Python, m: &PyModule) -> PyResult<()> { + m.add_wrapped(wrap_pyfunction!(get_num))?; + Ok(()) +} diff --git a/tests/import_hook/rust_file_import/packages/__init__.py b/tests/import_hook/rust_file_import/packages/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/import_hook/rust_file_import/packages/multiple_import_helper.py b/tests/import_hook/rust_file_import/packages/multiple_import_helper.py new file mode 100644 index 000000000..a3ae81356 --- /dev/null +++ b/tests/import_hook/rust_file_import/packages/multiple_import_helper.py @@ -0,0 +1,5 @@ +from .subpackage import my_rust_module + + +def foo() -> int: + return my_rust_module.get_num() + 100 diff --git a/tests/import_hook/rust_file_import/packages/my_py_module.py b/tests/import_hook/rust_file_import/packages/my_py_module.py new file mode 100644 index 000000000..23ca8a78f --- /dev/null +++ b/tests/import_hook/rust_file_import/packages/my_py_module.py @@ -0,0 +1,2 @@ +def do_something_py(a: int, b: int) -> int: + return a + b diff --git a/tests/import_hook/rust_file_import/packages/my_rust_module.pyi b/tests/import_hook/rust_file_import/packages/my_rust_module.pyi new file mode 100644 index 000000000..6c09e817b --- /dev/null +++ b/tests/import_hook/rust_file_import/packages/my_rust_module.pyi @@ -0,0 +1 @@ +def do_something(a: int, b: int) -> int: ... diff --git a/tests/import_hook/rust_file_import/packages/my_rust_module.rs b/tests/import_hook/rust_file_import/packages/my_rust_module.rs new file mode 100644 index 000000000..12e07b998 --- /dev/null +++ b/tests/import_hook/rust_file_import/packages/my_rust_module.rs @@ -0,0 +1,13 @@ +use pyo3::prelude::*; +use pyo3::wrap_pyfunction; + +#[pyfunction] +pub fn do_something(a: usize, b: usize) -> PyResult { + Ok(a + b) +} + +#[pymodule] +pub fn my_rust_module(_py: Python, m: &PyModule) -> PyResult<()> { + m.add_wrapped(wrap_pyfunction!(do_something))?; + Ok(()) +} diff --git a/tests/import_hook/rust_file_import/packages/subpackage/__init__.py b/tests/import_hook/rust_file_import/packages/subpackage/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/import_hook/rust_file_import/packages/subpackage/my_rust_module.pyi b/tests/import_hook/rust_file_import/packages/subpackage/my_rust_module.pyi new file mode 100644 index 000000000..b8a0add39 --- /dev/null +++ b/tests/import_hook/rust_file_import/packages/subpackage/my_rust_module.pyi @@ -0,0 +1 @@ +def get_num() -> int: ... diff --git a/tests/import_hook/rust_file_import/packages/subpackage/my_rust_module.rs b/tests/import_hook/rust_file_import/packages/subpackage/my_rust_module.rs new file mode 100644 index 000000000..f610cc69d --- /dev/null +++ b/tests/import_hook/rust_file_import/packages/subpackage/my_rust_module.rs @@ -0,0 +1,13 @@ +use pyo3::prelude::*; +use pyo3::wrap_pyfunction; + +#[pyfunction] +pub fn get_num() -> PyResult { + Ok(42) +} + +#[pymodule] +pub fn my_rust_module(_py: Python, m: &PyModule) -> PyResult<()> { + m.add_wrapped(wrap_pyfunction!(get_num))?; + Ok(()) +} diff --git a/tests/import_hook/rust_file_import/packages/top_level_import_helper.py b/tests/import_hook/rust_file_import/packages/top_level_import_helper.py new file mode 100644 index 000000000..6ffda728e --- /dev/null +++ b/tests/import_hook/rust_file_import/packages/top_level_import_helper.py @@ -0,0 +1,23 @@ +# ruff: noqa: E402 +import logging + +logging.basicConfig(format="%(name)s [%(levelname)s] %(message)s", level=logging.DEBUG) + +from maturin import import_hook + +import_hook.reset_logger() +import_hook.install() + +import my_py_module + +assert my_py_module.do_something_py(1, 2) == 3 + +import my_rust_module + +assert my_rust_module.do_something(1, 2) == 3 + +import my_rust_module + +assert my_rust_module.do_something(1, 2) == 3 + +print("SUCCESS") diff --git a/tests/import_hook/rust_file_import/rebuild_on_change_helper.py b/tests/import_hook/rust_file_import/rebuild_on_change_helper.py new file mode 100644 index 000000000..ea804a027 --- /dev/null +++ b/tests/import_hook/rust_file_import/rebuild_on_change_helper.py @@ -0,0 +1,22 @@ +# ruff: noqa: E402 +import logging + +logging.basicConfig(format="%(name)s [%(levelname)s] %(message)s", level=logging.DEBUG) + +from maturin import import_hook + +import_hook.reset_logger() +import_hook.install() + +from my_script import get_num + +print(f"get_num = {get_num()}") + +try: + from my_script import get_other_num +except ImportError: + print("failed to import get_other_num") +else: + print(f"get_other_num = {get_other_num()}") + +print("SUCCESS") diff --git a/tests/import_hook/rust_file_import/rebuild_on_settings_change_helper.py b/tests/import_hook/rust_file_import/rebuild_on_settings_change_helper.py new file mode 100644 index 000000000..aa0b2a023 --- /dev/null +++ b/tests/import_hook/rust_file_import/rebuild_on_settings_change_helper.py @@ -0,0 +1,30 @@ +# ruff: noqa: E402 +import logging +import sys +from pathlib import Path + +logging.basicConfig(format="%(name)s [%(levelname)s] %(message)s", level=logging.DEBUG) + +from maturin import import_hook +from maturin.import_hook.settings import MaturinSettings, MaturinSettingsProvider + +import_hook.reset_logger() + + +class CustomSettingsProvider(MaturinSettingsProvider): + def get_settings(self, module_path: str, source_path: Path) -> MaturinSettings: + if len(sys.argv) > 1 and sys.argv[1] == "LARGE_NUMBER": + print(f"building {module_path} with large_number feature enabled") + return MaturinSettings(features=["pyo3/extension-module", "large_number"]) + else: + print(f"building {module_path} with default settings") + return MaturinSettings() + + +import_hook.install(settings=CustomSettingsProvider()) + + +from my_script import get_num + +print(f"get_num = {get_num()}") +print("SUCCESS") diff --git a/tests/import_hook/rust_file_import/relative_import_helper.py b/tests/import_hook/rust_file_import/relative_import_helper.py new file mode 100644 index 000000000..0888bb2a7 --- /dev/null +++ b/tests/import_hook/rust_file_import/relative_import_helper.py @@ -0,0 +1,29 @@ +# ruff: noqa: E402 +import logging + +logging.basicConfig(format="%(name)s [%(levelname)s] %(message)s", level=logging.DEBUG) + +from maturin import import_hook + +import_hook.reset_logger() +import_hook.install() + +from .packages import my_py_module + +assert my_py_module.do_something_py(1, 2) == 3 + +from .packages import my_rust_module + +assert my_rust_module.do_something(1, 2) == 3 + +from .packages import my_rust_module + +assert my_rust_module.do_something(1, 2) == 3 + + +# modules with the same name do not clash +from .packages.subpackage import my_rust_module as other_module + +assert other_module.get_num() == 42 + +print("SUCCESS") diff --git a/tests/import_hook/test_project_importer.py b/tests/import_hook/test_project_importer.py new file mode 100644 index 000000000..91e3d88cd --- /dev/null +++ b/tests/import_hook/test_project_importer.py @@ -0,0 +1,727 @@ +import multiprocessing +import os +import re +import shutil +from pathlib import Path + +import pytest + +from .common import ( + IMPORT_HOOK_HEADER, + all_test_crate_names, + create_project_from_blank_template, + get_project_copy, + install_editable, + install_non_editable, + is_installed_correctly, + log, + mixed_test_crate_names, + run_python, + run_python_code, + script_dir, + test_crates, + uninstall, + with_underscores, +) + +""" +These tests ensure the correct functioning of the project importer import hook. +The tests are intended to be run as part of the tests in `run.rs` +which provides a clean virtual environment for these tests to use. +""" + +MATURIN_BUILD_CACHE = test_crates / "targets/import_hook_project_importer_build_cache" + +os.environ["CARGO_TARGET_DIR"] = str( + test_crates / "targets/import_hook_project_importer" +) +os.environ["MATURIN_BUILD_DIR"] = str(MATURIN_BUILD_CACHE) + + +def _clear_build_cache() -> None: + if MATURIN_BUILD_CACHE.exists(): + log("clearing build cache") + shutil.rmtree(MATURIN_BUILD_CACHE) + + +@pytest.mark.parametrize( + "project_name", + # path dependencies tested separately + sorted(set(all_test_crate_names()) - {"pyo3-mixed-with-path-dep"}), +) +def test_install_from_script_inside(tmp_path: Path, project_name: str) -> None: + """This test ensures that when a script is run from within a maturin project, the + import hook can identify and install the containing project even if it is not + already installed. + + limitation: if the project has python dependencies then those dependencies will be installed + when the import hook triggers installation of the project but unlike the maturin project + which the import hook handles specially, other installed projects may not become available + until the interpreter is restarted (or the site module is reloaded) + """ + _clear_build_cache() + uninstall(project_name) + + project_dir = get_project_copy(test_crates / project_name, tmp_path / project_name) + + check_installed_dir = project_dir / "check_installed" + check_installed_path = check_installed_dir / "check_installed.py" + check_installed_path.write_text( + f"{IMPORT_HOOK_HEADER}\n\n{check_installed_path.read_text()}" + ) + + empty_dir = tmp_path / "empty" + empty_dir.mkdir() + + output1, duration1 = run_python([str(check_installed_path)], cwd=empty_dir) + assert "SUCCESS" in output1 + assert _rebuilt_message(project_name) in output1 + assert _up_to_date_message(project_name) not in output1 + + assert is_installed_correctly(project_name, project_dir, "mixed" in project_name) + + output2, duration2 = run_python([str(check_installed_path)], cwd=empty_dir) + assert "SUCCESS" in output2 + assert _rebuilt_message(project_name) not in output2 + assert _up_to_date_message(project_name) in output2 + + assert duration2 < duration1 + + assert is_installed_correctly(project_name, project_dir, "mixed" in project_name) + + +@pytest.mark.parametrize("project_name", ["pyo3-mixed", "pyo3-pure"]) +def test_do_not_install_from_script_inside(tmp_path: Path, project_name: str) -> None: + """This test ensures that when the import hook works correctly when it is + configured to not rebuild/install projects if they aren't already installed. + """ + _clear_build_cache() + uninstall(project_name) + + project_dir = get_project_copy(test_crates / project_name, tmp_path / project_name) + + check_installed_path = project_dir / "check_installed/check_installed.py" + header = """ +import logging +logging.basicConfig(format='%(name)s [%(levelname)s] %(message)s', level=logging.DEBUG) + +from maturin import import_hook +import_hook.reset_logger() +from maturin.import_hook import project_importer +project_importer.install(install_new_packages=False) +""" + check_installed_path.write_text(f"{header}\n\n{check_installed_path.read_text()}") + + empty_dir = tmp_path / "empty" + empty_dir.mkdir() + + output1, _ = run_python( + [str(check_installed_path)], cwd=empty_dir, expect_error=True, quiet=True + ) + assert ( + f'package "{with_underscores(project_name)}" is not already ' + f"installed and install_new_packages=False. Not importing" + ) in output1 + assert "SUCCESS" not in output1 + + install_editable(project_dir) + assert is_installed_correctly(project_name, project_dir, "mixed" in project_name) + + output2, _ = run_python([str(check_installed_path)], cwd=empty_dir) + assert "SUCCESS" in output2 + assert ( + f'package "{with_underscores(project_name)}" will be rebuilt because: no build status found' + in output2 + ) + assert _rebuilt_message(project_name) in output2 + + output3, _ = run_python([str(check_installed_path)], cwd=empty_dir) + assert "SUCCESS" in output3 + assert _rebuilt_message(project_name) not in output3 + assert _up_to_date_message(project_name) in output3 + + +@pytest.mark.parametrize("project_name", ["pyo3-mixed", "pyo3-pure"]) +def test_do_not_rebuild_if_installed_non_editable( + tmp_path: Path, project_name: str +) -> None: + """This test ensures that if a maturin project is installed in non-editable + mode then the import hook will not rebuild it or re-install it in editable mode. + """ + _clear_build_cache() + uninstall(project_name) + project_dir = get_project_copy(test_crates / project_name, tmp_path / project_name) + install_non_editable(project_dir) + + workspace = tmp_path / "workspace" + workspace.mkdir() + + check_installed_dir = project_dir / "check_installed" + check_installed_path = check_installed_dir / "check_installed.py" + header = """ +import sys +import logging +logging.basicConfig(format='%(name)s [%(levelname)s] %(message)s', level=logging.DEBUG) +from maturin import import_hook +import_hook.reset_logger() +install_new_packages = len(sys.argv) > 1 and sys.argv[1] == 'INSTALL_NEW' +print(f'{install_new_packages=}') +import_hook.install(install_new_packages=install_new_packages) +""" + check_installed_path.write_text(f"{header}\n\n{check_installed_path.read_text()}") + shutil.copy(check_installed_path, workspace) + + (project_dir / "src/lib.rs").write_text("") # will break once rebuilt + + # when outside the project, can still detect non-editable installed projects via dist-info + output1, _ = run_python(["check_installed.py"], cwd=workspace) + assert "SUCCESS" in output1 + assert "install_new_packages=False" in output1 + assert f'found project linked by dist-info: "{project_dir}"' in output1 + assert ( + "package not installed in editable-mode and install_new_packages=False. not rebuilding" + in output1 + ) + + # when inside the project, will detect the project above + output2, _ = run_python(["check_installed.py"], cwd=check_installed_dir) + assert "SUCCESS" in output2 + assert "install_new_packages=False" in output2 + assert "found project above the search path:" in output2 + assert ( + "package not installed in editable-mode and install_new_packages=False. not rebuilding" + in output2 + ) + + output3, _ = run_python( + ["check_installed.py", "INSTALL_NEW"], + cwd=workspace, + quiet=True, + expect_error=True, + ) + assert "SUCCESS" not in output3 + assert "install_new_packages=True" in output3 + assert ( + f"ImportError: dynamic module does not define module " + f"export function (PyInit_{with_underscores(project_name)})" + ) in output3 + + +@pytest.mark.parametrize("initially_mixed", [False, True]) +@pytest.mark.parametrize( + "project_name", + # path dependencies tested separately + sorted(set(all_test_crate_names()) - {"pyo3-mixed-with-path-dep"}), +) +def test_import_editable_installed_rebuild( + tmp_path: Path, project_name: str, initially_mixed: bool +) -> None: + """This test ensures that an editable installed project is rebuilt when necessary if the import + hook is active. This applies to mixed projects (which are installed as .pth files into + site-packages when installed in editable mode) as well as pure projects (which are copied to site-packages + when with a link back to the source directory when installed in editable mode). + + This is tested with the project initially being mixed and initially being pure to test that the import hook + works even if the project changes significantly (eg from mixed to pure) + """ + _clear_build_cache() + uninstall(project_name) + + check_installed = ( + test_crates / project_name / "check_installed/check_installed.py" + ).read_text() + + project_dir = create_project_from_blank_template( + project_name, tmp_path / project_name, mixed=initially_mixed + ) + + log(f"installing blank project as {project_name}") + + install_editable(project_dir) + assert is_installed_correctly(project_name, project_dir, initially_mixed) + + # without the import hook the installation test is expected to fail because the project should not be installed yet + output0, _ = run_python_code(check_installed, quiet=True, expect_error=True) + assert ( + "AttributeError" in output0 + or "ImportError" in output0 + or "ModuleNotFoundError" in output0 + ) + + check_installed = f"{IMPORT_HOOK_HEADER}\n\n{check_installed}" + + log("overwriting blank project with genuine project without re-installing") + shutil.rmtree(project_dir) + get_project_copy(test_crates / project_name, project_dir) + + output1, duration1 = run_python_code(check_installed) + assert "SUCCESS" in output1 + assert _rebuilt_message(project_name) in output1 + assert _up_to_date_message(project_name) not in output1 + + assert is_installed_correctly(project_name, project_dir, "mixed" in project_name) + + output2, duration2 = run_python_code(check_installed) + assert "SUCCESS" in output2 + assert _rebuilt_message(project_name) not in output2 + assert _up_to_date_message(project_name) in output2 + + assert duration2 < duration1 + + assert is_installed_correctly(project_name, project_dir, "mixed" in project_name) + + +@pytest.mark.parametrize( + "project_name", + # path dependencies tested separately + sorted(set(mixed_test_crate_names()) - {"pyo3-mixed-with-path-dep"}), +) +def test_import_editable_installed_mixed_missing( + tmp_path: Path, project_name: str +) -> None: + """This test ensures that editable installed mixed projects are rebuilt if they are imported + and their artifacts are missing. + + This can happen when cleaning untracked files from git for example. + + This only affects mixed projects because artifacts of editable installed pure projects are + copied to site-packages instead. + """ + _clear_build_cache() + uninstall(project_name) + + # making a copy because editable installation may write files into the project directory + project_dir = get_project_copy(test_crates / project_name, tmp_path / project_name) + project_backup_dir = get_project_copy( + test_crates / project_name, tmp_path / f"backup_{project_name}" + ) + + install_editable(project_dir) + assert is_installed_correctly(project_name, project_dir, "mixed" in project_name) + + check_installed = test_crates / project_name / "check_installed/check_installed.py" + + log( + "checking that check_installed works without the import hook right after installing" + ) + output0, _ = run_python_code(check_installed.read_text()) + assert "SUCCESS" in output0 + + check_installed_script = f"{IMPORT_HOOK_HEADER}\n\n{check_installed.read_text()}" + + shutil.rmtree(project_dir) + shutil.copytree(project_backup_dir, project_dir) + + log("checking that the import hook rebuilds the project") + + output1, duration1 = run_python_code(check_installed_script) + assert "SUCCESS" in output1 + assert _rebuilt_message(project_name) in output1 + assert _up_to_date_message(project_name) not in output1 + + output2, duration2 = run_python_code(check_installed_script) + assert "SUCCESS" in output2 + assert _rebuilt_message(project_name) not in output2 + assert _up_to_date_message(project_name) in output2 + + assert duration2 < duration1 + + assert is_installed_correctly(project_name, project_dir, True) + + +@pytest.mark.parametrize("mixed", [False, True]) +@pytest.mark.parametrize("initially_mixed", [False, True]) +def test_concurrent_import(tmp_path: Path, initially_mixed: bool, mixed: bool) -> None: + """This test ensures that if multiple scripts attempt to use the import hook concurrently, + that the project still installs correctly and does not crash. + + This test uses a blank project initially to ensure that a rebuild is necessary to be + able to use the project. + """ + if mixed: + project_name = "pyo3-mixed" + check_installed = """ +import pyo3_mixed +assert pyo3_mixed.get_42() == 42 +print('SUCCESS') +""" + else: + project_name = "pyo3-pure" + check_installed = """ +import pyo3_pure +assert pyo3_pure.DummyClass.get_42() == 42 +print('SUCCESS') +""" + + _clear_build_cache() + uninstall(project_name) + + check_installed_with_hook = f"{IMPORT_HOOK_HEADER}\n\n{check_installed}" + + project_dir = create_project_from_blank_template( + project_name, tmp_path / project_name, mixed=initially_mixed + ) + + log(f"initially mixed: {initially_mixed}, mixed: {mixed}") + log(f"installing blank project as {project_name}") + + install_editable(project_dir) + assert is_installed_correctly(project_name, project_dir, initially_mixed) + + shutil.rmtree(project_dir) + get_project_copy(test_crates / project_name, project_dir) + + args = {"python_script": check_installed_with_hook, "quiet": True} + with multiprocessing.Pool(processes=3) as pool: + p1 = pool.apply_async(run_python_code, kwds=args) + p2 = pool.apply_async(run_python_code, kwds=args) + p3 = pool.apply_async(run_python_code, kwds=args) + + output_1, duration_1 = p1.get() + output_2, duration_2 = p2.get() + output_3, duration_3 = p3.get() + + log("output 1") + log(output_1) + log("output 2") + log(output_2) + log("output 3") + log(output_3) + + num_compilations = 0 + num_up_to_date = 0 + num_waiting = 0 + for output in [output_1, output_2, output_3]: + assert "SUCCESS" in output + + if "waiting on lock" in output: + num_waiting += 1 + + if _up_to_date_message(project_name) in output: + num_up_to_date += 1 + + if _rebuilt_message(project_name) in output: + num_compilations += 1 + + assert num_compilations == 1 + assert num_up_to_date == 2 + assert num_waiting == 2 + + assert is_installed_correctly(project_name, project_dir, mixed) + + +def test_import_multiple_projects(tmp_path: Path) -> None: + """This test ensures that the import hook can be used to load multiple projects + in the same run. + + A single pair of projects is chosen for this test because it should not make + any difference which projects are imported + """ + _clear_build_cache() + uninstall("pyo3-mixed") + uninstall("pyo3-pure") + + mixed_dir = create_project_from_blank_template( + "pyo3-mixed", tmp_path / "pyo3-mixed", mixed=True + ) + pure_dir = create_project_from_blank_template( + "pyo3-pure", tmp_path / "pyo3-pure", mixed=False + ) + + install_editable(mixed_dir) + assert is_installed_correctly("pyo3-mixed", mixed_dir, True) + install_editable(pure_dir) + assert is_installed_correctly("pyo3-pure", pure_dir, False) + + shutil.rmtree(mixed_dir) + shutil.rmtree(pure_dir) + get_project_copy(test_crates / "pyo3-mixed", mixed_dir) + get_project_copy(test_crates / "pyo3-pure", pure_dir) + + check_installed = "{}\n\n{}\n\n{}".format( + IMPORT_HOOK_HEADER, + (mixed_dir / "check_installed/check_installed.py").read_text(), + (pure_dir / "check_installed/check_installed.py").read_text(), + ) + + output1, duration1 = run_python_code(check_installed) + assert "SUCCESS" in output1 + assert _rebuilt_message("pyo3-mixed") in output1 + assert _rebuilt_message("pyo3-pure") in output1 + assert _up_to_date_message("pyo3-mixed") not in output1 + assert _up_to_date_message("pyo3-pure") not in output1 + + output2, duration2 = run_python_code(check_installed) + assert "SUCCESS" in output2 + assert _rebuilt_message("pyo3-mixed") not in output2 + assert _rebuilt_message("pyo3-pure") not in output2 + assert _up_to_date_message("pyo3-mixed") in output2 + assert _up_to_date_message("pyo3-pure") in output2 + + assert duration2 < duration1 + + assert is_installed_correctly("pyo3-mixed", mixed_dir, True) + assert is_installed_correctly("pyo3-pure", pure_dir, False) + + +def test_rebuild_on_change_to_path_dependency(tmp_path: Path) -> None: + """This test ensures that the imported project is rebuilt if any of its path + dependencies are edited. + """ + _clear_build_cache() + project_name = "pyo3-mixed-with-path-dep" + uninstall(project_name) + + project_dir = get_project_copy(test_crates / project_name, tmp_path / project_name) + get_project_copy(test_crates / "some_path_dep", tmp_path / "some_path_dep") + transitive_dep_dir = get_project_copy( + test_crates / "transitive_path_dep", tmp_path / "transitive_path_dep" + ) + + install_editable(project_dir) + assert is_installed_correctly(project_name, project_dir, True) + + check_installed = f""" +{IMPORT_HOOK_HEADER} + +import pyo3_mixed_with_path_dep + +assert pyo3_mixed_with_path_dep.get_42() == 42, 'get_42 did not return 42' + +print('21 is half 42:', pyo3_mixed_with_path_dep.is_half(21, 42)) +print('21 is half 63:', pyo3_mixed_with_path_dep.is_half(21, 63)) +""" + + output1, duration1 = run_python_code(check_installed) + assert "21 is half 42: True" in output1 + assert "21 is half 63: False" in output1 + + transitive_dep_lib = transitive_dep_dir / "src/lib.rs" + transitive_dep_lib.write_text( + transitive_dep_lib.read_text().replace("x + y == sum", "x + x + y == sum") + ) + + output2, duration2 = run_python_code(check_installed) + assert "21 is half 42: False" in output2 + assert "21 is half 63: True" in output2 + + assert is_installed_correctly(project_name, project_dir, True) + + +@pytest.mark.parametrize("is_mixed", [False, True]) +def test_rebuild_on_settings_change(tmp_path: Path, is_mixed: bool) -> None: + """When the source code has not changed but the import hook uses different maturin flags + the project is rebuilt. + """ + _clear_build_cache() + uninstall("my-script") + + project_dir = create_project_from_blank_template( + "my-script", tmp_path / "my-script", mixed=is_mixed + ) + shutil.copy( + script_dir / "rust_file_import/my_script_3.rs", project_dir / "src/lib.rs" + ) + manifest_path = project_dir / "Cargo.toml" + manifest_path.write_text( + f"{manifest_path.read_text()}\n[features]\nlarge_number = []\n" + ) + + install_editable(project_dir) + assert is_installed_correctly("my-script", project_dir, is_mixed) + + helper_path = script_dir / "rust_file_import/rebuild_on_settings_change_helper.py" + + output1, _ = run_python([str(helper_path)], cwd=tmp_path) + assert "get_num = 10" in output1 + assert "SUCCESS" in output1 + assert ( + 'package "my_script" will be rebuilt because: no build status found' in output1 + ) + + output2, _ = run_python([str(helper_path)], cwd=tmp_path) + assert "get_num = 10" in output2 + assert "SUCCESS" in output2 + assert 'package up to date: "my_script"' in output2 + + output3, _ = run_python([str(helper_path), "LARGE_NUMBER"], cwd=tmp_path) + assert "building my_script with large_number feature enabled" in output3 + assert ( + 'package "my_script" will be rebuilt because: ' + "current maturin args do not match the previous build" + ) in output3 + assert "get_num = 100" in output3 + assert "SUCCESS" in output3 + + output4, _ = run_python([str(helper_path), "LARGE_NUMBER"], cwd=tmp_path) + assert "building my_script with large_number feature enabled" in output4 + assert 'package up to date: "my_script"' in output4 + assert "get_num = 100" in output4 + assert "SUCCESS" in output4 + + +class TestLogging: + """These tests ensure that the desired messages are visible to the user in the default logging configuration.""" + + loader_script = """\ +import sys +from maturin import import_hook + +if len(sys.argv) > 1 and sys.argv[1] == 'RESET_LOGGER': + import_hook.reset_logger() + +import_hook.install() + +try: + import test_project +except ImportError as e: + # catch instead of printing the traceback since that may depend on the interpreter + print(f'caught ImportError: {e}') +else: + print("value", test_project.value) + print("SUCCESS") +""" + + @staticmethod + def _create_clean_project(tmp_dir: Path, is_mixed: bool) -> Path: + _clear_build_cache() + uninstall("test-project") + project_dir = create_project_from_blank_template( + "test-project", tmp_dir / "test-project", mixed=is_mixed + ) + install_editable(project_dir) + assert is_installed_correctly("test-project", project_dir, is_mixed) + + lib_path = project_dir / "src/lib.rs" + lib_src = ( + lib_path.read_text() + .replace("_m:", "m:") + .replace("Ok(())", 'm.add("value", 10)?;Ok(())') + ) + lib_path.write_text(lib_src) + + return project_dir + + @pytest.mark.parametrize("is_mixed", [False, True]) + def test_default_rebuild(self, tmp_path: Path, is_mixed: bool) -> None: + """By default, when a module is out of date the import hook logs messages + before and after rebuilding but hides the underlying details. + """ + self._create_clean_project(tmp_path, is_mixed) + + output, _ = run_python_code(self.loader_script) + pattern = ( + 'building "test_project"\n' + 'rebuilt and loaded package "test_project" in [0-9.]+s\n' + "value 10\n" + "SUCCESS\n" + ) + assert re.fullmatch(pattern, output, flags=re.MULTILINE) is not None + + @pytest.mark.parametrize("is_mixed", [False, True]) + def test_default_up_to_date(self, tmp_path: Path, is_mixed: bool) -> None: + """By default, when the module is up-to-date nothing is printed.""" + self._create_clean_project(tmp_path / "project", is_mixed) + + run_python_code(self.loader_script) # run once to rebuild + + output, _ = run_python_code(self.loader_script) + assert output == "value 10\nSUCCESS\n" + + @pytest.mark.parametrize("is_mixed", [False, True]) + def test_default_compile_error(self, tmp_path: Path, is_mixed: bool) -> None: + """If compilation fails then the error message from maturin is printed and an ImportError is raised.""" + project_dir = self._create_clean_project(tmp_path / "project", is_mixed) + + lib_path = project_dir / "src/lib.rs" + lib_path.write_text(lib_path.read_text().replace("Ok(())", "")) + + output, _ = run_python_code(self.loader_script) + pattern = ( + 'building "test_project"\n' + 'maturin\\.import_hook \\[ERROR\\] command ".*" returned non-zero exit status: 1\n' + "maturin\\.import_hook \\[ERROR\\] maturin output:\n" + ".*" + "expected `Result<\\(\\), PyErr>`, found `\\(\\)`" + ".*" + "maturin failed" + ".*" + "caught ImportError: Failed to build package with maturin\n" + ) + assert re.fullmatch(pattern, output, flags=re.MULTILINE | re.DOTALL) is not None + + @pytest.mark.parametrize("is_mixed", [False, True]) + def test_default_compile_warning(self, tmp_path: Path, is_mixed: bool) -> None: + """If compilation succeeds with warnings then the output of maturin is printed. + If the module is already up to date but warnings were raised when it was first + built, the warnings will be printed again. + """ + project_dir = self._create_clean_project(tmp_path / "project", is_mixed) + lib_path = project_dir / "src/lib.rs" + lib_path.write_text(lib_path.read_text().replace("Ok(())", "let x = 12;Ok(())")) + + output1, _ = run_python_code(self.loader_script) + pattern = ( + 'building "test_project"\n' + 'maturin.import_hook \\[WARNING\\] build of "test_project" succeeded with warnings:\n' + ".*" + "warning: unused variable: `x`" + ".*" + 'rebuilt and loaded package "test_project" in [0-9.]+s\n' + "value 10\n" + "SUCCESS\n" + ) + assert ( + re.fullmatch(pattern, output1, flags=re.MULTILINE | re.DOTALL) is not None + ) + + output2, _ = run_python_code(self.loader_script) + pattern = ( + 'maturin.import_hook \\[WARNING\\] the last build of "test_project" succeeded with warnings:\n' + ".*" + "warning: unused variable: `x`" + ".*" + "value 10\n" + "SUCCESS\n" + ) + assert ( + re.fullmatch(pattern, output2, flags=re.MULTILINE | re.DOTALL) is not None + ) + + @pytest.mark.parametrize("is_mixed", [False, True]) + def test_reset_logger_without_configuring( + self, tmp_path: Path, is_mixed: bool + ) -> None: + """If reset_logger is called then by default logging level INFO is not printed + (because the messages are handled by the root logger). + """ + self._create_clean_project(tmp_path / "project", is_mixed) + output, _ = run_python_code(self.loader_script, args=["RESET_LOGGER"]) + assert output == "value 10\nSUCCESS\n" + + @pytest.mark.parametrize("is_mixed", [False, True]) + def test_successful_compilation_but_not_valid( + self, tmp_path: Path, is_mixed: bool + ) -> None: + """If the project compiles but does not import correctly an ImportError is raised.""" + project_dir = self._create_clean_project(tmp_path / "project", is_mixed) + lib_path = project_dir / "src/lib.rs" + lib_path.write_text( + lib_path.read_text().replace("test_project", "test_project_new_name") + ) + + output, _ = run_python_code(self.loader_script, quiet=True) + pattern = ( + 'building "test_project"\n' + 'rebuilt and loaded package "test_project" in [0-9.]+s\n' + "caught ImportError: dynamic module does not define module export function \\(PyInit_test_project\\)\n" + ) + assert re.fullmatch(pattern, output, flags=re.MULTILINE) is not None + + +def _up_to_date_message(project_name: str) -> str: + return f'package up to date: "{with_underscores(project_name)}"' + + +def _rebuilt_message(project_name: str) -> str: + return f'rebuilt and loaded package "{with_underscores(project_name)}"' diff --git a/tests/import_hook/test_rust_file_importer.py b/tests/import_hook/test_rust_file_importer.py new file mode 100644 index 000000000..30f57a4a2 --- /dev/null +++ b/tests/import_hook/test_rust_file_importer.py @@ -0,0 +1,340 @@ +import multiprocessing +import os +import re +import shutil +from pathlib import Path +from typing import Tuple + +from .common import log, run_python, script_dir, test_crates + +""" +These tests ensure the correct functioning of the rust file importer import hook. +The tests are intended to be run as part of the tests in `run.rs` +which provides a clean virtual environment for these tests to use. +""" + +MATURIN_BUILD_CACHE = test_crates / "targets/import_hook_file_importer_build_cache" + +os.environ["CARGO_TARGET_DIR"] = str(test_crates / "targets/import_hook_file_importer") +os.environ["MATURIN_BUILD_DIR"] = str(MATURIN_BUILD_CACHE) + + +def _clear_build_cache() -> None: + if MATURIN_BUILD_CACHE.exists(): + log("clearing build cache") + shutil.rmtree(MATURIN_BUILD_CACHE) + + +def test_absolute_import(tmp_path: Path) -> None: + """test imports of the form `import ab.cd.ef`""" + _clear_build_cache() + + helper_path = script_dir / "rust_file_import/absolute_import_helper.py" + + output1, duration1 = run_python([str(helper_path)], cwd=tmp_path) + assert "SUCCESS" in output1 + assert "module up to date" not in output1 + assert "creating project for" in output1 + + output2, duration2 = run_python([str(helper_path)], cwd=tmp_path) + assert "SUCCESS" in output2 + assert "module up to date" in output2 + assert "creating project for" not in output2 + + assert duration2 < duration1 + + +def test_relative_import(tmp_path: Path) -> None: + """test imports of the form `from .ab import cd`""" + _clear_build_cache() + + output1, duration1 = run_python( + ["-m", "rust_file_import.relative_import_helper"], cwd=script_dir + ) + assert "SUCCESS" in output1 + assert "module up to date" not in output1 + assert "creating project for" in output1 + + output2, duration2 = run_python( + ["-m", "rust_file_import.relative_import_helper"], cwd=script_dir + ) + assert "SUCCESS" in output2 + assert "module up to date" in output2 + assert "creating project for" not in output2 + + assert duration2 < duration1 + + +def test_top_level_import(tmp_path: Path) -> None: + """test imports of the form `import ab`""" + _clear_build_cache() + + helper_path = script_dir / "rust_file_import/packages/top_level_import_helper.py" + + output1, duration1 = run_python([str(helper_path)], cwd=tmp_path) + assert "SUCCESS" in output1 + assert "module up to date" not in output1 + assert "creating project for" in output1 + + output2, duration2 = run_python([str(helper_path)], cwd=tmp_path) + assert "SUCCESS" in output2 + assert "module up to date" in output2 + assert "creating project for" not in output2 + + assert duration2 < duration1 + + +def test_multiple_imports(tmp_path: Path) -> None: + """test importing the same rs file multiple times by different names in the same session""" + _clear_build_cache() + + helper_path = script_dir / "rust_file_import/multiple_import_helper.py" + + output, _ = run_python([str(helper_path)], cwd=tmp_path) + assert "SUCCESS" in output + assert 'rebuilt and loaded module "packages.subpackage.my_rust_module"' in output + assert output.count("importing rust file") == 1 + + +def test_concurrent_import() -> None: + """test multiple processes attempting to import the same modules at the same time""" + _clear_build_cache() + args = { + "args": ["rust_file_import/concurrent_import_helper.py"], + "cwd": script_dir, + "quiet": True, + } + + with multiprocessing.Pool(processes=3) as pool: + p1 = pool.apply_async(run_python, kwds=args) + p2 = pool.apply_async(run_python, kwds=args) + p3 = pool.apply_async(run_python, kwds=args) + + output_1, duration_1 = p1.get() + output_2, duration_2 = p2.get() + output_3, duration_3 = p3.get() + + log("output 1") + log(output_1) + log("output 2") + log(output_2) + log("output 3") + log(output_3) + + num_compilations = 0 + num_up_to_date = 0 + num_waiting = 0 + for output in [output_1, output_2, output_3]: + assert "SUCCESS" in output + assert "importing rust file" in output + if "waiting on lock" in output: + num_waiting += 1 + if "creating project for" in output: + num_compilations += 1 + if "module up to date" in output: + num_up_to_date += 1 + + assert num_compilations == 1 + assert num_up_to_date == 2 + assert num_waiting == 2 + + +def test_rebuild_on_change(tmp_path: Path) -> None: + """test that modules are rebuilt if they are edited""" + _clear_build_cache() + + script_path = tmp_path / "my_script.rs" + helper_path = shutil.copy( + script_dir / "rust_file_import/rebuild_on_change_helper.py", tmp_path + ) + + shutil.copy(script_dir / "rust_file_import/my_script_1.rs", script_path) + + output1, _ = run_python([str(helper_path)], cwd=tmp_path) + assert "get_num = 10" in output1 + assert "failed to import get_other_num" in output1 + assert "SUCCESS" in output1 + + assert "module up to date" not in output1 + assert "creating project for" in output1 + + shutil.copy(script_dir / "rust_file_import/my_script_2.rs", script_path) + + output2, _ = run_python([str(helper_path)], cwd=tmp_path) + assert "get_num = 20" in output2 + assert "get_other_num = 100" in output2 + assert "SUCCESS" in output2 + + assert "module up to date" not in output2 + assert "creating project for" in output2 + + +def test_rebuild_on_settings_change(tmp_path: Path) -> None: + """test that modules are rebuilt if the settings (eg maturin flags) used by the import hook changes""" + _clear_build_cache() + + script_path = tmp_path / "my_script.rs" + helper_path = shutil.copy( + script_dir / "rust_file_import/rebuild_on_settings_change_helper.py", tmp_path + ) + + shutil.copy(script_dir / "rust_file_import/my_script_3.rs", script_path) + + output1, _ = run_python([str(helper_path)], cwd=tmp_path) + assert "get_num = 10" in output1 + assert "SUCCESS" in output1 + assert "building my_script with default settings" in output1 + assert "module up to date" not in output1 + assert "creating project for" in output1 + + output2, _ = run_python([str(helper_path)], cwd=tmp_path) + assert "get_num = 10" in output2 + assert "SUCCESS" in output2 + assert "module up to date" in output2 + + output3, _ = run_python([str(helper_path), "LARGE_NUMBER"], cwd=tmp_path) + assert "building my_script with large_number feature enabled" in output3 + assert "module up to date" not in output3 + assert "creating project for" in output3 + assert "get_num = 100" in output3 + assert "SUCCESS" in output3 + + output4, _ = run_python([str(helper_path), "LARGE_NUMBER"], cwd=tmp_path) + assert "building my_script with large_number feature enabled" in output4 + assert "module up to date" in output4 + assert "get_num = 100" in output4 + assert "SUCCESS" in output4 + + +class TestLogging: + """test the desired messages are visible to the user in the default logging configuration.""" + + loader_script = """\ +import sys +from maturin import import_hook + +if len(sys.argv) > 1 and sys.argv[1] == 'RESET_LOGGER': + import_hook.reset_logger() + +import_hook.install() + +try: + import my_script +except ImportError as e: + # catch instead of printing the traceback since that may depend on the interpreter + print(f'caught ImportError: {e}') +else: + print("get_num", my_script.get_num()) + print("SUCCESS") +""" + + def _create_clean_package(self, package_path: Path) -> Tuple[Path, Path]: + _clear_build_cache() + + package_path.mkdir() + original_script_path = script_dir / "rust_file_import/my_script_1.rs" + rs_path = Path(shutil.copy(original_script_path, package_path / "my_script.rs")) + py_path = package_path / "loader.py" + py_path.write_text(self.loader_script) + return rs_path, py_path + + def test_default_rebuild(self, tmp_path: Path) -> None: + """By default, when a module is out of date the import hook logs messages + before and after rebuilding but hides the underlying details. + """ + rs_path, py_path = self._create_clean_package(tmp_path / "package") + + output, _ = run_python([str(py_path)], tmp_path) + pattern = ( + 'building "my_script"\n' + 'rebuilt and loaded module "my_script" in [0-9.]+s\n' + "get_num 10\n" + "SUCCESS\n" + ) + assert re.fullmatch(pattern, output, flags=re.MULTILINE) is not None + + def test_default_up_to_date(self, tmp_path: Path) -> None: + """By default, when the module is up-to-date nothing is printed.""" + rs_path, py_path = self._create_clean_package(tmp_path / "package") + + run_python([str(py_path)], tmp_path) # run once to rebuild + + output, _ = run_python([str(py_path)], tmp_path) + assert output == "get_num 10\nSUCCESS\n" + + def test_default_compile_error(self, tmp_path: Path) -> None: + """If compilation fails then the error message from maturin is printed and an ImportError is raised.""" + rs_path, py_path = self._create_clean_package(tmp_path / "package") + + rs_path.write_text(rs_path.read_text().replace("10", "")) + output, _ = run_python([str(py_path)], tmp_path, quiet=True) + pattern = ( + 'building "my_script"\n' + 'maturin\\.import_hook \\[ERROR\\] command ".*" returned non-zero exit status: 1\n' + "maturin\\.import_hook \\[ERROR\\] maturin output:\n" + ".*" + "expected `usize`, found `\\(\\)`" + ".*" + "maturin failed" + ".*" + "caught ImportError: Failed to build wheel with maturin\n" + ) + assert re.fullmatch(pattern, output, flags=re.MULTILINE | re.DOTALL) is not None + + def test_default_compile_warning(self, tmp_path: Path) -> None: + """If compilation succeeds with warnings then the output of maturin is printed. + If the module is already up to date but warnings were raised when it was first + built, the warnings will be printed again. + """ + rs_path, py_path = self._create_clean_package(tmp_path / "package") + rs_path.write_text(rs_path.read_text().replace("10", "let x = 12; 20")) + + output1, _ = run_python([str(py_path)], tmp_path) + pattern = ( + 'building "my_script"\n' + 'maturin.import_hook \\[WARNING\\] build of "my_script" succeeded with warnings:\n' + ".*" + "warning: unused variable: `x`" + ".*" + 'rebuilt and loaded module "my_script" in [0-9.]+s\n' + "get_num 20\n" + "SUCCESS\n" + ) + assert ( + re.fullmatch(pattern, output1, flags=re.MULTILINE | re.DOTALL) is not None + ) + + output2, _ = run_python([str(py_path)], tmp_path) + pattern = ( + 'maturin.import_hook \\[WARNING\\] the last build of "my_script" succeeded with warnings:\n' + ".*" + "warning: unused variable: `x`" + ".*" + "get_num 20\n" + "SUCCESS\n" + ) + assert ( + re.fullmatch(pattern, output2, flags=re.MULTILINE | re.DOTALL) is not None + ) + + def test_reset_logger_without_configuring(self, tmp_path: Path) -> None: + """If reset_logger is called then by default logging level INFO is not printed + (because the messages are handled by the root logger). + """ + rs_path, py_path = self._create_clean_package(tmp_path / "package") + output, _ = run_python([str(py_path), "RESET_LOGGER"], tmp_path) + assert output == "get_num 10\nSUCCESS\n" + + def test_successful_compilation_but_not_valid(self, tmp_path: Path) -> None: + """If the script compiles but does not import correctly an ImportError is raised.""" + rs_path, py_path = self._create_clean_package(tmp_path / "package") + rs_path.write_text( + rs_path.read_text().replace("my_script", "my_script_new_name") + ) + output, _ = run_python([str(py_path)], tmp_path, quiet=True) + pattern = ( + 'building "my_script"\n' + 'rebuilt and loaded module "my_script" in [0-9.]+s\n' + "caught ImportError: dynamic module does not define module export function \\(PyInit_my_script\\)\n" + ) + assert re.fullmatch(pattern, output, flags=re.MULTILINE) is not None diff --git a/tests/import_hook/test_utilities.py b/tests/import_hook/test_utilities.py new file mode 100644 index 000000000..0cf803b76 --- /dev/null +++ b/tests/import_hook/test_utilities.py @@ -0,0 +1,334 @@ +import json +import os +import random +import shutil +import string +import threading +import time +from concurrent.futures import ProcessPoolExecutor +from operator import itemgetter +from pathlib import Path +from typing import Any, Dict, List, Optional + +import pytest + +from maturin.import_hook._building import BuildCache, BuildStatus, LockNotHeldError +from maturin.import_hook._file_lock import AtomicOpenLock, FileLock +from maturin.import_hook._resolve_project import ProjectResolveError, _resolve_project +from maturin.import_hook.project_importer import ( + _get_installed_package_mtime, + _get_project_mtime, +) + +from .common import log, test_crates + +# set this to be able to run these tests without going through run.rs each time +SAVED_RESOLVED_PACKAGES_PATH: Optional[Path] = None + +if SAVED_RESOLVED_PACKAGES_PATH is not None: + if "RESOLVED_PACKAGES_PATH" in os.environ: + shutil.copy(os.environ["RESOLVED_PACKAGES_PATH"], SAVED_RESOLVED_PACKAGES_PATH) + os.environ["RESOLVED_PACKAGES_PATH"] = str(SAVED_RESOLVED_PACKAGES_PATH) + + +class TestFileLock: + @staticmethod + def _create_lock(path: Path, timeout_seconds: float, fallback: bool) -> FileLock: + if fallback: + return AtomicOpenLock(path, timeout_seconds=timeout_seconds) + else: + return FileLock.new(path, timeout_seconds=timeout_seconds) + + @staticmethod + def _random_string(size: int = 1000) -> str: + return "".join(random.choice(string.ascii_lowercase) for _ in range(size)) + + @staticmethod + def _unlocked_worker(workspace: Path) -> str: + path = workspace / "my_file.txt" + data = TestFileLock._random_string() + for _ in range(10): + path.write_text(data) + time.sleep(0.001) + assert path.read_text() == data + return "SUCCESS" + + @staticmethod + def _locked_worker(workspace: Path, use_fallback_lock: bool) -> str: + path = workspace / "my_file.txt" + lock = TestFileLock._create_lock(workspace / "lock", 10, use_fallback_lock) + data = TestFileLock._random_string() + for _ in range(10): + with lock: + path.write_text(data) + time.sleep(0.001) + assert path.read_text() == data + return "SUCCESS" + + @pytest.mark.parametrize("use_fallback_lock", [False, True]) + def test_lock_unlock(self, tmp_path: Path, use_fallback_lock: bool) -> None: + lock = self._create_lock(tmp_path / "lock", 5, use_fallback_lock) + + assert not lock.is_locked + for _i in range(2): + with lock: + assert lock.is_locked + assert not lock.is_locked + + @pytest.mark.parametrize("use_fallback_lock", [False, True]) + def test_timeout(self, tmp_path: Path, use_fallback_lock: bool) -> None: + lock_path = tmp_path / "lock" + lock1 = self._create_lock(lock_path, 5, use_fallback_lock) + with lock1: + lock2 = self._create_lock(lock_path, 0.1, use_fallback_lock) + with pytest.raises(TimeoutError): + lock2.acquire() + + @pytest.mark.parametrize("use_fallback_lock", [False, True]) + def test_waiting(self, tmp_path: Path, use_fallback_lock: bool) -> None: + lock_path = tmp_path / "lock" + lock1 = self._create_lock(lock_path, 5, use_fallback_lock) + lock2 = self._create_lock(lock_path, 5, use_fallback_lock) + + lock1.acquire() + t = threading.Timer(0.2, lock1.release) + t.start() + lock2.acquire() + lock2.release() + + @pytest.mark.parametrize("use_fallback_lock", [False, True]) + def test_concurrent_access(self, tmp_path: Path, use_fallback_lock: bool) -> None: + num_workers = 25 + num_threads = 4 + + working_dir = tmp_path / "unlocked" + working_dir.mkdir() + with ProcessPoolExecutor(max_workers=num_threads) as executor: + futures = [ + executor.submit(TestFileLock._unlocked_worker, working_dir) + for _ in range(num_workers) + ] + with pytest.raises(AssertionError): + for f in futures: + f.result() + + working_dir = tmp_path / "locked" + working_dir.mkdir() + with ProcessPoolExecutor(max_workers=num_threads) as executor: + futures = [ + executor.submit( + TestFileLock._locked_worker, working_dir, use_fallback_lock + ) + for _ in range(num_workers) + ] + for f in futures: + assert f.result() == "SUCCESS" + + +class TestGetProjectMtime: + def test_missing_extension(self, tmp_path: Path) -> None: + assert _get_project_mtime(tmp_path, [], tmp_path / "missing", set()) is None + extension_dir = tmp_path / "extension" + extension_dir.mkdir() + assert _get_project_mtime(tmp_path, [], extension_dir, set()) is None + + def test_missing_path_dep(self, tmp_path: Path) -> None: + (tmp_path / "extension").touch() + project_mtime = _get_project_mtime( + tmp_path, [tmp_path / "missing"], tmp_path / "extension", set() + ) + assert project_mtime is None + + def test_simple(self, tmp_path: Path) -> None: + src_dir = tmp_path / "src" + src_dir.mkdir() + (src_dir / "source_file.rs").touch() + _small_sleep() + (tmp_path / "extension_module").touch() + project_mtime = _get_project_mtime( + tmp_path, [], tmp_path / "extension_module", set() + ) + assert project_mtime == (tmp_path / "extension_module").stat().st_mtime + + (tmp_path / "extension_module").unlink() + (tmp_path / "extension_module").mkdir() + (tmp_path / "extension_module/stuff").touch() + + # if the extension module is a directory then it should be excluded from the project mtime + # calculation as it may contain pycache files that are generated after installation + project_mtime = _get_project_mtime( + tmp_path, [], tmp_path / "extension_module", set() + ) + assert project_mtime == (src_dir / "source_file.rs").stat().st_mtime + + project_mtime = _get_project_mtime( + tmp_path, [], tmp_path / "extension_module", {"src"} + ) + assert project_mtime is None + + def test_simple_path_dep(self, tmp_path: Path) -> None: + project_a = tmp_path / "a" + project_b = tmp_path / "b" + project_a.mkdir() + project_b.mkdir() + + (project_a / "source").touch() + _small_sleep() + extension_module = project_a / "extension" + extension_module.touch() + _small_sleep() + (project_b / "source").touch() + + project_mtime = _get_project_mtime( + project_a, [project_b], extension_module, set() + ) + assert project_mtime == (project_b / "source").stat().st_mtime + + extension_module.touch() + project_mtime = _get_project_mtime( + project_a, [project_b], extension_module, set() + ) + assert project_mtime == (project_a / "extension").stat().st_mtime + + def test_extension_module_dir_with_some_newer(self, tmp_path: Path) -> None: + src_dir = tmp_path / "src" + extension_dir = tmp_path / "extension_module" + src_dir.mkdir() + extension_dir.mkdir() + + (extension_dir / "a").touch() + _small_sleep() + (src_dir / "source").touch() + _small_sleep() + (extension_dir / "b").touch() + + extension_mtime = _get_installed_package_mtime(extension_dir, set()) + assert extension_mtime == (extension_dir / "a").stat().st_mtime + project_mtime = _get_project_mtime(tmp_path, [], extension_dir, set()) + assert project_mtime == (src_dir / "source").stat().st_mtime + + _small_sleep() + (extension_dir / "a").touch() + extension_mtime = _get_installed_package_mtime(extension_dir, set()) + assert extension_mtime == (extension_dir / "b").stat().st_mtime + project_mtime = _get_project_mtime(tmp_path, [], extension_dir, set()) + assert project_mtime == (src_dir / "source").stat().st_mtime + + def test_extension_module_dir_with_newer_pycache(self, tmp_path: Path) -> None: + mixed_src_dir = tmp_path / "mixed_dir" + mixed_src_dir.mkdir() + + (mixed_src_dir / "__init__.py").touch() + _small_sleep() + extension_path = mixed_src_dir / "extension" + extension_path.touch() # project is built + _small_sleep() + (mixed_src_dir / "__pycache__").mkdir() # pycache is created later when loaded + (mixed_src_dir / "__pycache__/some_cache.pyc").touch() + + extension_mtime = _get_installed_package_mtime(extension_path, set()) + assert extension_mtime == extension_path.stat().st_mtime + project_mtime = _get_project_mtime(tmp_path, [], extension_path, set()) + assert ( + project_mtime + == (mixed_src_dir / "__pycache__/some_cache.pyc").stat().st_mtime + ) + + project_mtime = _get_project_mtime( + tmp_path, [], extension_path, {"__pycache__"} + ) + assert project_mtime == extension_path.stat().st_mtime + + def test_extension_outside_project_source(self, tmp_path: Path) -> None: + project_dir = tmp_path / "project" + installed_dir = tmp_path / "site-packages" + project_dir.mkdir() + installed_dir.mkdir() + + (project_dir / "source").touch() + _small_sleep() + extension_path = installed_dir / "extension" + extension_path.touch() + + project_mtime = _get_project_mtime(project_dir, [], extension_path, set()) + assert project_mtime == (project_dir / "source").stat().st_mtime + + _small_sleep() + (project_dir / "source").touch() + + project_mtime = _get_project_mtime(project_dir, [], extension_path, set()) + assert project_mtime == (project_dir / "source").stat().st_mtime + + +def _get_ground_truth_resolved_project_names() -> List[str]: + # passed in by the test runner + resolved_packages_path = Path(os.environ["RESOLVED_PACKAGES_PATH"]) + resolved_data = json.loads(resolved_packages_path.read_text()) + return sorted(resolved_data.keys(), key=itemgetter(0)) + + +def _get_ground_truth_resolved_project(project_name: str) -> Dict[str, Any]: + # passed in by the test runner + resolved_packages_path = Path(os.environ["RESOLVED_PACKAGES_PATH"]) + resolved_data = json.loads(resolved_packages_path.read_text()) + return resolved_data[project_name] + + +@pytest.mark.parametrize("project_name", _get_ground_truth_resolved_project_names()) +def test_resolve_project(project_name: str) -> None: + ground_truth = _get_ground_truth_resolved_project(project_name) + + log("ground truth:") + log(json.dumps(ground_truth, indent=2, sort_keys=True)) + + try: + resolved = _resolve_project(test_crates / project_name) + except ProjectResolveError: + calculated = None + else: + calculated = { + "cargo_manifest_path": _optional_path_to_str(resolved.cargo_manifest_path), + "python_dir": _optional_path_to_str(resolved.python_dir), + "python_module": _optional_path_to_str(resolved.python_module), + "extension_module_dir": _optional_path_to_str( + resolved.extension_module_dir + ), + "module_full_name": resolved.module_full_name, + } + log("calculated:") + log(json.dumps(calculated, indent=2, sort_keys=True)) + + assert ground_truth == calculated + + +def test_build_cache(tmp_path: Path) -> None: + cache = BuildCache(tmp_path / "build", lock_timeout_seconds=1) + with pytest.raises(LockNotHeldError): + cache.tmp_project_dir(tmp_path / "a", "a") + + with pytest.raises(LockNotHeldError): + cache.store_build_status(BuildStatus(1, tmp_path / "source", [], "")) + + with pytest.raises(LockNotHeldError): + cache.get_build_status(tmp_path / "source") + + with cache.lock: + dir_1 = cache.tmp_project_dir(tmp_path / "my_module", "my_module") + dir_2 = cache.tmp_project_dir(tmp_path / "other_place", "my_module") + assert dir_1 != dir_2 + + status1 = BuildStatus(1.2, tmp_path / "source1", [], "") + status2 = BuildStatus(1.2, tmp_path / "source2", [], "") + cache.store_build_status(status1) + cache.store_build_status(status2) + assert cache.get_build_status(tmp_path / "source1") == status1 + assert cache.get_build_status(tmp_path / "source2") == status2 + assert cache.get_build_status(tmp_path / "source3") is None + + +def _optional_path_to_str(path: Optional[Path]) -> Optional[str]: + return str(path) if path is not None else None + + +def _small_sleep() -> None: + time.sleep(0.05) diff --git a/tests/run.rs b/tests/run.rs index 3c0a2e4d4..0215230f1 100644 --- a/tests/run.rs +++ b/tests/run.rs @@ -1,13 +1,15 @@ -//! To speed up the tests, they are tests all collected in a single module +//! To speed up the tests, they are all collected in a single module +use crate::common::import_hook; use common::{ develop, errors, get_python_implementation, handle_result, integration, other, test_python_path, }; use indoc::indoc; use maturin::pyproject_toml::SdistGenerator; use maturin::Target; -use std::env; +use std::collections::BTreeMap; use std::path::{Path, PathBuf}; +use std::{env, fs}; use time::macros::datetime; use which::which; @@ -763,3 +765,46 @@ fn pyo3_source_date_epoch() { "pyo3_source_date_epoch", )) } + +#[test] +fn import_hook_project_importer() { + handle_result(import_hook::test_import_hook( + "import_hook_project_importer", + &PathBuf::from("tests/import_hook/test_project_importer.py"), + vec!["boltons"], + BTreeMap::new(), + true, + )); +} + +#[test] +fn import_hook_rust_file_importer() { + handle_result(import_hook::test_import_hook( + "import_hook_rust_file_importer", + &PathBuf::from("tests/import_hook/test_rust_file_importer.py"), + vec![], + BTreeMap::new(), + true, + )); +} + +#[test] +fn import_hook_utilities() { + let tmpdir = tempfile::tempdir().unwrap(); + let resolved_packages_path = tmpdir.path().join("resolved.json"); + fs::write( + &resolved_packages_path, + import_hook::resolve_all_packages().unwrap(), + ) + .unwrap(); + handle_result(import_hook::test_import_hook( + "import_hook_utilities", + &PathBuf::from("tests/import_hook/test_utilities.py"), + vec![], + BTreeMap::from([( + "RESOLVED_PACKAGES_PATH", + resolved_packages_path.to_str().unwrap(), + )]), + true, + )); +} From 4ce765dcf02580d0f57bb9c6af983f619d7b4e2c Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 27 Aug 2023 21:12:47 +0100 Subject: [PATCH 07/57] added to changelog --- Changelog.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Changelog.md b/Changelog.md index f2cd2c9e2..213195541 100644 --- a/Changelog.md +++ b/Changelog.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +* Add new import hook with support for many more use cases [#1748](https://github.com/PyO3/maturin/pull/1748) + ## [1.2.3] - 2023-08-17 * Fix sdist build failure with workspace path dependencies by HerringtonDarkholme in [#1739](https://github.com/PyO3/maturin/pull/1739) From ca62147e94bf22dfdc7da14b8f0d53f79d40079b Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 27 Aug 2023 22:01:17 +0100 Subject: [PATCH 08/57] test improvements to fix CI --- tests/common/import_hook.rs | 18 ++++++++++++++++++ tests/import_hook/common.py | 24 ++++++++++++++++-------- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/tests/common/import_hook.rs b/tests/common/import_hook.rs index 1eaf0cea0..280925c2a 100644 --- a/tests/common/import_hook.rs +++ b/tests/common/import_hook.rs @@ -22,6 +22,24 @@ pub fn test_import_hook( let (venv_dir, python) = create_virtualenv(virtualenv_name, Some(python)).unwrap(); + println!("installing maturin binary into virtualenv"); + let status = Command::new("cargo") + .args([ + "build", + "--target-dir", + venv_dir.join("maturin").as_os_str().to_str().unwrap(), + ]) + .status() + .unwrap(); + if !status.success() { + bail!("failed to install maturin"); + } + fs::copy( + venv_dir.join("maturin/debug/maturin"), + venv_dir.join("bin/maturin"), + ) + .unwrap(); + let pytest_args = vec![ vec!["pytest"], vec!["uniffi-bindgen"], diff --git a/tests/import_hook/common.py b/tests/import_hook/common.py index 8b540ec46..28a9b8be1 100644 --- a/tests/import_hook/common.py +++ b/tests/import_hook/common.py @@ -6,7 +6,7 @@ import tempfile import time from pathlib import Path -from typing import List, Optional, Tuple +from typing import List, Optional, Tuple, Iterable from maturin.import_hook.project_importer import _fix_direct_url, _load_dist_info @@ -198,14 +198,22 @@ def is_installed_correctly( def get_project_copy(project_dir: Path, output_path: Path) -> Path: - # using shutil.copy instead of the default shutil.copy2 because we want mtimes to be updated on copy - project_copy_dir = Path( - shutil.copytree(project_dir, output_path, copy_function=shutil.copy) + for relative_path in _get_relative_files_tracked_by_git(project_dir): + output_file_path = output_path / relative_path + output_file_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy(project_dir / relative_path, output_file_path) + return output_path + + +def _get_relative_files_tracked_by_git(root: Path) -> Iterable[Path]: + """this is used to ignore built artifacts to create a clean copy""" + output = subprocess.check_output( + ["git", "ls-tree", "--name-only", "-z", "-r", "HEAD"], cwd=root ) - assert ( - next(project_copy_dir.rglob("*.so"), None) is None - ), f"project {project_dir.name} is not clean" - return project_copy_dir + for relative_path_bytes in output.split(b"\x00"): + relative_path = Path(os.fsdecode(relative_path_bytes)) + if (root / relative_path).is_file(): + yield relative_path def create_project_from_blank_template( From 1a7005f183cd6af5bd97c07888ae1f10dcbb1e4a Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Mon, 28 Aug 2023 15:12:04 +0100 Subject: [PATCH 09/57] small changes --- guide/src/develop.md | 32 ++++++++++++++------ maturin/import_hook/__init__.py | 6 ++-- maturin/import_hook/_building.py | 4 +-- maturin/import_hook/settings.py | 8 +++++ tests/import_hook/test_project_importer.py | 6 +++- tests/import_hook/test_rust_file_importer.py | 6 +++- 6 files changed, 46 insertions(+), 16 deletions(-) diff --git a/guide/src/develop.md b/guide/src/develop.md index fe977d232..4fc930717 100644 --- a/guide/src/develop.md +++ b/guide/src/develop.md @@ -145,18 +145,17 @@ The maturin project importer and the rust file importer can be used separately ```python from maturin.import_hook import rust_file_importer rust_file_importer.install() -from maturin.import_hook import package_importer -package_importer.install() +from maturin.import_hook import project_importer +project_importer.install() ``` The import hook can be configured to control its behaviour ```python -from pathlib import Path from maturin import import_hook -from maturin.import_hook.settings import MaturinSettings, MaturinSettingsProvider +from maturin.import_hook.settings import MaturinSettings import_hook.install( - enable_package_importer=True, + enable_project_importer=True, enable_rs_file_importer=True, settings=MaturinSettings( release=True, @@ -166,9 +165,15 @@ import_hook.install( show_warnings=True, # ... ) +``` + +Custom settings providers can be used to override settings of particular projects +or implement custom logic such as loading settings from configuration files +```python +from pathlib import Path +from maturin import import_hook +from maturin.import_hook.settings import MaturinSettings, MaturinSettingsProvider -# custom settings providers can be used to override settings of particular projects -# or implement custom logic such as loading settings from configuration files class CustomSettings(MaturinSettingsProvider): def get_settings(self, module_path: str, source_path: Path) -> MaturinSettings: return MaturinSettings( @@ -178,7 +183,7 @@ class CustomSettings(MaturinSettingsProvider): ) import_hook.install( - enable_package_importer=True, + enable_project_importer=True, enable_rs_file_importer=True, settings=CustomSettings(), show_warnings=True, @@ -186,9 +191,18 @@ import_hook.install( ) ``` +Since the import hook is intended for use in development environments and not for +production environments, it may be a good idea to put the call to `import_hook.install()` +into `site-packages/sitecustomize.py` of your development virtual environment +([documentation](https://docs.python.org/3/library/site.html)). This will +enable the hook for every script run by that interpreter without calling `import_hook.install()` +in every script, meaning the scripts do not need alteration before deployment. + + The import hook internals can be examined by configuring the root logger and calling `reset_logger` to propagate messages from the `maturin.import_hook` logger -to the root logger. +to the root logger. You can also run with the environment variable `RUST_LOG=maturin=debug` +to get more information from maturin. ```python import logging logging.basicConfig(format='%(name)s [%(levelname)s] %(message)s', level=logging.DEBUG) diff --git a/maturin/import_hook/__init__.py b/maturin/import_hook/__init__.py index 5f55b9100..ac97b484e 100644 --- a/maturin/import_hook/__init__.py +++ b/maturin/import_hook/__init__.py @@ -10,7 +10,7 @@ def install( *, - enable_package_importer: bool = True, + enable_project_importer: bool = True, enable_rs_file_importer: bool = True, settings: Optional[Union[MaturinSettings, MaturinSettingsProvider]] = None, build_dir: Optional[Path] = None, @@ -22,7 +22,7 @@ def install( ) -> None: """Install import hooks for automatically rebuilding and importing maturin projects or .rs files. - :param enable_package_importer: enable the hook for automatically rebuilding editable installed maturin projects + :param enable_project_importer: enable the hook for automatically rebuilding editable installed maturin projects :param enable_rs_file_importer: enable the hook for importing .rs files as though they were regular python modules @@ -55,7 +55,7 @@ def install( lock_timeout_seconds=lock_timeout_seconds, show_warnings=show_warnings, ) - if enable_package_importer: + if enable_project_importer: project_importer.install( settings=settings, build_dir=build_dir, diff --git a/maturin/import_hook/_building.py b/maturin/import_hook/_building.py index 0d5e9791e..fee488650 100644 --- a/maturin/import_hook/_building.py +++ b/maturin/import_hook/_building.py @@ -84,11 +84,11 @@ def get_build_status(self, source_path: Path) -> Optional[BuildStatus]: except FileNotFoundError: return None - def tmp_project_dir(self, project_path: Path, module_path: str) -> Path: + def tmp_project_dir(self, project_path: Path, module_name: str) -> Path: if not self._lock.is_locked: raise LockNotHeldError path_hash = hashlib.sha1(bytes(project_path)).hexdigest() - return self._build_dir / "project" / f"{module_path}_{path_hash}" + return self._build_dir / "project" / f"{module_name}_{path_hash}" def _get_default_build_dir() -> Path: diff --git a/maturin/import_hook/settings.py b/maturin/import_hook/settings.py index d94b031e0..a667164d3 100644 --- a/maturin/import_hook/settings.py +++ b/maturin/import_hook/settings.py @@ -18,6 +18,12 @@ class MaturinSettings: frozen: bool = False locked: bool = False offline: bool = False + verbose: int = 0 + + def __post_init__(self) -> None: + if self.verbose not in (0, 1, 2): + msg = f"invalid verbose value: {self.verbose}" + raise ValueError(msg) def to_args(self) -> List[str]: args = [] @@ -43,6 +49,8 @@ def to_args(self) -> List[str]: args.append("--locked") if self.offline: args.append("--offline") + if self.verbose > 0: + args.append("-{}".format("v" * self.verbose)) return args diff --git a/tests/import_hook/test_project_importer.py b/tests/import_hook/test_project_importer.py index 91e3d88cd..face0fd44 100644 --- a/tests/import_hook/test_project_importer.py +++ b/tests/import_hook/test_project_importer.py @@ -658,7 +658,11 @@ def test_default_compile_warning(self, tmp_path: Path, is_mixed: bool) -> None: """ project_dir = self._create_clean_project(tmp_path / "project", is_mixed) lib_path = project_dir / "src/lib.rs" - lib_path.write_text(lib_path.read_text().replace("Ok(())", "let x = 12;Ok(())")) + lib_path.write_text( + lib_path.read_text().replace( + "Ok(())", "#[warn(unused_variables)]{let x = 12;}; Ok(())" + ) + ) output1, _ = run_python_code(self.loader_script) pattern = ( diff --git a/tests/import_hook/test_rust_file_importer.py b/tests/import_hook/test_rust_file_importer.py index 30f57a4a2..718b9d3fd 100644 --- a/tests/import_hook/test_rust_file_importer.py +++ b/tests/import_hook/test_rust_file_importer.py @@ -287,7 +287,11 @@ def test_default_compile_warning(self, tmp_path: Path) -> None: built, the warnings will be printed again. """ rs_path, py_path = self._create_clean_package(tmp_path / "package") - rs_path.write_text(rs_path.read_text().replace("10", "let x = 12; 20")) + rs_path.write_text( + rs_path.read_text().replace( + "10", "#[warn(unused_variables)]{let x = 12;}; 20" + ) + ) output1, _ = run_python([str(py_path)], tmp_path) pattern = ( From 6b0eccf11bc03d78b007bfaaf0e2f2fecc03fa2a Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Mon, 28 Aug 2023 15:12:21 +0100 Subject: [PATCH 10/57] clean up test temporary directories by default --- tests/import_hook/test_project_importer.py | 104 +++++++++++-------- tests/import_hook/test_rust_file_importer.py | 99 ++++++++++-------- tests/import_hook/test_utilities.py | 10 +- 3 files changed, 121 insertions(+), 92 deletions(-) diff --git a/tests/import_hook/test_project_importer.py b/tests/import_hook/test_project_importer.py index face0fd44..cc77b1b4c 100644 --- a/tests/import_hook/test_project_importer.py +++ b/tests/import_hook/test_project_importer.py @@ -3,6 +3,7 @@ import re import shutil from pathlib import Path +from typing import Generator import pytest @@ -31,6 +32,9 @@ """ MATURIN_BUILD_CACHE = test_crates / "targets/import_hook_project_importer_build_cache" +# the CI does not have enough space to keep the outputs. +# When running locally you may set this to False for debugging +CLEAR_WORKSPACE = True os.environ["CARGO_TARGET_DIR"] = str( test_crates / "targets/import_hook_project_importer" @@ -38,6 +42,16 @@ os.environ["MATURIN_BUILD_DIR"] = str(MATURIN_BUILD_CACHE) +@pytest.fixture() +def workspace(tmp_path: Path) -> Generator[Path, None, None]: + try: + yield tmp_path + finally: + if CLEAR_WORKSPACE: + log(f"clearing workspace {tmp_path}") + shutil.rmtree(tmp_path, ignore_errors=True) + + def _clear_build_cache() -> None: if MATURIN_BUILD_CACHE.exists(): log("clearing build cache") @@ -49,7 +63,7 @@ def _clear_build_cache() -> None: # path dependencies tested separately sorted(set(all_test_crate_names()) - {"pyo3-mixed-with-path-dep"}), ) -def test_install_from_script_inside(tmp_path: Path, project_name: str) -> None: +def test_install_from_script_inside(workspace: Path, project_name: str) -> None: """This test ensures that when a script is run from within a maturin project, the import hook can identify and install the containing project even if it is not already installed. @@ -62,7 +76,7 @@ def test_install_from_script_inside(tmp_path: Path, project_name: str) -> None: _clear_build_cache() uninstall(project_name) - project_dir = get_project_copy(test_crates / project_name, tmp_path / project_name) + project_dir = get_project_copy(test_crates / project_name, workspace / project_name) check_installed_dir = project_dir / "check_installed" check_installed_path = check_installed_dir / "check_installed.py" @@ -70,7 +84,7 @@ def test_install_from_script_inside(tmp_path: Path, project_name: str) -> None: f"{IMPORT_HOOK_HEADER}\n\n{check_installed_path.read_text()}" ) - empty_dir = tmp_path / "empty" + empty_dir = workspace / "empty" empty_dir.mkdir() output1, duration1 = run_python([str(check_installed_path)], cwd=empty_dir) @@ -91,14 +105,14 @@ def test_install_from_script_inside(tmp_path: Path, project_name: str) -> None: @pytest.mark.parametrize("project_name", ["pyo3-mixed", "pyo3-pure"]) -def test_do_not_install_from_script_inside(tmp_path: Path, project_name: str) -> None: +def test_do_not_install_from_script_inside(workspace: Path, project_name: str) -> None: """This test ensures that when the import hook works correctly when it is configured to not rebuild/install projects if they aren't already installed. """ _clear_build_cache() uninstall(project_name) - project_dir = get_project_copy(test_crates / project_name, tmp_path / project_name) + project_dir = get_project_copy(test_crates / project_name, workspace / project_name) check_installed_path = project_dir / "check_installed/check_installed.py" header = """ @@ -112,7 +126,7 @@ def test_do_not_install_from_script_inside(tmp_path: Path, project_name: str) -> """ check_installed_path.write_text(f"{header}\n\n{check_installed_path.read_text()}") - empty_dir = tmp_path / "empty" + empty_dir = workspace / "empty" empty_dir.mkdir() output1, _ = run_python( @@ -143,18 +157,18 @@ def test_do_not_install_from_script_inside(tmp_path: Path, project_name: str) -> @pytest.mark.parametrize("project_name", ["pyo3-mixed", "pyo3-pure"]) def test_do_not_rebuild_if_installed_non_editable( - tmp_path: Path, project_name: str + workspace: Path, project_name: str ) -> None: """This test ensures that if a maturin project is installed in non-editable mode then the import hook will not rebuild it or re-install it in editable mode. """ _clear_build_cache() uninstall(project_name) - project_dir = get_project_copy(test_crates / project_name, tmp_path / project_name) + project_dir = get_project_copy(test_crates / project_name, workspace / project_name) install_non_editable(project_dir) - workspace = tmp_path / "workspace" - workspace.mkdir() + check_installed_outside_project = workspace / "check_installed" + check_installed_outside_project.mkdir() check_installed_dir = project_dir / "check_installed" check_installed_path = check_installed_dir / "check_installed.py" @@ -169,12 +183,12 @@ def test_do_not_rebuild_if_installed_non_editable( import_hook.install(install_new_packages=install_new_packages) """ check_installed_path.write_text(f"{header}\n\n{check_installed_path.read_text()}") - shutil.copy(check_installed_path, workspace) + shutil.copy(check_installed_path, check_installed_outside_project) (project_dir / "src/lib.rs").write_text("") # will break once rebuilt # when outside the project, can still detect non-editable installed projects via dist-info - output1, _ = run_python(["check_installed.py"], cwd=workspace) + output1, _ = run_python(["check_installed.py"], cwd=check_installed_outside_project) assert "SUCCESS" in output1 assert "install_new_packages=False" in output1 assert f'found project linked by dist-info: "{project_dir}"' in output1 @@ -195,7 +209,7 @@ def test_do_not_rebuild_if_installed_non_editable( output3, _ = run_python( ["check_installed.py", "INSTALL_NEW"], - cwd=workspace, + cwd=check_installed_outside_project, quiet=True, expect_error=True, ) @@ -214,7 +228,7 @@ def test_do_not_rebuild_if_installed_non_editable( sorted(set(all_test_crate_names()) - {"pyo3-mixed-with-path-dep"}), ) def test_import_editable_installed_rebuild( - tmp_path: Path, project_name: str, initially_mixed: bool + workspace: Path, project_name: str, initially_mixed: bool ) -> None: """This test ensures that an editable installed project is rebuilt when necessary if the import hook is active. This applies to mixed projects (which are installed as .pth files into @@ -232,7 +246,7 @@ def test_import_editable_installed_rebuild( ).read_text() project_dir = create_project_from_blank_template( - project_name, tmp_path / project_name, mixed=initially_mixed + project_name, workspace / project_name, mixed=initially_mixed ) log(f"installing blank project as {project_name}") @@ -277,7 +291,7 @@ def test_import_editable_installed_rebuild( sorted(set(mixed_test_crate_names()) - {"pyo3-mixed-with-path-dep"}), ) def test_import_editable_installed_mixed_missing( - tmp_path: Path, project_name: str + workspace: Path, project_name: str ) -> None: """This test ensures that editable installed mixed projects are rebuilt if they are imported and their artifacts are missing. @@ -291,9 +305,9 @@ def test_import_editable_installed_mixed_missing( uninstall(project_name) # making a copy because editable installation may write files into the project directory - project_dir = get_project_copy(test_crates / project_name, tmp_path / project_name) + project_dir = get_project_copy(test_crates / project_name, workspace / project_name) project_backup_dir = get_project_copy( - test_crates / project_name, tmp_path / f"backup_{project_name}" + test_crates / project_name, workspace / f"backup_{project_name}" ) install_editable(project_dir) @@ -331,7 +345,7 @@ def test_import_editable_installed_mixed_missing( @pytest.mark.parametrize("mixed", [False, True]) @pytest.mark.parametrize("initially_mixed", [False, True]) -def test_concurrent_import(tmp_path: Path, initially_mixed: bool, mixed: bool) -> None: +def test_concurrent_import(workspace: Path, initially_mixed: bool, mixed: bool) -> None: """This test ensures that if multiple scripts attempt to use the import hook concurrently, that the project still installs correctly and does not crash. @@ -359,7 +373,7 @@ def test_concurrent_import(tmp_path: Path, initially_mixed: bool, mixed: bool) - check_installed_with_hook = f"{IMPORT_HOOK_HEADER}\n\n{check_installed}" project_dir = create_project_from_blank_template( - project_name, tmp_path / project_name, mixed=initially_mixed + project_name, workspace / project_name, mixed=initially_mixed ) log(f"initially mixed: {initially_mixed}, mixed: {mixed}") @@ -410,7 +424,7 @@ def test_concurrent_import(tmp_path: Path, initially_mixed: bool, mixed: bool) - assert is_installed_correctly(project_name, project_dir, mixed) -def test_import_multiple_projects(tmp_path: Path) -> None: +def test_import_multiple_projects(workspace: Path) -> None: """This test ensures that the import hook can be used to load multiple projects in the same run. @@ -422,10 +436,10 @@ def test_import_multiple_projects(tmp_path: Path) -> None: uninstall("pyo3-pure") mixed_dir = create_project_from_blank_template( - "pyo3-mixed", tmp_path / "pyo3-mixed", mixed=True + "pyo3-mixed", workspace / "pyo3-mixed", mixed=True ) pure_dir = create_project_from_blank_template( - "pyo3-pure", tmp_path / "pyo3-pure", mixed=False + "pyo3-pure", workspace / "pyo3-pure", mixed=False ) install_editable(mixed_dir) @@ -464,7 +478,7 @@ def test_import_multiple_projects(tmp_path: Path) -> None: assert is_installed_correctly("pyo3-pure", pure_dir, False) -def test_rebuild_on_change_to_path_dependency(tmp_path: Path) -> None: +def test_rebuild_on_change_to_path_dependency(workspace: Path) -> None: """This test ensures that the imported project is rebuilt if any of its path dependencies are edited. """ @@ -472,10 +486,10 @@ def test_rebuild_on_change_to_path_dependency(tmp_path: Path) -> None: project_name = "pyo3-mixed-with-path-dep" uninstall(project_name) - project_dir = get_project_copy(test_crates / project_name, tmp_path / project_name) - get_project_copy(test_crates / "some_path_dep", tmp_path / "some_path_dep") + project_dir = get_project_copy(test_crates / project_name, workspace / project_name) + get_project_copy(test_crates / "some_path_dep", workspace / "some_path_dep") transitive_dep_dir = get_project_copy( - test_crates / "transitive_path_dep", tmp_path / "transitive_path_dep" + test_crates / "transitive_path_dep", workspace / "transitive_path_dep" ) install_editable(project_dir) @@ -509,7 +523,7 @@ def test_rebuild_on_change_to_path_dependency(tmp_path: Path) -> None: @pytest.mark.parametrize("is_mixed", [False, True]) -def test_rebuild_on_settings_change(tmp_path: Path, is_mixed: bool) -> None: +def test_rebuild_on_settings_change(workspace: Path, is_mixed: bool) -> None: """When the source code has not changed but the import hook uses different maturin flags the project is rebuilt. """ @@ -517,7 +531,7 @@ def test_rebuild_on_settings_change(tmp_path: Path, is_mixed: bool) -> None: uninstall("my-script") project_dir = create_project_from_blank_template( - "my-script", tmp_path / "my-script", mixed=is_mixed + "my-script", workspace / "my-script", mixed=is_mixed ) shutil.copy( script_dir / "rust_file_import/my_script_3.rs", project_dir / "src/lib.rs" @@ -532,19 +546,19 @@ def test_rebuild_on_settings_change(tmp_path: Path, is_mixed: bool) -> None: helper_path = script_dir / "rust_file_import/rebuild_on_settings_change_helper.py" - output1, _ = run_python([str(helper_path)], cwd=tmp_path) + output1, _ = run_python([str(helper_path)], cwd=workspace) assert "get_num = 10" in output1 assert "SUCCESS" in output1 assert ( 'package "my_script" will be rebuilt because: no build status found' in output1 ) - output2, _ = run_python([str(helper_path)], cwd=tmp_path) + output2, _ = run_python([str(helper_path)], cwd=workspace) assert "get_num = 10" in output2 assert "SUCCESS" in output2 assert 'package up to date: "my_script"' in output2 - output3, _ = run_python([str(helper_path), "LARGE_NUMBER"], cwd=tmp_path) + output3, _ = run_python([str(helper_path), "LARGE_NUMBER"], cwd=workspace) assert "building my_script with large_number feature enabled" in output3 assert ( 'package "my_script" will be rebuilt because: ' @@ -553,7 +567,7 @@ def test_rebuild_on_settings_change(tmp_path: Path, is_mixed: bool) -> None: assert "get_num = 100" in output3 assert "SUCCESS" in output3 - output4, _ = run_python([str(helper_path), "LARGE_NUMBER"], cwd=tmp_path) + output4, _ = run_python([str(helper_path), "LARGE_NUMBER"], cwd=workspace) assert "building my_script with large_number feature enabled" in output4 assert 'package up to date: "my_script"' in output4 assert "get_num = 100" in output4 @@ -603,11 +617,11 @@ def _create_clean_project(tmp_dir: Path, is_mixed: bool) -> Path: return project_dir @pytest.mark.parametrize("is_mixed", [False, True]) - def test_default_rebuild(self, tmp_path: Path, is_mixed: bool) -> None: + def test_default_rebuild(self, workspace: Path, is_mixed: bool) -> None: """By default, when a module is out of date the import hook logs messages before and after rebuilding but hides the underlying details. """ - self._create_clean_project(tmp_path, is_mixed) + self._create_clean_project(workspace, is_mixed) output, _ = run_python_code(self.loader_script) pattern = ( @@ -619,9 +633,9 @@ def test_default_rebuild(self, tmp_path: Path, is_mixed: bool) -> None: assert re.fullmatch(pattern, output, flags=re.MULTILINE) is not None @pytest.mark.parametrize("is_mixed", [False, True]) - def test_default_up_to_date(self, tmp_path: Path, is_mixed: bool) -> None: + def test_default_up_to_date(self, workspace: Path, is_mixed: bool) -> None: """By default, when the module is up-to-date nothing is printed.""" - self._create_clean_project(tmp_path / "project", is_mixed) + self._create_clean_project(workspace / "project", is_mixed) run_python_code(self.loader_script) # run once to rebuild @@ -629,9 +643,9 @@ def test_default_up_to_date(self, tmp_path: Path, is_mixed: bool) -> None: assert output == "value 10\nSUCCESS\n" @pytest.mark.parametrize("is_mixed", [False, True]) - def test_default_compile_error(self, tmp_path: Path, is_mixed: bool) -> None: + def test_default_compile_error(self, workspace: Path, is_mixed: bool) -> None: """If compilation fails then the error message from maturin is printed and an ImportError is raised.""" - project_dir = self._create_clean_project(tmp_path / "project", is_mixed) + project_dir = self._create_clean_project(workspace / "project", is_mixed) lib_path = project_dir / "src/lib.rs" lib_path.write_text(lib_path.read_text().replace("Ok(())", "")) @@ -651,12 +665,12 @@ def test_default_compile_error(self, tmp_path: Path, is_mixed: bool) -> None: assert re.fullmatch(pattern, output, flags=re.MULTILINE | re.DOTALL) is not None @pytest.mark.parametrize("is_mixed", [False, True]) - def test_default_compile_warning(self, tmp_path: Path, is_mixed: bool) -> None: + def test_default_compile_warning(self, workspace: Path, is_mixed: bool) -> None: """If compilation succeeds with warnings then the output of maturin is printed. If the module is already up to date but warnings were raised when it was first built, the warnings will be printed again. """ - project_dir = self._create_clean_project(tmp_path / "project", is_mixed) + project_dir = self._create_clean_project(workspace / "project", is_mixed) lib_path = project_dir / "src/lib.rs" lib_path.write_text( lib_path.read_text().replace( @@ -694,21 +708,21 @@ def test_default_compile_warning(self, tmp_path: Path, is_mixed: bool) -> None: @pytest.mark.parametrize("is_mixed", [False, True]) def test_reset_logger_without_configuring( - self, tmp_path: Path, is_mixed: bool + self, workspace: Path, is_mixed: bool ) -> None: """If reset_logger is called then by default logging level INFO is not printed (because the messages are handled by the root logger). """ - self._create_clean_project(tmp_path / "project", is_mixed) + self._create_clean_project(workspace / "project", is_mixed) output, _ = run_python_code(self.loader_script, args=["RESET_LOGGER"]) assert output == "value 10\nSUCCESS\n" @pytest.mark.parametrize("is_mixed", [False, True]) def test_successful_compilation_but_not_valid( - self, tmp_path: Path, is_mixed: bool + self, workspace: Path, is_mixed: bool ) -> None: """If the project compiles but does not import correctly an ImportError is raised.""" - project_dir = self._create_clean_project(tmp_path / "project", is_mixed) + project_dir = self._create_clean_project(workspace / "project", is_mixed) lib_path = project_dir / "src/lib.rs" lib_path.write_text( lib_path.read_text().replace("test_project", "test_project_new_name") diff --git a/tests/import_hook/test_rust_file_importer.py b/tests/import_hook/test_rust_file_importer.py index 718b9d3fd..960ebcc0c 100644 --- a/tests/import_hook/test_rust_file_importer.py +++ b/tests/import_hook/test_rust_file_importer.py @@ -3,7 +3,9 @@ import re import shutil from pathlib import Path -from typing import Tuple +from typing import Tuple, Generator + +import pytest from .common import log, run_python, script_dir, test_crates @@ -14,6 +16,9 @@ """ MATURIN_BUILD_CACHE = test_crates / "targets/import_hook_file_importer_build_cache" +# the CI does not have enough space to keep the outputs. +# When running locally you may set this to False for debugging +CLEAR_WORKSPACE = True os.environ["CARGO_TARGET_DIR"] = str(test_crates / "targets/import_hook_file_importer") os.environ["MATURIN_BUILD_DIR"] = str(MATURIN_BUILD_CACHE) @@ -25,18 +30,28 @@ def _clear_build_cache() -> None: shutil.rmtree(MATURIN_BUILD_CACHE) -def test_absolute_import(tmp_path: Path) -> None: +@pytest.fixture() +def workspace(tmp_path: Path) -> Generator[Path, None, None]: + try: + yield tmp_path + finally: + if CLEAR_WORKSPACE: + log(f"clearing workspace {tmp_path}") + shutil.rmtree(tmp_path, ignore_errors=True) + + +def test_absolute_import(workspace: Path) -> None: """test imports of the form `import ab.cd.ef`""" _clear_build_cache() helper_path = script_dir / "rust_file_import/absolute_import_helper.py" - output1, duration1 = run_python([str(helper_path)], cwd=tmp_path) + output1, duration1 = run_python([str(helper_path)], cwd=workspace) assert "SUCCESS" in output1 assert "module up to date" not in output1 assert "creating project for" in output1 - output2, duration2 = run_python([str(helper_path)], cwd=tmp_path) + output2, duration2 = run_python([str(helper_path)], cwd=workspace) assert "SUCCESS" in output2 assert "module up to date" in output2 assert "creating project for" not in output2 @@ -44,7 +59,7 @@ def test_absolute_import(tmp_path: Path) -> None: assert duration2 < duration1 -def test_relative_import(tmp_path: Path) -> None: +def test_relative_import() -> None: """test imports of the form `from .ab import cd`""" _clear_build_cache() @@ -65,18 +80,18 @@ def test_relative_import(tmp_path: Path) -> None: assert duration2 < duration1 -def test_top_level_import(tmp_path: Path) -> None: +def test_top_level_import(workspace: Path) -> None: """test imports of the form `import ab`""" _clear_build_cache() helper_path = script_dir / "rust_file_import/packages/top_level_import_helper.py" - output1, duration1 = run_python([str(helper_path)], cwd=tmp_path) + output1, duration1 = run_python([str(helper_path)], cwd=workspace) assert "SUCCESS" in output1 assert "module up to date" not in output1 assert "creating project for" in output1 - output2, duration2 = run_python([str(helper_path)], cwd=tmp_path) + output2, duration2 = run_python([str(helper_path)], cwd=workspace) assert "SUCCESS" in output2 assert "module up to date" in output2 assert "creating project for" not in output2 @@ -84,13 +99,13 @@ def test_top_level_import(tmp_path: Path) -> None: assert duration2 < duration1 -def test_multiple_imports(tmp_path: Path) -> None: +def test_multiple_imports(workspace: Path) -> None: """test importing the same rs file multiple times by different names in the same session""" _clear_build_cache() helper_path = script_dir / "rust_file_import/multiple_import_helper.py" - output, _ = run_python([str(helper_path)], cwd=tmp_path) + output, _ = run_python([str(helper_path)], cwd=workspace) assert "SUCCESS" in output assert 'rebuilt and loaded module "packages.subpackage.my_rust_module"' in output assert output.count("importing rust file") == 1 @@ -139,18 +154,18 @@ def test_concurrent_import() -> None: assert num_waiting == 2 -def test_rebuild_on_change(tmp_path: Path) -> None: +def test_rebuild_on_change(workspace: Path) -> None: """test that modules are rebuilt if they are edited""" _clear_build_cache() - script_path = tmp_path / "my_script.rs" + script_path = workspace / "my_script.rs" helper_path = shutil.copy( - script_dir / "rust_file_import/rebuild_on_change_helper.py", tmp_path + script_dir / "rust_file_import/rebuild_on_change_helper.py", workspace ) shutil.copy(script_dir / "rust_file_import/my_script_1.rs", script_path) - output1, _ = run_python([str(helper_path)], cwd=tmp_path) + output1, _ = run_python([str(helper_path)], cwd=workspace) assert "get_num = 10" in output1 assert "failed to import get_other_num" in output1 assert "SUCCESS" in output1 @@ -160,7 +175,7 @@ def test_rebuild_on_change(tmp_path: Path) -> None: shutil.copy(script_dir / "rust_file_import/my_script_2.rs", script_path) - output2, _ = run_python([str(helper_path)], cwd=tmp_path) + output2, _ = run_python([str(helper_path)], cwd=workspace) assert "get_num = 20" in output2 assert "get_other_num = 100" in output2 assert "SUCCESS" in output2 @@ -169,37 +184,37 @@ def test_rebuild_on_change(tmp_path: Path) -> None: assert "creating project for" in output2 -def test_rebuild_on_settings_change(tmp_path: Path) -> None: +def test_rebuild_on_settings_change(workspace: Path) -> None: """test that modules are rebuilt if the settings (eg maturin flags) used by the import hook changes""" _clear_build_cache() - script_path = tmp_path / "my_script.rs" + script_path = workspace / "my_script.rs" helper_path = shutil.copy( - script_dir / "rust_file_import/rebuild_on_settings_change_helper.py", tmp_path + script_dir / "rust_file_import/rebuild_on_settings_change_helper.py", workspace ) shutil.copy(script_dir / "rust_file_import/my_script_3.rs", script_path) - output1, _ = run_python([str(helper_path)], cwd=tmp_path) + output1, _ = run_python([str(helper_path)], cwd=workspace) assert "get_num = 10" in output1 assert "SUCCESS" in output1 assert "building my_script with default settings" in output1 assert "module up to date" not in output1 assert "creating project for" in output1 - output2, _ = run_python([str(helper_path)], cwd=tmp_path) + output2, _ = run_python([str(helper_path)], cwd=workspace) assert "get_num = 10" in output2 assert "SUCCESS" in output2 assert "module up to date" in output2 - output3, _ = run_python([str(helper_path), "LARGE_NUMBER"], cwd=tmp_path) + output3, _ = run_python([str(helper_path), "LARGE_NUMBER"], cwd=workspace) assert "building my_script with large_number feature enabled" in output3 assert "module up to date" not in output3 assert "creating project for" in output3 assert "get_num = 100" in output3 assert "SUCCESS" in output3 - output4, _ = run_python([str(helper_path), "LARGE_NUMBER"], cwd=tmp_path) + output4, _ = run_python([str(helper_path), "LARGE_NUMBER"], cwd=workspace) assert "building my_script with large_number feature enabled" in output4 assert "module up to date" in output4 assert "get_num = 100" in output4 @@ -238,13 +253,13 @@ def _create_clean_package(self, package_path: Path) -> Tuple[Path, Path]: py_path.write_text(self.loader_script) return rs_path, py_path - def test_default_rebuild(self, tmp_path: Path) -> None: + def test_default_rebuild(self, workspace: Path) -> None: """By default, when a module is out of date the import hook logs messages before and after rebuilding but hides the underlying details. """ - rs_path, py_path = self._create_clean_package(tmp_path / "package") + rs_path, py_path = self._create_clean_package(workspace / "package") - output, _ = run_python([str(py_path)], tmp_path) + output, _ = run_python([str(py_path)], workspace) pattern = ( 'building "my_script"\n' 'rebuilt and loaded module "my_script" in [0-9.]+s\n' @@ -253,21 +268,21 @@ def test_default_rebuild(self, tmp_path: Path) -> None: ) assert re.fullmatch(pattern, output, flags=re.MULTILINE) is not None - def test_default_up_to_date(self, tmp_path: Path) -> None: + def test_default_up_to_date(self, workspace: Path) -> None: """By default, when the module is up-to-date nothing is printed.""" - rs_path, py_path = self._create_clean_package(tmp_path / "package") + rs_path, py_path = self._create_clean_package(workspace / "package") - run_python([str(py_path)], tmp_path) # run once to rebuild + run_python([str(py_path)], workspace) # run once to rebuild - output, _ = run_python([str(py_path)], tmp_path) + output, _ = run_python([str(py_path)], workspace) assert output == "get_num 10\nSUCCESS\n" - def test_default_compile_error(self, tmp_path: Path) -> None: + def test_default_compile_error(self, workspace: Path) -> None: """If compilation fails then the error message from maturin is printed and an ImportError is raised.""" - rs_path, py_path = self._create_clean_package(tmp_path / "package") + rs_path, py_path = self._create_clean_package(workspace / "package") rs_path.write_text(rs_path.read_text().replace("10", "")) - output, _ = run_python([str(py_path)], tmp_path, quiet=True) + output, _ = run_python([str(py_path)], workspace, quiet=True) pattern = ( 'building "my_script"\n' 'maturin\\.import_hook \\[ERROR\\] command ".*" returned non-zero exit status: 1\n' @@ -281,19 +296,19 @@ def test_default_compile_error(self, tmp_path: Path) -> None: ) assert re.fullmatch(pattern, output, flags=re.MULTILINE | re.DOTALL) is not None - def test_default_compile_warning(self, tmp_path: Path) -> None: + def test_default_compile_warning(self, workspace: Path) -> None: """If compilation succeeds with warnings then the output of maturin is printed. If the module is already up to date but warnings were raised when it was first built, the warnings will be printed again. """ - rs_path, py_path = self._create_clean_package(tmp_path / "package") + rs_path, py_path = self._create_clean_package(workspace / "package") rs_path.write_text( rs_path.read_text().replace( "10", "#[warn(unused_variables)]{let x = 12;}; 20" ) ) - output1, _ = run_python([str(py_path)], tmp_path) + output1, _ = run_python([str(py_path)], workspace) pattern = ( 'building "my_script"\n' 'maturin.import_hook \\[WARNING\\] build of "my_script" succeeded with warnings:\n' @@ -308,7 +323,7 @@ def test_default_compile_warning(self, tmp_path: Path) -> None: re.fullmatch(pattern, output1, flags=re.MULTILINE | re.DOTALL) is not None ) - output2, _ = run_python([str(py_path)], tmp_path) + output2, _ = run_python([str(py_path)], workspace) pattern = ( 'maturin.import_hook \\[WARNING\\] the last build of "my_script" succeeded with warnings:\n' ".*" @@ -321,21 +336,21 @@ def test_default_compile_warning(self, tmp_path: Path) -> None: re.fullmatch(pattern, output2, flags=re.MULTILINE | re.DOTALL) is not None ) - def test_reset_logger_without_configuring(self, tmp_path: Path) -> None: + def test_reset_logger_without_configuring(self, workspace: Path) -> None: """If reset_logger is called then by default logging level INFO is not printed (because the messages are handled by the root logger). """ - rs_path, py_path = self._create_clean_package(tmp_path / "package") - output, _ = run_python([str(py_path), "RESET_LOGGER"], tmp_path) + rs_path, py_path = self._create_clean_package(workspace / "package") + output, _ = run_python([str(py_path), "RESET_LOGGER"], workspace) assert output == "get_num 10\nSUCCESS\n" - def test_successful_compilation_but_not_valid(self, tmp_path: Path) -> None: + def test_successful_compilation_but_not_valid(self, workspace: Path) -> None: """If the script compiles but does not import correctly an ImportError is raised.""" - rs_path, py_path = self._create_clean_package(tmp_path / "package") + rs_path, py_path = self._create_clean_package(workspace / "package") rs_path.write_text( rs_path.read_text().replace("my_script", "my_script_new_name") ) - output, _ = run_python([str(py_path)], tmp_path, quiet=True) + output, _ = run_python([str(py_path)], workspace, quiet=True) pattern = ( 'building "my_script"\n' 'rebuilt and loaded module "my_script" in [0-9.]+s\n' diff --git a/tests/import_hook/test_utilities.py b/tests/import_hook/test_utilities.py index 0cf803b76..bae61cf04 100644 --- a/tests/import_hook/test_utilities.py +++ b/tests/import_hook/test_utilities.py @@ -44,8 +44,8 @@ def _random_string(size: int = 1000) -> str: return "".join(random.choice(string.ascii_lowercase) for _ in range(size)) @staticmethod - def _unlocked_worker(workspace: Path) -> str: - path = workspace / "my_file.txt" + def _unlocked_worker(work_dir: Path) -> str: + path = work_dir / "my_file.txt" data = TestFileLock._random_string() for _ in range(10): path.write_text(data) @@ -54,9 +54,9 @@ def _unlocked_worker(workspace: Path) -> str: return "SUCCESS" @staticmethod - def _locked_worker(workspace: Path, use_fallback_lock: bool) -> str: - path = workspace / "my_file.txt" - lock = TestFileLock._create_lock(workspace / "lock", 10, use_fallback_lock) + def _locked_worker(work_dir: Path, use_fallback_lock: bool) -> str: + path = work_dir / "my_file.txt" + lock = TestFileLock._create_lock(work_dir / "lock", 10, use_fallback_lock) data = TestFileLock._random_string() for _ in range(10): with lock: From 1be23703643cd611f4ec6f5314dcf9e9f2741055 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Mon, 28 Aug 2023 18:23:05 +0100 Subject: [PATCH 11/57] support more maturin flags and properly support parsing output with color --- maturin/import_hook/_building.py | 17 +++- maturin/import_hook/project_importer.py | 2 +- maturin/import_hook/rust_file_importer.py | 2 +- maturin/import_hook/settings.py | 99 +++++++++++++++++-- tests/import_hook/common.py | 9 ++ .../rebuild_on_settings_change_helper.py | 2 +- tests/import_hook/test_project_importer.py | 3 + tests/import_hook/test_rust_file_importer.py | 10 +- tests/import_hook/test_utilities.py | 88 ++++++++++++++++- 9 files changed, 218 insertions(+), 14 deletions(-) diff --git a/maturin/import_hook/_building.py b/maturin/import_hook/_building.py index fee488650..66c943d48 100644 --- a/maturin/import_hook/_building.py +++ b/maturin/import_hook/_building.py @@ -162,6 +162,9 @@ def build_wheel( output_dir: Path, settings: MaturinSettings, ) -> str: + if "build" not in settings.supported_commands(): + msg = f'provided {type(settings).__name__} does not support the "build" command' + raise ImportError(msg) success, output = _run_maturin( [ "build", @@ -186,6 +189,11 @@ def develop_build_project( args = ["develop", "--manifest-path", str(manifest_path)] if skip_install: args.append("--skip-install") + if "develop" not in settings.supported_commands(): + msg = ( + f'provided {type(settings).__name__} does not support the "develop" command' + ) + raise ImportError(msg) args.extend(settings.to_args()) success, output = _run_maturin(args) if not success: @@ -211,7 +219,11 @@ def _run_maturin(args: list[str]) -> Tuple[bool, str]: logger.error("maturin output:\n%s", output) return False, output if logger.isEnabledFor(logging.DEBUG): - logger.debug("maturin output:\n%s", output) + logger.debug( + "maturin output (has warnings: %r):\n%s", + maturin_output_has_warnings(output), + output, + ) return True, output @@ -242,6 +254,5 @@ def _find_single_file(dir_path: Path, extension: Optional[str]) -> Optional[Path def maturin_output_has_warnings(output: str) -> bool: return ( - re.search(r"warning: `.*` \((lib|bin)\) generated [0-9]+ warnings?", output) - is not None + re.search(r"`.*` \((lib|bin)\) generated [0-9]+ warnings?", output) is not None ) diff --git a/maturin/import_hook/project_importer.py b/maturin/import_hook/project_importer.py index 6dd3825f8..ab4d79f59 100644 --- a/maturin/import_hook/project_importer.py +++ b/maturin/import_hook/project_importer.py @@ -79,7 +79,7 @@ def _get_settings(self, module_path: str, source_path: Path) -> MaturinSettings: elif isinstance(self._settings, MaturinSettingsProvider): return self._settings.get_settings(module_path, source_path) else: - return MaturinSettings() + return MaturinSettings.default() def find_spec( self, diff --git a/maturin/import_hook/rust_file_importer.py b/maturin/import_hook/rust_file_importer.py index 8600f5102..1bccb5fee 100644 --- a/maturin/import_hook/rust_file_importer.py +++ b/maturin/import_hook/rust_file_importer.py @@ -49,7 +49,7 @@ def _get_settings(self, module_path: str, source_path: Path) -> MaturinSettings: elif isinstance(self._settings, MaturinSettingsProvider): return self._settings.get_settings(module_path, source_path) else: - return MaturinSettings() + return MaturinSettings.default() def find_spec( self, diff --git a/maturin/import_hook/settings.py b/maturin/import_hook/settings.py index a667164d3..67bef6c21 100644 --- a/maturin/import_hook/settings.py +++ b/maturin/import_hook/settings.py @@ -1,29 +1,49 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from pathlib import Path -from typing import List, Optional +from typing import List, Optional, Dict, Set -__all__ = ["MaturinSettings", "MaturinSettingsProvider"] +__all__ = [ + "MaturinSettings", + "MaturinBuildSettings", + "MaturinDevelopSettings", + "MaturinSettingsProvider", +] @dataclass class MaturinSettings: + """Settings common to `maturin build` and `maturin develop`""" + release: bool = False strip: bool = False quiet: bool = False jobs: Optional[int] = None + profile: Optional[str] = None features: Optional[List[str]] = None all_features: bool = False no_default_features: bool = False + target: Optional[str] = None + ignore_rust_version: bool = False + color: Optional[bool] = None frozen: bool = False locked: bool = False offline: bool = False + config: Optional[Dict[str, str]] = None + unstable_flags: Optional[List[str]] = None verbose: int = 0 + rustc_flags: Optional[List[str]] = None + + @staticmethod + def supported_commands() -> Set[str]: + return {"build", "develop"} - def __post_init__(self) -> None: - if self.verbose not in (0, 1, 2): - msg = f"invalid verbose value: {self.verbose}" - raise ValueError(msg) + @staticmethod + def default() -> "MaturinSettings": + """MaturinSettings() sets no flags but default() corresponds to some sensible defaults""" + return MaturinSettings( + color=True, + ) def to_args(self) -> List[str]: args = [] @@ -36,6 +56,9 @@ def to_args(self) -> List[str]: if self.jobs is not None: args.append("--jobs") args.append(str(self.jobs)) + if self.profile is not None: + args.append("--profile") + args.append(self.profile) if self.features: args.append("--features") args.append(",".join(self.features)) @@ -43,14 +66,78 @@ def to_args(self) -> List[str]: args.append("--all-features") if self.no_default_features: args.append("--no-default-features") + if self.target is not None: + args.append("--target") + args.append(self.target) + if self.ignore_rust_version: + args.append("--ignore-rust-version") + if self.color is not None: + args.append("--color") + if self.color: + args.append("always") + else: + args.append("never") if self.frozen: args.append("--frozen") if self.locked: args.append("--locked") if self.offline: args.append("--offline") + if self.config is not None: + for key, value in self.config.items(): + args.append("--config") + args.append(f"{key}={value}") + if self.unstable_flags is not None: + for flag in self.unstable_flags: + args.append("-Z") + args.append(flag) if self.verbose > 0: args.append("-{}".format("v" * self.verbose)) + if self.rustc_flags is not None: + args.extend(self.rustc_flags) + return args + + +@dataclass +class MaturinBuildSettings(MaturinSettings): + """settings for `maturin build`""" + + skip_auditwheel: bool = False + zig: bool = False + + @staticmethod + def supported_commands() -> Set[str]: + return {"build"} + + def to_args(self) -> List[str]: + args = [] + if self.skip_auditwheel: + args.append("--skip-auditwheel") + if self.zig: + args.append("--zig") + args.extend(super().to_args()) + return args + + +@dataclass +class MaturinDevelopSettings(MaturinSettings): + """settings for `maturin develop`""" + + extras: Optional[List[str]] = None + skip_install: bool = False + + @staticmethod + def supported_commands() -> Set[str]: + return {"develop"} + + def to_args(self) -> List[str]: + args = [] + if self.extras is not None: + args.append("--extras") + args.append(",".join(self.extras)) + if self.skip_install: + args.append("--skip-install") + args.extend(super().to_args()) return args diff --git a/tests/import_hook/common.py b/tests/import_hook/common.py index 28a9b8be1..cd9b8e385 100644 --- a/tests/import_hook/common.py +++ b/tests/import_hook/common.py @@ -1,4 +1,5 @@ import os +import re import shutil import site import subprocess @@ -238,3 +239,11 @@ def create_project_from_blank_template( f"from .{package_name} import *" ) return project_dir + + +def remove_ansii_escape_characters(text: str) -> str: + """Remove escape characters (eg used to color terminal output) from the given string + + based on: https://stackoverflow.com/a/14693789 + """ + return re.sub(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", text) diff --git a/tests/import_hook/rust_file_import/rebuild_on_settings_change_helper.py b/tests/import_hook/rust_file_import/rebuild_on_settings_change_helper.py index aa0b2a023..9b6aff61f 100644 --- a/tests/import_hook/rust_file_import/rebuild_on_settings_change_helper.py +++ b/tests/import_hook/rust_file_import/rebuild_on_settings_change_helper.py @@ -18,7 +18,7 @@ def get_settings(self, module_path: str, source_path: Path) -> MaturinSettings: return MaturinSettings(features=["pyo3/extension-module", "large_number"]) else: print(f"building {module_path} with default settings") - return MaturinSettings() + return MaturinSettings.default() import_hook.install(settings=CustomSettingsProvider()) diff --git a/tests/import_hook/test_project_importer.py b/tests/import_hook/test_project_importer.py index cc77b1b4c..773f8dafc 100644 --- a/tests/import_hook/test_project_importer.py +++ b/tests/import_hook/test_project_importer.py @@ -23,6 +23,7 @@ test_crates, uninstall, with_underscores, + remove_ansii_escape_characters, ) """ @@ -679,6 +680,7 @@ def test_default_compile_warning(self, workspace: Path, is_mixed: bool) -> None: ) output1, _ = run_python_code(self.loader_script) + output1 = remove_ansii_escape_characters(output1) pattern = ( 'building "test_project"\n' 'maturin.import_hook \\[WARNING\\] build of "test_project" succeeded with warnings:\n' @@ -694,6 +696,7 @@ def test_default_compile_warning(self, workspace: Path, is_mixed: bool) -> None: ) output2, _ = run_python_code(self.loader_script) + output2 = remove_ansii_escape_characters(output2) pattern = ( 'maturin.import_hook \\[WARNING\\] the last build of "test_project" succeeded with warnings:\n' ".*" diff --git a/tests/import_hook/test_rust_file_importer.py b/tests/import_hook/test_rust_file_importer.py index 960ebcc0c..e2259241c 100644 --- a/tests/import_hook/test_rust_file_importer.py +++ b/tests/import_hook/test_rust_file_importer.py @@ -7,7 +7,13 @@ import pytest -from .common import log, run_python, script_dir, test_crates +from .common import ( + log, + run_python, + script_dir, + test_crates, + remove_ansii_escape_characters, +) """ These tests ensure the correct functioning of the rust file importer import hook. @@ -309,6 +315,7 @@ def test_default_compile_warning(self, workspace: Path) -> None: ) output1, _ = run_python([str(py_path)], workspace) + output1 = remove_ansii_escape_characters(output1) pattern = ( 'building "my_script"\n' 'maturin.import_hook \\[WARNING\\] build of "my_script" succeeded with warnings:\n' @@ -324,6 +331,7 @@ def test_default_compile_warning(self, workspace: Path) -> None: ) output2, _ = run_python([str(py_path)], workspace) + output2 = remove_ansii_escape_characters(output2) pattern = ( 'maturin.import_hook \\[WARNING\\] the last build of "my_script" succeeded with warnings:\n' ".*" diff --git a/tests/import_hook/test_utilities.py b/tests/import_hook/test_utilities.py index bae61cf04..cbb89858b 100644 --- a/tests/import_hook/test_utilities.py +++ b/tests/import_hook/test_utilities.py @@ -12,13 +12,19 @@ import pytest -from maturin.import_hook._building import BuildCache, BuildStatus, LockNotHeldError +from maturin.import_hook import MaturinSettings +from maturin.import_hook._building import ( + BuildCache, + BuildStatus, + LockNotHeldError, +) from maturin.import_hook._file_lock import AtomicOpenLock, FileLock from maturin.import_hook._resolve_project import ProjectResolveError, _resolve_project from maturin.import_hook.project_importer import ( _get_installed_package_mtime, _get_project_mtime, ) +from maturin.import_hook.settings import MaturinDevelopSettings, MaturinBuildSettings from .common import log, test_crates @@ -31,6 +37,86 @@ os.environ["RESOLVED_PACKAGES_PATH"] = str(SAVED_RESOLVED_PACKAGES_PATH) +def test_settings() -> None: + assert MaturinSettings().to_args() == [] + assert MaturinBuildSettings().to_args() == [] + assert MaturinDevelopSettings().to_args() == [] + + settings = MaturinSettings( + release=True, + strip=True, + quiet=True, + jobs=1, + profile="profile1", + features=["feature1", "feature2"], + all_features=True, + no_default_features=True, + target="target1", + ignore_rust_version=True, + color=True, + frozen=True, + locked=True, + offline=True, + config={"key1": "value1", "key2": "value2"}, + unstable_flags=["unstable1", "unstable2"], + verbose=2, + rustc_flags=["flag1", "flag2"], + ) + # fmt: off + assert settings.to_args() == [ + '--release', + '--strip', + '--quiet', + '--jobs', '1', + '--profile', 'profile1', + '--features', 'feature1,feature2', + '--all-features', + '--no-default-features', + '--target', 'target1', + '--ignore-rust-version', + '--color', 'always', + '--frozen', + '--locked', + '--offline', + '--config', 'key1=value1', + '--config', 'key2=value2', + '-Z', 'unstable1', + '-Z', 'unstable2', + '-vv', + 'flag1', + 'flag2', + ] + # fmt: on + + build_settings = MaturinBuildSettings( + skip_auditwheel=True, zig=True, color=False, rustc_flags=["flag1", "flag2"] + ) + assert build_settings.to_args() == [ + "--skip-auditwheel", + "--zig", + "--color", + "never", + "flag1", + "flag2", + ] + + develop_settings = MaturinDevelopSettings( + extras=["extra1", "extra2"], + skip_install=True, + color=False, + rustc_flags=["flag1", "flag2"], + ) + assert develop_settings.to_args() == [ + "--extras", + "extra1,extra2", + "--skip-install", + "--color", + "never", + "flag1", + "flag2", + ] + + class TestFileLock: @staticmethod def _create_lock(path: Path, timeout_seconds: float, fallback: bool) -> FileLock: From f4059a0291c242fdfbad993c19c735f1913fb79e Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Mon, 28 Aug 2023 18:41:48 +0100 Subject: [PATCH 12/57] refactor build cache to make operations on an unlocked cache impossible --- maturin/import_hook/_building.py | 45 +++++++++---------- maturin/import_hook/project_importer.py | 10 +++-- maturin/import_hook/rust_file_importer.py | 16 +++---- maturin/import_hook/settings.py | 10 ++--- .../pyo3-mixed-with-path-dep/Cargo.toml | 1 - tests/import_hook/test_utilities.py | 35 +++++---------- 6 files changed, 50 insertions(+), 67 deletions(-) diff --git a/maturin/import_hook/_building.py b/maturin/import_hook/_building.py index 66c943d48..c37e54069 100644 --- a/maturin/import_hook/_building.py +++ b/maturin/import_hook/_building.py @@ -8,9 +8,10 @@ import subprocess import sys import zipfile +from contextlib import contextmanager from dataclasses import dataclass from pathlib import Path -from typing import List, Optional, Tuple +from typing import Generator, List, Optional, Tuple from ._file_lock import FileLock from ._logging import logger @@ -46,28 +47,11 @@ def from_json(json_data: dict) -> Optional["BuildStatus"]: return None -class LockNotHeldError(Exception): - pass - - -class BuildCache: - def __init__( - self, build_dir: Optional[Path], lock_timeout_seconds: Optional[float] - ) -> None: - self._build_dir = ( - build_dir if build_dir is not None else _get_default_build_dir() - ) - self._lock = FileLock.new( - self._build_dir / "lock", timeout_seconds=lock_timeout_seconds - ) - - @property - def lock(self) -> FileLock: - return self._lock +class LockedBuildCache: + def __init__(self, build_dir: Path) -> None: + self._build_dir = build_dir def _build_status_path(self, source_path: Path) -> Path: - if not self._lock.is_locked: - raise LockNotHeldError path_hash = hashlib.sha1(bytes(source_path)).hexdigest() build_status_dir = self._build_dir / "build_status" build_status_dir.mkdir(parents=True, exist_ok=True) @@ -85,12 +69,27 @@ def get_build_status(self, source_path: Path) -> Optional[BuildStatus]: return None def tmp_project_dir(self, project_path: Path, module_name: str) -> Path: - if not self._lock.is_locked: - raise LockNotHeldError path_hash = hashlib.sha1(bytes(project_path)).hexdigest() return self._build_dir / "project" / f"{module_name}_{path_hash}" +class BuildCache: + def __init__( + self, build_dir: Optional[Path], lock_timeout_seconds: Optional[float] + ) -> None: + self._build_dir = ( + build_dir if build_dir is not None else _get_default_build_dir() + ) + self._lock = FileLock.new( + self._build_dir / "lock", timeout_seconds=lock_timeout_seconds + ) + + @contextmanager + def lock(self) -> Generator[LockedBuildCache, None, None]: + with self._lock: + yield LockedBuildCache(self._build_dir) + + def _get_default_build_dir() -> Path: build_dir = os.environ.get("MATURIN_BUILD_DIR", None) if build_dir and os.access(sys.exec_prefix, os.W_OK): diff --git a/maturin/import_hook/project_importer.py b/maturin/import_hook/project_importer.py index ab4d79f59..cc567ee6a 100644 --- a/maturin/import_hook/project_importer.py +++ b/maturin/import_hook/project_importer.py @@ -16,6 +16,7 @@ from maturin.import_hook._building import ( BuildCache, BuildStatus, + LockedBuildCache, develop_build_project, maturin_output_has_warnings, ) @@ -170,10 +171,10 @@ def _rebuild_project( logger.debug('importing project "%s" as "%s"', project_dir, package_name) - with self._build_cache.lock: + with self._build_cache.lock() as build_cache: settings = self._get_settings(package_name, project_dir) spec, reason = self._get_spec_for_up_to_date_package( - package_name, project_dir, resolved, settings + package_name, project_dir, resolved, settings, build_cache ) if spec is not None: return spec, False @@ -214,7 +215,7 @@ def _rebuild_project( build_status = BuildStatus( mtime, project_dir, settings.to_args(), maturin_output ) - self._build_cache.store_build_status(build_status) + build_cache.store_build_status(build_status) return spec, True @@ -224,6 +225,7 @@ def _get_spec_for_up_to_date_package( project_dir: Path, resolved: MaturinProject, settings: MaturinSettings, + build_cache: LockedBuildCache, ) -> Tuple[Optional[ModuleSpec], Optional[str]]: """Return a spec for the given module at the given search_dir if it exists and is newer than the source code that it is derived from. @@ -256,7 +258,7 @@ def _get_spec_for_up_to_date_package( ): return None, "package is out of date" - build_status = self._build_cache.get_build_status(project_dir) + build_status = build_cache.get_build_status(project_dir) if build_status is None: return None, "no build status found" if build_status.source_path != project_dir: diff --git a/maturin/import_hook/rust_file_importer.py b/maturin/import_hook/rust_file_importer.py index 1bccb5fee..a9ca4c2fc 100644 --- a/maturin/import_hook/rust_file_importer.py +++ b/maturin/import_hook/rust_file_importer.py @@ -14,6 +14,7 @@ from maturin.import_hook._building import ( BuildCache, BuildStatus, + LockedBuildCache, build_unpacked_wheel, generate_project_for_single_rust_file, maturin_output_has_warnings, @@ -100,19 +101,15 @@ def _import_rust_file( ) -> Tuple[Optional[ModuleSpec], bool]: logger.debug('importing rust file "%s" as "%s"', file_path, module_path) - with self._build_cache.lock: - output_dir = self._build_cache.tmp_project_dir(file_path, module_name) + with self._build_cache.lock() as build_cache: + output_dir = build_cache.tmp_project_dir(file_path, module_name) logger.debug("output dir: %s", output_dir) settings = self._get_settings(module_path, file_path) dist_dir = output_dir / "dist" package_dir = dist_dir / module_name spec, reason = self._get_spec_for_up_to_date_extension_module( - package_dir, - module_path, - module_name, - file_path, - settings, + package_dir, module_path, module_name, file_path, settings, build_cache ) if spec is not None: return spec, False @@ -149,7 +146,7 @@ def _import_rust_file( settings.to_args(), maturin_output, ) - self._build_cache.store_build_status(build_status) + build_cache.store_build_status(build_status) return ( _get_spec_for_extension_module(module_path, extension_module_path), True, @@ -162,6 +159,7 @@ def _get_spec_for_up_to_date_extension_module( module_name: str, source_path: Path, settings: MaturinSettings, + build_cache: LockedBuildCache, ) -> Tuple[Optional[ModuleSpec], Optional[str]]: """Return a spec for the given module at the given search_dir if it exists and is newer than the source code that it is derived from. @@ -180,7 +178,7 @@ def _get_spec_for_up_to_date_extension_module( if extension_module_mtime < source_path.stat().st_mtime: return None, "module is out of date" - build_status = self._build_cache.get_build_status(source_path) + build_status = build_cache.get_build_status(source_path) if build_status is None: return None, "no build status found" if build_status.source_path != source_path: diff --git a/maturin/import_hook/settings.py b/maturin/import_hook/settings.py index 67bef6c21..9a86dfa0a 100644 --- a/maturin/import_hook/settings.py +++ b/maturin/import_hook/settings.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from pathlib import Path -from typing import List, Optional, Dict, Set +from typing import Dict, List, Optional, Set __all__ = [ "MaturinSettings", @@ -13,7 +13,7 @@ @dataclass class MaturinSettings: - """Settings common to `maturin build` and `maturin develop`""" + """Settings common to `maturin build` and `maturin develop`.""" release: bool = False strip: bool = False @@ -40,7 +40,7 @@ def supported_commands() -> Set[str]: @staticmethod def default() -> "MaturinSettings": - """MaturinSettings() sets no flags but default() corresponds to some sensible defaults""" + """MaturinSettings() sets no flags but default() corresponds to some sensible defaults.""" return MaturinSettings( color=True, ) @@ -100,7 +100,7 @@ def to_args(self) -> List[str]: @dataclass class MaturinBuildSettings(MaturinSettings): - """settings for `maturin build`""" + """settings for `maturin build`.""" skip_auditwheel: bool = False zig: bool = False @@ -121,7 +121,7 @@ def to_args(self) -> List[str]: @dataclass class MaturinDevelopSettings(MaturinSettings): - """settings for `maturin develop`""" + """settings for `maturin develop`.""" extras: Optional[List[str]] = None skip_install: bool = False diff --git a/test-crates/pyo3-mixed-with-path-dep/Cargo.toml b/test-crates/pyo3-mixed-with-path-dep/Cargo.toml index ad022c116..89502e081 100644 --- a/test-crates/pyo3-mixed-with-path-dep/Cargo.toml +++ b/test-crates/pyo3-mixed-with-path-dep/Cargo.toml @@ -2,7 +2,6 @@ authors = [] name = "pyo3-mixed-with-path-dep" version = "2.1.3" -description = "a library using " edition = "2021" [dependencies] diff --git a/tests/import_hook/test_utilities.py b/tests/import_hook/test_utilities.py index cbb89858b..4be6ec9a8 100644 --- a/tests/import_hook/test_utilities.py +++ b/tests/import_hook/test_utilities.py @@ -13,17 +13,10 @@ import pytest from maturin.import_hook import MaturinSettings -from maturin.import_hook._building import ( - BuildCache, - BuildStatus, - LockNotHeldError, -) +from maturin.import_hook._building import BuildCache, BuildStatus from maturin.import_hook._file_lock import AtomicOpenLock, FileLock from maturin.import_hook._resolve_project import ProjectResolveError, _resolve_project -from maturin.import_hook.project_importer import ( - _get_installed_package_mtime, - _get_project_mtime, -) +from maturin.import_hook.project_importer import _get_installed_package_mtime, _get_project_mtime from maturin.import_hook.settings import MaturinDevelopSettings, MaturinBuildSettings from .common import log, test_crates @@ -389,27 +382,19 @@ def test_resolve_project(project_name: str) -> None: def test_build_cache(tmp_path: Path) -> None: cache = BuildCache(tmp_path / "build", lock_timeout_seconds=1) - with pytest.raises(LockNotHeldError): - cache.tmp_project_dir(tmp_path / "a", "a") - with pytest.raises(LockNotHeldError): - cache.store_build_status(BuildStatus(1, tmp_path / "source", [], "")) - - with pytest.raises(LockNotHeldError): - cache.get_build_status(tmp_path / "source") - - with cache.lock: - dir_1 = cache.tmp_project_dir(tmp_path / "my_module", "my_module") - dir_2 = cache.tmp_project_dir(tmp_path / "other_place", "my_module") + with cache.lock() as locked_cache: + dir_1 = locked_cache.tmp_project_dir(tmp_path / "my_module", "my_module") + dir_2 = locked_cache.tmp_project_dir(tmp_path / "other_place", "my_module") assert dir_1 != dir_2 status1 = BuildStatus(1.2, tmp_path / "source1", [], "") status2 = BuildStatus(1.2, tmp_path / "source2", [], "") - cache.store_build_status(status1) - cache.store_build_status(status2) - assert cache.get_build_status(tmp_path / "source1") == status1 - assert cache.get_build_status(tmp_path / "source2") == status2 - assert cache.get_build_status(tmp_path / "source3") is None + locked_cache.store_build_status(status1) + locked_cache.store_build_status(status2) + assert locked_cache.get_build_status(tmp_path / "source1") == status1 + assert locked_cache.get_build_status(tmp_path / "source2") == status2 + assert locked_cache.get_build_status(tmp_path / "source3") is None def _optional_path_to_str(path: Optional[Path]) -> Optional[str]: From 75c89812865425180057153b96cb5d82dc236d15 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Tue, 29 Aug 2023 21:13:45 +0100 Subject: [PATCH 13/57] small improvements from reviewing --- guide/src/develop.md | 8 ++++-- maturin/import_hook/_building.py | 38 +++++++++++++------------ maturin/import_hook/_resolve_project.py | 11 ++++--- maturin/import_hook/project_importer.py | 22 +++++++++----- tests/import_hook/test_utilities.py | 29 ++++++++++++++++--- 5 files changed, 70 insertions(+), 38 deletions(-) diff --git a/guide/src/develop.md b/guide/src/develop.md index 4fc930717..3cc716601 100644 --- a/guide/src/develop.md +++ b/guide/src/develop.md @@ -123,8 +123,8 @@ Then Python source code changes will take effect immediately. Starting from v0.12.4, the [Python maturin package](https://pypi.org/project/maturin/) provides a Python import hook to allow quickly build and load a Rust module into Python. -It supports pure Rust and mixed Rust/Python project layout as well as a -standalone `.rs` file. +It supports importing editable-installed pure Rust and mixed Rust/Python project +layouts as well as importing standalone `.rs` files. ```python from maturin import import_hook @@ -137,7 +137,9 @@ import_hook.install() import pyo3_pure # when a .rs file is imported a project will be created for it in the -# maturin build cache and the resulting library will be loaded +# maturin build cache and the resulting library will be loaded. +# +# assuming subpackage/my_rust_script.rs defines a pyo3 module: import subpackage.my_rust_script ``` diff --git a/maturin/import_hook/_building.py b/maturin/import_hook/_building.py index c37e54069..3e9e8b2cb 100644 --- a/maturin/import_hook/_building.py +++ b/maturin/import_hook/_building.py @@ -13,13 +13,18 @@ from pathlib import Path from typing import Generator, List, Optional, Tuple -from ._file_lock import FileLock -from ._logging import logger -from .settings import MaturinSettings +from maturin.import_hook._file_lock import FileLock +from maturin.import_hook._logging import logger +from maturin.import_hook.settings import MaturinSettings @dataclass class BuildStatus: + """Information about the build of a project triggered by the import hook. + + Used to decide whether a project needs to be rebuilt. + """ + build_mtime: float source_path: Path maturin_args: List[str] @@ -92,17 +97,15 @@ def lock(self) -> Generator[LockedBuildCache, None, None]: def _get_default_build_dir() -> Path: build_dir = os.environ.get("MATURIN_BUILD_DIR", None) - if build_dir and os.access(sys.exec_prefix, os.W_OK): - return Path(build_dir) + if build_dir: + shared_build_dir = Path(build_dir) elif os.access(sys.exec_prefix, os.W_OK): return Path(sys.exec_prefix) / "maturin_build_cache" else: - version_string = sys.version.split()[0] - interpreter_hash = hashlib.sha1(sys.exec_prefix.encode()).hexdigest() - return ( - _get_cache_dir() - / f"maturin_build_cache/{version_string}_{interpreter_hash}" - ) + shared_build_dir = _get_cache_dir() / "maturin_build_cache" + version_string = sys.version.split()[0] + interpreter_hash = hashlib.sha1(sys.exec_prefix.encode()).hexdigest() + return shared_build_dir / f"{version_string}_{interpreter_hash}" def _get_cache_dir() -> Path: @@ -169,6 +172,8 @@ def build_wheel( "build", "--manifest-path", str(manifest_path), + "--interpreter", + sys.executable, "--out", str(output_dir), *settings.to_args(), @@ -183,18 +188,15 @@ def build_wheel( def develop_build_project( manifest_path: Path, settings: MaturinSettings, - skip_install: bool, ) -> str: - args = ["develop", "--manifest-path", str(manifest_path)] - if skip_install: - args.append("--skip-install") if "develop" not in settings.supported_commands(): msg = ( f'provided {type(settings).__name__} does not support the "develop" command' ) raise ImportError(msg) - args.extend(settings.to_args()) - success, output = _run_maturin(args) + success, output = _run_maturin( + ["develop", "--manifest-path", str(manifest_path), *settings.to_args()] + ) if not success: msg = "Failed to build package with maturin" raise ImportError(msg) @@ -208,7 +210,7 @@ def _run_maturin(args: list[str]) -> Tuple[bool, str]: raise ImportError(msg) logger.debug('using maturin at: "%s"', maturin_path) - command: List[str] = [maturin_path, *args] + command = [maturin_path, *args] result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output = result.stdout.decode() if result.returncode != 0: diff --git a/maturin/import_hook/_resolve_project.py b/maturin/import_hook/_resolve_project.py index 342e9ff4c..60d56939a 100644 --- a/maturin/import_hook/_resolve_project.py +++ b/maturin/import_hook/_resolve_project.py @@ -30,7 +30,7 @@ def is_maybe_maturin_project(project_dir: Path) -> bool: class ProjectResolver: def __init__(self) -> None: - self._resolved_project_cache: Dict[Path, MaturinProject] = {} + self._resolved_project_cache: Dict[Path, Optional[MaturinProject]] = {} def resolve(self, project_dir: Path) -> Optional["MaturinProject"]: if project_dir not in self._resolved_project_cache: @@ -39,8 +39,7 @@ def resolve(self, project_dir: Path) -> Optional["MaturinProject"]: resolved = _resolve_project(project_dir) except ProjectResolveError as e: logger.info('failed to resolve project "%s": %s', project_dir, e) - else: - self._resolved_project_cache[project_dir] = resolved + self._resolved_project_cache[project_dir] = resolved else: resolved = self._resolved_project_cache[project_dir] return resolved @@ -190,7 +189,7 @@ def _resolve_module_name( module_name = cargo.get("lib", {}).get("name", None) if module_name is not None: return module_name - module_name = pyproject.get("project", {}).get("name") + module_name = pyproject.get("project", {}).get("name", None) if module_name is not None: return module_name return cargo.get("package", {}).get("name", None) @@ -223,10 +222,10 @@ def _resolve_py_root(project_dir: Path, pyproject: Dict[str, Any]) -> Path: pyproject.get("tool", {}).get("maturin", {}).get("python-packages", []) ) - import_name = project_name.replace("-", "_") + package_name = project_name.replace("-", "_") python_src_found = any( (project_dir / p / "__init__.py").is_file() - for p in itertools.chain((f"src/{import_name}/",), python_packages) + for p in itertools.chain((f"src/{package_name}/",), python_packages) ) if rust_cargo_toml_found and python_src_found: return project_dir / "src" diff --git a/maturin/import_hook/project_importer.py b/maturin/import_hook/project_importer.py index cc567ee6a..ba3a5fb65 100644 --- a/maturin/import_hook/project_importer.py +++ b/maturin/import_hook/project_importer.py @@ -150,7 +150,11 @@ def _rebuild_project( resolved = self._resolver.resolve(project_dir) if resolved is None: return None, False - logger.debug("module name %s", resolved.module_full_name) + logger.debug( + 'resolved package "%s", module "%s"', + resolved.package_name, + resolved.module_full_name, + ) if package_name != resolved.package_name: logger.debug( 'package name "%s" of project does not match "%s". Not importing', @@ -185,7 +189,7 @@ def _rebuild_project( logger.info('building "%s"', package_name) start = time.perf_counter() maturin_output = develop_build_project( - resolved.cargo_manifest_path, settings, skip_install=False + resolved.cargo_manifest_path, settings ) _fix_direct_url(project_dir, package_name) logger.debug( @@ -227,7 +231,7 @@ def _get_spec_for_up_to_date_package( settings: MaturinSettings, build_cache: LockedBuildCache, ) -> Tuple[Optional[ModuleSpec], Optional[str]]: - """Return a spec for the given module at the given search_dir if it exists and is newer than the source + """Return a spec for the package if it exists and is newer than the source code that it is derived from. """ logger.debug('checking whether the package "%s" is up to date', package_name) @@ -337,7 +341,9 @@ def _find_maturin_project_above(path: Path) -> Optional[Path]: return None -def _load_dist_info(path: Path, package_name: str) -> Tuple[Optional[Path], bool]: +def _load_dist_info( + path: Path, package_name: str, *, require_project_target: bool = True +) -> Tuple[Optional[Path], bool]: dist_info_path = next(path.glob(f"{package_name}-*.dist-info"), None) if dist_info_path is None: return None, False @@ -354,8 +360,8 @@ def _load_dist_info(path: Path, package_name: str) -> Tuple[Optional[Path], bool prefix = "file://" if not url.startswith(prefix): return None, is_editable - linked_path = Path(url[len(prefix) :]) - if is_maybe_maturin_project(linked_path): + linked_path = Path(urllib.parse.unquote(url[len(prefix) :])) + if not require_project_target or is_maybe_maturin_project(linked_path): return linked_path, is_editable else: return None, is_editable @@ -447,7 +453,9 @@ def _get_project_mtime( return max( path.stat().st_mtime for path in _get_files_in_dirs( - [project_dir, *all_path_dependencies], excluded_dir_names, excluded_dirs + itertools.chain((project_dir,), all_path_dependencies), + excluded_dir_names, + excluded_dirs, ) ) except (FileNotFoundError, ValueError): diff --git a/tests/import_hook/test_utilities.py b/tests/import_hook/test_utilities.py index 4be6ec9a8..77783a97e 100644 --- a/tests/import_hook/test_utilities.py +++ b/tests/import_hook/test_utilities.py @@ -16,9 +16,12 @@ from maturin.import_hook._building import BuildCache, BuildStatus from maturin.import_hook._file_lock import AtomicOpenLock, FileLock from maturin.import_hook._resolve_project import ProjectResolveError, _resolve_project -from maturin.import_hook.project_importer import _get_installed_package_mtime, _get_project_mtime +from maturin.import_hook.project_importer import ( + _get_installed_package_mtime, + _get_project_mtime, + _load_dist_info, +) from maturin.import_hook.settings import MaturinDevelopSettings, MaturinBuildSettings - from .common import log, test_crates # set this to be able to run these tests without going through run.rs each time @@ -388,14 +391,32 @@ def test_build_cache(tmp_path: Path) -> None: dir_2 = locked_cache.tmp_project_dir(tmp_path / "other_place", "my_module") assert dir_1 != dir_2 - status1 = BuildStatus(1.2, tmp_path / "source1", [], "") - status2 = BuildStatus(1.2, tmp_path / "source2", [], "") + status1 = BuildStatus(1.2, tmp_path / "source1", ["arg1"], "output1") + status2 = BuildStatus(1.2, tmp_path / "source2", ["arg2"], "output2") locked_cache.store_build_status(status1) locked_cache.store_build_status(status2) assert locked_cache.get_build_status(tmp_path / "source1") == status1 assert locked_cache.get_build_status(tmp_path / "source2") == status2 assert locked_cache.get_build_status(tmp_path / "source3") is None + status1b = BuildStatus(1.3, tmp_path / "source1", ["arg1b"], "output1b") + locked_cache.store_build_status(status1b) + assert locked_cache.get_build_status(tmp_path / "source1") == status1b + + +def test_load_dist_info(tmp_path: Path) -> None: + dist_info = tmp_path / "package_foo-1.0.0.dist-info" + dist_info.mkdir(parents=True) + (dist_info / "direct_url.json").write_text( + '{"dir_info": {"editable": true}, "url": "file:///tmp/some%20directory/foo"}' + ) + + linked_path, is_editable = _load_dist_info( + tmp_path, "package_foo", require_project_target=False + ) + assert linked_path == Path("/tmp/some directory/foo") + assert is_editable + def _optional_path_to_str(path: Optional[Path]) -> Optional[str]: return str(path) if path is not None else None From dfe850f3803ce311b64373ab8f09a9c8dee71b90 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Tue, 29 Aug 2023 22:03:51 +0100 Subject: [PATCH 14/57] refactored to make more customization possible by subclassing importer classes --- guide/src/develop.md | 37 +++++----- maturin/import_hook/__init__.py | 9 +-- maturin/import_hook/_building.py | 36 +-------- maturin/import_hook/project_importer.py | 23 +++--- maturin/import_hook/rust_file_importer.py | 74 ++++++++++++++----- maturin/import_hook/settings.py | 9 --- .../rebuild_on_settings_change_helper.py | 21 ++---- tests/import_hook/test_project_importer.py | 5 +- tests/import_hook/test_rust_file_importer.py | 6 +- 9 files changed, 102 insertions(+), 118 deletions(-) diff --git a/guide/src/develop.md b/guide/src/develop.md index 3cc716601..8756a3a51 100644 --- a/guide/src/develop.md +++ b/guide/src/develop.md @@ -169,14 +169,25 @@ import_hook.install( ) ``` -Custom settings providers can be used to override settings of particular projects -or implement custom logic such as loading settings from configuration files +Since the import hook is intended for use in development environments and not for +production environments, it may be a good idea to put the call to `import_hook.install()` +into `site-packages/sitecustomize.py` of your development virtual environment +([documentation](https://docs.python.org/3/library/site.html)). This will +enable the hook for every script run by that interpreter without calling `import_hook.install()` +in every script, meaning the scripts do not need alteration before deployment. + + +### Advanced Usage + +The import hook classes can be subclassed to further customize to specific use cases. +For example settings can be configured per-project or loaded from configuration files. ```python +import sys from pathlib import Path -from maturin import import_hook -from maturin.import_hook.settings import MaturinSettings, MaturinSettingsProvider +from maturin.import_hook.settings import MaturinSettings +from maturin.import_hook.project_importer import MaturinProjectImporter -class CustomSettings(MaturinSettingsProvider): +class CustomImporter(MaturinProjectImporter): def get_settings(self, module_path: str, source_path: Path) -> MaturinSettings: return MaturinSettings( release=True, @@ -184,23 +195,9 @@ class CustomSettings(MaturinSettingsProvider): # ... ) -import_hook.install( - enable_project_importer=True, - enable_rs_file_importer=True, - settings=CustomSettings(), - show_warnings=True, - # ... -) +sys.meta_path.insert(0, CustomImporter()) ``` -Since the import hook is intended for use in development environments and not for -production environments, it may be a good idea to put the call to `import_hook.install()` -into `site-packages/sitecustomize.py` of your development virtual environment -([documentation](https://docs.python.org/3/library/site.html)). This will -enable the hook for every script run by that interpreter without calling `import_hook.install()` -in every script, meaning the scripts do not need alteration before deployment. - - The import hook internals can be examined by configuring the root logger and calling `reset_logger` to propagate messages from the `maturin.import_hook` logger to the root logger. You can also run with the environment variable `RUST_LOG=maturin=debug` diff --git a/maturin/import_hook/__init__.py b/maturin/import_hook/__init__.py index ac97b484e..64991d31b 100644 --- a/maturin/import_hook/__init__.py +++ b/maturin/import_hook/__init__.py @@ -1,9 +1,9 @@ from pathlib import Path -from typing import Optional, Set, Union +from typing import Optional, Set from maturin.import_hook import project_importer, rust_file_importer from maturin.import_hook._logging import reset_logger -from maturin.import_hook.settings import MaturinSettings, MaturinSettingsProvider +from maturin.import_hook.settings import MaturinSettings __all__ = ["install", "uninstall", "reset_logger"] @@ -12,7 +12,7 @@ def install( *, enable_project_importer: bool = True, enable_rs_file_importer: bool = True, - settings: Optional[Union[MaturinSettings, MaturinSettingsProvider]] = None, + settings: Optional[MaturinSettings] = None, build_dir: Optional[Path] = None, install_new_packages: bool = True, force_rebuild: bool = False, @@ -26,8 +26,7 @@ def install( :param enable_rs_file_importer: enable the hook for importing .rs files as though they were regular python modules - :param settings: settings corresponding to flags passed to maturin. Pass MaturinSettings to use the same - settings for every project or MaturinSettingsProvider to customize + :param settings: settings corresponding to flags passed to maturin. :param build_dir: where to put the compiled artifacts. defaults to `$MATURIN_BUILD_DIR`, `sys.exec_prefix / 'maturin_build_cache'` or diff --git a/maturin/import_hook/_building.py b/maturin/import_hook/_building.py index 3e9e8b2cb..08d55973b 100644 --- a/maturin/import_hook/_building.py +++ b/maturin/import_hook/_building.py @@ -129,36 +129,6 @@ def _get_cache_dir() -> Path: return Path("~/.cache").expanduser() -def generate_project_for_single_rust_file( - build_dir: Path, - rust_file: Path, - available_features: Optional[list[str]], -) -> Path: - project_dir = build_dir / rust_file.stem - if project_dir.exists(): - shutil.rmtree(project_dir) - - success, output = _run_maturin(["new", "--bindings", "pyo3", str(project_dir)]) - if not success: - msg = "Failed to generate project for rust file" - raise ImportError(msg) - - if available_features is not None: - available_features = [ - feature for feature in available_features if "/" not in feature - ] - cargo_manifest = project_dir / "Cargo.toml" - cargo_manifest.write_text( - "{}\n[features]\n{}".format( - cargo_manifest.read_text(), - "\n".join(f"{feature} = []" for feature in available_features), - ) - ) - - shutil.copy(rust_file, project_dir / "src/lib.rs") - return project_dir - - def build_wheel( manifest_path: Path, output_dir: Path, @@ -167,7 +137,7 @@ def build_wheel( if "build" not in settings.supported_commands(): msg = f'provided {type(settings).__name__} does not support the "build" command' raise ImportError(msg) - success, output = _run_maturin( + success, output = run_maturin( [ "build", "--manifest-path", @@ -194,7 +164,7 @@ def develop_build_project( f'provided {type(settings).__name__} does not support the "develop" command' ) raise ImportError(msg) - success, output = _run_maturin( + success, output = run_maturin( ["develop", "--manifest-path", str(manifest_path), *settings.to_args()] ) if not success: @@ -203,7 +173,7 @@ def develop_build_project( return output -def _run_maturin(args: list[str]) -> Tuple[bool, str]: +def run_maturin(args: list[str]) -> Tuple[bool, str]: maturin_path = shutil.which("maturin") if maturin_path is None: msg = "maturin not found in the PATH" diff --git a/maturin/import_hook/project_importer.py b/maturin/import_hook/project_importer.py index ba3a5fb65..6d3ef440b 100644 --- a/maturin/import_hook/project_importer.py +++ b/maturin/import_hook/project_importer.py @@ -26,7 +26,7 @@ ProjectResolver, is_maybe_maturin_project, ) -from maturin.import_hook.settings import MaturinSettings, MaturinSettingsProvider +from maturin.import_hook.settings import MaturinSettings __all__ = [ "MaturinProjectImporter", @@ -54,7 +54,7 @@ class MaturinProjectImporter(importlib.abc.MetaPathFinder): def __init__( self, *, - settings: Optional[Union[MaturinSettings, MaturinSettingsProvider]] = None, + settings: Optional[MaturinSettings] = None, build_dir: Optional[Path] = None, lock_timeout_seconds: Optional[float] = 120, install_new_packages: bool = True, @@ -74,13 +74,11 @@ def __init__( else excluded_dir_names ) - def _get_settings(self, module_path: str, source_path: Path) -> MaturinSettings: - if isinstance(self._settings, MaturinSettings): - return self._settings - elif isinstance(self._settings, MaturinSettingsProvider): - return self._settings.get_settings(module_path, source_path) - else: - return MaturinSettings.default() + def get_settings(self, module_path: str, source_path: Path) -> MaturinSettings: + """this method can be overridden in subclasses to customize settings for specific projects""" + return ( + self._settings if self._settings is not None else MaturinSettings.default() + ) def find_spec( self, @@ -176,7 +174,7 @@ def _rebuild_project( logger.debug('importing project "%s" as "%s"', project_dir, package_name) with self._build_cache.lock() as build_cache: - settings = self._get_settings(package_name, project_dir) + settings = self.get_settings(package_name, project_dir) spec, reason = self._get_spec_for_up_to_date_package( package_name, project_dir, resolved, settings, build_cache ) @@ -526,7 +524,7 @@ def _get_files_in_dirs( def install( *, - settings: Optional[Union[MaturinSettings, MaturinSettingsProvider]] = None, + settings: Optional[MaturinSettings] = None, build_dir: Optional[Path] = None, install_new_packages: bool = True, force_rebuild: bool = False, @@ -536,8 +534,7 @@ def install( ) -> MaturinProjectImporter: """Install an import hook for automatically rebuilding editable installed maturin projects. - :param settings: settings corresponding to flags passed to maturin. Pass MaturinSettings to use the same - settings for every project or MaturinSettingsProvider to customize + :param settings: settings corresponding to flags passed to maturin. :param build_dir: where to put the compiled artifacts. defaults to `$MATURIN_BUILD_DIR`, `sys.exec_prefix / 'maturin_build_cache'` or diff --git a/maturin/import_hook/rust_file_importer.py b/maturin/import_hook/rust_file_importer.py index a9ca4c2fc..350ce16de 100644 --- a/maturin/import_hook/rust_file_importer.py +++ b/maturin/import_hook/rust_file_importer.py @@ -4,6 +4,7 @@ import logging import math import os +import shutil import sys import time from importlib.machinery import ExtensionFileLoader, ModuleSpec @@ -16,12 +17,12 @@ BuildStatus, LockedBuildCache, build_unpacked_wheel, - generate_project_for_single_rust_file, maturin_output_has_warnings, + run_maturin, ) from maturin.import_hook._logging import logger -from maturin.import_hook._resolve_project import ProjectResolver -from maturin.import_hook.settings import MaturinSettings, MaturinSettingsProvider +from maturin.import_hook._resolve_project import ProjectResolver, find_cargo_manifest +from maturin.import_hook.settings import MaturinSettings __all__ = ["MaturinRustFileImporter", "install", "uninstall", "IMPORTER"] @@ -32,7 +33,7 @@ class MaturinRustFileImporter(importlib.abc.MetaPathFinder): def __init__( self, *, - settings: Optional[Union[MaturinSettings, MaturinSettingsProvider]] = None, + settings: Optional[MaturinSettings] = None, build_dir: Optional[Path] = None, force_rebuild: bool = False, lock_timeout_seconds: Optional[float] = 120, @@ -44,13 +45,42 @@ def __init__( self._build_cache = BuildCache(build_dir, lock_timeout_seconds) self._show_warnings = show_warnings - def _get_settings(self, module_path: str, source_path: Path) -> MaturinSettings: - if isinstance(self._settings, MaturinSettings): - return self._settings - elif isinstance(self._settings, MaturinSettingsProvider): - return self._settings.get_settings(module_path, source_path) - else: - return MaturinSettings.default() + def get_settings(self, module_path: str, source_path: Path) -> MaturinSettings: + """this method can be overridden in subclasses to customize settings for specific projects""" + return ( + self._settings if self._settings is not None else MaturinSettings.default() + ) + + @staticmethod + def generate_project_for_single_rust_file( + module_path: str, + project_dir: Path, + rust_file: Path, + settings: MaturinSettings, + ) -> Path: + """this method can be overridden in subclasses to customize project generation""" + if project_dir.exists(): + shutil.rmtree(project_dir) + + success, output = run_maturin(["new", "--bindings", "pyo3", str(project_dir)]) + if not success: + msg = "Failed to generate project for rust file" + raise ImportError(msg) + + if settings.features is not None: + available_features = [ + feature for feature in settings.features if "/" not in feature + ] + cargo_manifest = project_dir / "Cargo.toml" + cargo_manifest.write_text( + "{}\n[features]\n{}".format( + cargo_manifest.read_text(), + "\n".join(f"{feature} = []" for feature in available_features), + ) + ) + + shutil.copy(rust_file, project_dir / "src/lib.rs") + return project_dir def find_spec( self, @@ -104,7 +134,7 @@ def _import_rust_file( with self._build_cache.lock() as build_cache: output_dir = build_cache.tmp_project_dir(file_path, module_name) logger.debug("output dir: %s", output_dir) - settings = self._get_settings(module_path, file_path) + settings = self.get_settings(module_path, file_path) dist_dir = output_dir / "dist" package_dir = dist_dir / module_name @@ -118,12 +148,17 @@ def _import_rust_file( logger.info('building "%s"', module_path) logger.debug('creating project for "%s" and compiling', file_path) start = time.perf_counter() - output_dir = generate_project_for_single_rust_file( - output_dir, file_path, settings.features - ) - maturin_output = build_unpacked_wheel( - output_dir / "Cargo.toml", dist_dir, settings + project_dir = self.generate_project_for_single_rust_file( + module_path, output_dir / file_path.stem, file_path, settings ) + manifest_path = find_cargo_manifest(project_dir) + if manifest_path is None: + msg = ( + f"cargo manifest not found in the project generated for {file_path}" + ) + raise ImportError(msg) + + maturin_output = build_unpacked_wheel(manifest_path, dist_dir, settings) logger.debug( 'compiled "%s" in %.3fs', file_path, @@ -241,7 +276,7 @@ def _get_spec_for_extension_module( def install( *, - settings: Optional[Union[MaturinSettings, MaturinSettingsProvider]] = None, + settings: Optional[MaturinSettings] = None, build_dir: Optional[Path] = None, force_rebuild: bool = False, lock_timeout_seconds: Optional[float] = 120, @@ -250,8 +285,7 @@ def install( """Install the 'rust file' importer to import .rs files as though they were regular python modules. - :param settings: settings corresponding to flags passed to maturin. Pass MaturinSettings to use the same - settings for every project or MaturinSettingsProvider to customize + :param settings: settings corresponding to flags passed to maturin. :param build_dir: where to put the compiled artifacts. defaults to `$MATURIN_BUILD_DIR`, `sys.exec_prefix / 'maturin_build_cache'` or diff --git a/maturin/import_hook/settings.py b/maturin/import_hook/settings.py index 9a86dfa0a..1812bb520 100644 --- a/maturin/import_hook/settings.py +++ b/maturin/import_hook/settings.py @@ -1,13 +1,10 @@ -from abc import ABC, abstractmethod from dataclasses import dataclass -from pathlib import Path from typing import Dict, List, Optional, Set __all__ = [ "MaturinSettings", "MaturinBuildSettings", "MaturinDevelopSettings", - "MaturinSettingsProvider", ] @@ -139,9 +136,3 @@ def to_args(self) -> List[str]: args.append("--skip-install") args.extend(super().to_args()) return args - - -class MaturinSettingsProvider(ABC): - @abstractmethod - def get_settings(self, module_path: str, source_path: Path) -> MaturinSettings: - raise NotImplementedError diff --git a/tests/import_hook/rust_file_import/rebuild_on_settings_change_helper.py b/tests/import_hook/rust_file_import/rebuild_on_settings_change_helper.py index 9b6aff61f..9fbeb66c2 100644 --- a/tests/import_hook/rust_file_import/rebuild_on_settings_change_helper.py +++ b/tests/import_hook/rust_file_import/rebuild_on_settings_change_helper.py @@ -1,27 +1,22 @@ # ruff: noqa: E402 import logging import sys -from pathlib import Path logging.basicConfig(format="%(name)s [%(levelname)s] %(message)s", level=logging.DEBUG) from maturin import import_hook -from maturin.import_hook.settings import MaturinSettings, MaturinSettingsProvider +from maturin.import_hook.settings import MaturinSettings import_hook.reset_logger() +if len(sys.argv) > 1 and sys.argv[1] == "LARGE_NUMBER": + print("building with large_number feature enabled") + settings = MaturinSettings(features=["pyo3/extension-module", "large_number"]) +else: + print("building with default settings") + settings = MaturinSettings.default() -class CustomSettingsProvider(MaturinSettingsProvider): - def get_settings(self, module_path: str, source_path: Path) -> MaturinSettings: - if len(sys.argv) > 1 and sys.argv[1] == "LARGE_NUMBER": - print(f"building {module_path} with large_number feature enabled") - return MaturinSettings(features=["pyo3/extension-module", "large_number"]) - else: - print(f"building {module_path} with default settings") - return MaturinSettings.default() - - -import_hook.install(settings=CustomSettingsProvider()) +import_hook.install(settings=settings) from my_script import get_num diff --git a/tests/import_hook/test_project_importer.py b/tests/import_hook/test_project_importer.py index 773f8dafc..b2724fd9a 100644 --- a/tests/import_hook/test_project_importer.py +++ b/tests/import_hook/test_project_importer.py @@ -548,6 +548,7 @@ def test_rebuild_on_settings_change(workspace: Path, is_mixed: bool) -> None: helper_path = script_dir / "rust_file_import/rebuild_on_settings_change_helper.py" output1, _ = run_python([str(helper_path)], cwd=workspace) + assert "building with default settings" in output1 assert "get_num = 10" in output1 assert "SUCCESS" in output1 assert ( @@ -560,7 +561,7 @@ def test_rebuild_on_settings_change(workspace: Path, is_mixed: bool) -> None: assert 'package up to date: "my_script"' in output2 output3, _ = run_python([str(helper_path), "LARGE_NUMBER"], cwd=workspace) - assert "building my_script with large_number feature enabled" in output3 + assert "building with large_number feature enabled" in output3 assert ( 'package "my_script" will be rebuilt because: ' "current maturin args do not match the previous build" @@ -569,7 +570,7 @@ def test_rebuild_on_settings_change(workspace: Path, is_mixed: bool) -> None: assert "SUCCESS" in output3 output4, _ = run_python([str(helper_path), "LARGE_NUMBER"], cwd=workspace) - assert "building my_script with large_number feature enabled" in output4 + assert "building with large_number feature enabled" in output4 assert 'package up to date: "my_script"' in output4 assert "get_num = 100" in output4 assert "SUCCESS" in output4 diff --git a/tests/import_hook/test_rust_file_importer.py b/tests/import_hook/test_rust_file_importer.py index e2259241c..fb9c29e1e 100644 --- a/tests/import_hook/test_rust_file_importer.py +++ b/tests/import_hook/test_rust_file_importer.py @@ -204,7 +204,7 @@ def test_rebuild_on_settings_change(workspace: Path) -> None: output1, _ = run_python([str(helper_path)], cwd=workspace) assert "get_num = 10" in output1 assert "SUCCESS" in output1 - assert "building my_script with default settings" in output1 + assert "building with default settings" in output1 assert "module up to date" not in output1 assert "creating project for" in output1 @@ -214,14 +214,14 @@ def test_rebuild_on_settings_change(workspace: Path) -> None: assert "module up to date" in output2 output3, _ = run_python([str(helper_path), "LARGE_NUMBER"], cwd=workspace) - assert "building my_script with large_number feature enabled" in output3 + assert "building with large_number feature enabled" in output3 assert "module up to date" not in output3 assert "creating project for" in output3 assert "get_num = 100" in output3 assert "SUCCESS" in output3 output4, _ = run_python([str(helper_path), "LARGE_NUMBER"], cwd=workspace) - assert "building my_script with large_number feature enabled" in output4 + assert "building with large_number feature enabled" in output4 assert "module up to date" in output4 assert "get_num = 100" in output4 assert "SUCCESS" in output4 From 127d34bc82f3834fd153f6d138af1a99c8c503d9 Mon Sep 17 00:00:00 2001 From: messense Date: Fri, 1 Sep 2023 15:33:15 +0800 Subject: [PATCH 15/57] Use `env!("CARGO_BIN_EXE_maturin")` --- tests/common/import_hook.rs | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/tests/common/import_hook.rs b/tests/common/import_hook.rs index 280925c2a..9d40daf12 100644 --- a/tests/common/import_hook.rs +++ b/tests/common/import_hook.rs @@ -23,22 +23,7 @@ pub fn test_import_hook( let (venv_dir, python) = create_virtualenv(virtualenv_name, Some(python)).unwrap(); println!("installing maturin binary into virtualenv"); - let status = Command::new("cargo") - .args([ - "build", - "--target-dir", - venv_dir.join("maturin").as_os_str().to_str().unwrap(), - ]) - .status() - .unwrap(); - if !status.success() { - bail!("failed to install maturin"); - } - fs::copy( - venv_dir.join("maturin/debug/maturin"), - venv_dir.join("bin/maturin"), - ) - .unwrap(); + fs::copy(env!("CARGO_BIN_EXE_maturin"), venv_dir.join("bin/maturin")).unwrap(); let pytest_args = vec![ vec!["pytest"], From aa4767077a35536d6c344f612cf72877de342a4d Mon Sep 17 00:00:00 2001 From: messense Date: Sat, 2 Sep 2023 22:36:06 +0800 Subject: [PATCH 16/57] Skip lib_with_path_dep in import hook tests --- tests/common/import_hook.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/common/import_hook.rs b/tests/common/import_hook.rs index 9d40daf12..5b12bab35 100644 --- a/tests/common/import_hook.rs +++ b/tests/common/import_hook.rs @@ -22,23 +22,19 @@ pub fn test_import_hook( let (venv_dir, python) = create_virtualenv(virtualenv_name, Some(python)).unwrap(); - println!("installing maturin binary into virtualenv"); - fs::copy(env!("CARGO_BIN_EXE_maturin"), venv_dir.join("bin/maturin")).unwrap(); - - let pytest_args = vec![ - vec!["pytest"], - vec!["uniffi-bindgen"], - vec!["cffi"], - vec!["-e", "."], - ]; + let pip_install_args = vec![vec!["pytest", "uniffi-bindgen", "cffi"], vec!["-e", "."]]; let extras: Vec> = extra_packages.into_iter().map(|name| vec![name]).collect(); - for args in pytest_args.iter().chain(&extras) { + for args in pip_install_args.iter().chain(&extras) { if verbose { println!("installing {:?}", &args); } let status = Command::new(&python) .args(["-m", "pip", "install", "--disable-pip-version-check"]) .args(args) + .env( + "MATURIN_SETUP_ARGS", + env::var("MATURIN_SETUP_ARGS").unwrap_or_else(|_| "--features full".to_string()), + ) .status() .unwrap(); if !status.success() { @@ -82,6 +78,10 @@ pub fn resolve_all_packages() -> Result { let path = path?.path(); if path.join("pyproject.toml").exists() { let project_name = path.file_name().unwrap().to_str().unwrap().to_owned(); + if project_name == "lib_with_path_dep" { + // Skip lib_with_path_dep because it's used to test `--locked` + continue; + } resolved_packages.insert(project_name, resolve_package(&path).unwrap_or(Value::Null)); } } From 5bf8d1d627c7675665ab05a47ff1399dc0995f85 Mon Sep 17 00:00:00 2001 From: messense Date: Sat, 2 Sep 2023 23:03:26 +0800 Subject: [PATCH 17/57] Add maturin package path to PYTHONPATH --- tests/common/import_hook.rs | 16 +++++++++++----- tests/import_hook/common.py | 6 ++++-- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/tests/common/import_hook.rs b/tests/common/import_hook.rs index 5b12bab35..b60edd7d5 100644 --- a/tests/common/import_hook.rs +++ b/tests/common/import_hook.rs @@ -22,7 +22,10 @@ pub fn test_import_hook( let (venv_dir, python) = create_virtualenv(virtualenv_name, Some(python)).unwrap(); - let pip_install_args = vec![vec!["pytest", "uniffi-bindgen", "cffi"], vec!["-e", "."]]; + println!("installing maturin binary into virtualenv"); + fs::copy(env!("CARGO_BIN_EXE_maturin"), venv_dir.join("bin/maturin")).unwrap(); + + let pip_install_args = vec![vec!["pytest", "uniffi-bindgen", "cffi"]]; let extras: Vec> = extra_packages.into_iter().map(|name| vec![name]).collect(); for args in pip_install_args.iter().chain(&extras) { if verbose { @@ -31,10 +34,6 @@ pub fn test_import_hook( let status = Command::new(&python) .args(["-m", "pip", "install", "--disable-pip-version-check"]) .args(args) - .env( - "MATURIN_SETUP_ARGS", - env::var("MATURIN_SETUP_ARGS").unwrap_or_else(|_| "--features full".to_string()), - ) .status() .unwrap(); if !status.success() { @@ -45,6 +44,13 @@ pub fn test_import_hook( let path = env::var_os("PATH").unwrap(); let mut paths = env::split_paths(&path).collect::>(); paths.insert(0, venv_dir.join("bin")); + paths.insert( + 0, + Path::new(env!("CARGO_BIN_EXE_maturin")) + .parent() + .unwrap() + .to_path_buf(), + ); let path = env::join_paths(paths).unwrap(); let output = Command::new(&python) diff --git a/tests/import_hook/common.py b/tests/import_hook/common.py index cd9b8e385..a3e73210f 100644 --- a/tests/import_hook/common.py +++ b/tests/import_hook/common.py @@ -6,6 +6,7 @@ import sys import tempfile import time +import itertools from pathlib import Path from typing import List, Optional, Tuple, Iterable @@ -18,7 +19,6 @@ maturin_dir = script_dir.parent.parent test_crates = maturin_dir / "test-crates" - IMPORT_HOOK_HEADER = """ import logging logging.basicConfig(format='%(name)s [%(levelname)s] %(message)s', level=logging.DEBUG) @@ -66,7 +66,9 @@ def run_python( env = os.environ if python_path is not None: - env["PYTHONPATH"] = ":".join(str(p) for p in python_path) + env["PYTHONPATH"] = os.pathsep.join(str(p) for p in itertools.chain(python_path, [maturin_dir])) + else: + env["PYTHONPATH"] = str(maturin_dir) cmd = [sys.executable, *args] try: From b9e999acdca9e2c519593cbad5bec37029377bd8 Mon Sep 17 00:00:00 2001 From: messense Date: Sat, 2 Sep 2023 23:07:06 +0800 Subject: [PATCH 18/57] black format --- tests/import_hook/common.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/import_hook/common.py b/tests/import_hook/common.py index a3e73210f..a65ef48c4 100644 --- a/tests/import_hook/common.py +++ b/tests/import_hook/common.py @@ -66,7 +66,9 @@ def run_python( env = os.environ if python_path is not None: - env["PYTHONPATH"] = os.pathsep.join(str(p) for p in itertools.chain(python_path, [maturin_dir])) + env["PYTHONPATH"] = os.pathsep.join( + str(p) for p in itertools.chain(python_path, [maturin_dir]) + ) else: env["PYTHONPATH"] = str(maturin_dir) From e021fda2f09cc77bb97328458ef5c747c930338b Mon Sep 17 00:00:00 2001 From: messense Date: Sun, 3 Sep 2023 10:18:03 +0800 Subject: [PATCH 19/57] No need to copy maturin to venv bin now --- tests/common/import_hook.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/common/import_hook.rs b/tests/common/import_hook.rs index b60edd7d5..a177c503c 100644 --- a/tests/common/import_hook.rs +++ b/tests/common/import_hook.rs @@ -22,9 +22,6 @@ pub fn test_import_hook( let (venv_dir, python) = create_virtualenv(virtualenv_name, Some(python)).unwrap(); - println!("installing maturin binary into virtualenv"); - fs::copy(env!("CARGO_BIN_EXE_maturin"), venv_dir.join("bin/maturin")).unwrap(); - let pip_install_args = vec![vec!["pytest", "uniffi-bindgen", "cffi"]]; let extras: Vec> = extra_packages.into_iter().map(|name| vec![name]).collect(); for args in pip_install_args.iter().chain(&extras) { From 37a133b86508734cfc8b3df55c067c5436c3b654 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 3 Sep 2023 15:56:38 +0100 Subject: [PATCH 20/57] run import hook tests in parallel --- tests/common/import_hook.rs | 72 +++++++++++++++++--- tests/import_hook/test_project_importer.py | 8 ++- tests/import_hook/test_rust_file_importer.py | 9 ++- tests/run.rs | 56 +++++++++++---- 4 files changed, 117 insertions(+), 28 deletions(-) diff --git a/tests/common/import_hook.rs b/tests/common/import_hook.rs index a177c503c..4fae18585 100644 --- a/tests/common/import_hook.rs +++ b/tests/common/import_hook.rs @@ -1,18 +1,18 @@ use crate::common::{create_virtualenv, test_python_path}; use anyhow::{bail, Result}; use maturin::{BuildOptions, CargoOptions, Target}; +use regex::RegexBuilder; use serde_json; use serde_json::{json, Value}; -use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use std::process::Command; -use std::{env, fs, str}; +use std::{env, fs, str, thread}; -pub fn test_import_hook( +pub fn test_import_hook<'a>( virtualenv_name: &str, - test_script_path: &Path, - extra_packages: Vec<&str>, - extra_envs: BTreeMap<&str, &str>, + test_specifier: &str, + extra_packages: &[&str], + extra_envs: &[(&str, &str)], verbose: bool, ) -> Result<()> { let python = test_python_path().map(PathBuf::from).unwrap_or_else(|| { @@ -23,7 +23,7 @@ pub fn test_import_hook( let (venv_dir, python) = create_virtualenv(virtualenv_name, Some(python)).unwrap(); let pip_install_args = vec![vec!["pytest", "uniffi-bindgen", "cffi"]]; - let extras: Vec> = extra_packages.into_iter().map(|name| vec![name]).collect(); + let extras: Vec> = extra_packages.into_iter().map(|name| vec![*name]).collect(); for args in pip_install_args.iter().chain(&extras) { if verbose { println!("installing {:?}", &args); @@ -51,10 +51,10 @@ pub fn test_import_hook( let path = env::join_paths(paths).unwrap(); let output = Command::new(&python) - .args(["-m", "pytest", test_script_path.to_str().unwrap()]) + .args(["-m", "pytest", test_specifier]) .env("PATH", path) .env("VIRTUAL_ENV", venv_dir) - .envs(extra_envs) + .envs(extra_envs.iter().cloned()) .output() .unwrap(); @@ -75,6 +75,60 @@ pub fn test_import_hook( Ok(()) } +pub fn test_import_hook_parallel( + virtualenv_name: &str, + module: &Path, + extra_packages: &[&str], + extra_envs: &[(&str, &str)], + verbose: bool, +) -> Result<()> { + let functions = get_top_level_tests(module).unwrap(); + + thread::scope(|s| { + let mut handles = vec![]; + for function_name in &functions { + let test_specifier = format!("{}::{}", module.to_str().unwrap(), function_name); + let virtualenv_name = format!("{virtualenv_name}_{function_name}"); + let mut extra_envs_this_test = extra_envs.to_vec(); + extra_envs_this_test.push(("MATURIN_TEST_NAME", function_name)); + let handle = s.spawn(move || { + test_import_hook( + &virtualenv_name, + &test_specifier, + &extra_packages, + &extra_envs_this_test, + verbose, + ) + .unwrap() + }); + handles.push(handle); + } + for handle in handles { + handle.join().unwrap(); + } + }); + Ok(()) +} + +fn get_top_level_tests(module: &Path) -> Result> { + let source = String::from_utf8(fs::read(module)?)?; + let function_pattern = RegexBuilder::new("^def (test_[^(]+)[(]") + .multi_line(true) + .build()?; + let class_pattern = RegexBuilder::new("^class (Test[^:]+):") + .multi_line(true) + .build()?; + let mut top_level_tests = vec![]; + for pattern in [function_pattern, class_pattern] { + top_level_tests.extend( + pattern + .captures_iter(&source) + .map(|c| c.get(1).unwrap().as_str().to_owned()), + ) + } + Ok(top_level_tests) +} + pub fn resolve_all_packages() -> Result { let mut resolved_packages = serde_json::Map::new(); for path in fs::read_dir("test-crates")? { diff --git a/tests/import_hook/test_project_importer.py b/tests/import_hook/test_project_importer.py index b2724fd9a..27095d782 100644 --- a/tests/import_hook/test_project_importer.py +++ b/tests/import_hook/test_project_importer.py @@ -32,13 +32,17 @@ which provides a clean virtual environment for these tests to use. """ -MATURIN_BUILD_CACHE = test_crates / "targets/import_hook_project_importer_build_cache" +MATURIN_TEST_NAME = os.environ["MATURIN_TEST_NAME"] +MATURIN_BUILD_CACHE = ( + test_crates + / f"targets/import_hook_project_importer_build_cache_{MATURIN_TEST_NAME}" +) # the CI does not have enough space to keep the outputs. # When running locally you may set this to False for debugging CLEAR_WORKSPACE = True os.environ["CARGO_TARGET_DIR"] = str( - test_crates / "targets/import_hook_project_importer" + test_crates / f"targets/import_hook_project_importer_{MATURIN_TEST_NAME}" ) os.environ["MATURIN_BUILD_DIR"] = str(MATURIN_BUILD_CACHE) diff --git a/tests/import_hook/test_rust_file_importer.py b/tests/import_hook/test_rust_file_importer.py index fb9c29e1e..940931025 100644 --- a/tests/import_hook/test_rust_file_importer.py +++ b/tests/import_hook/test_rust_file_importer.py @@ -21,12 +21,17 @@ which provides a clean virtual environment for these tests to use. """ -MATURIN_BUILD_CACHE = test_crates / "targets/import_hook_file_importer_build_cache" +MATURIN_TEST_NAME = os.environ["MATURIN_TEST_NAME"] +MATURIN_BUILD_CACHE = ( + test_crates / f"targets/import_hook_file_importer_build_cache_{MATURIN_TEST_NAME}" +) # the CI does not have enough space to keep the outputs. # When running locally you may set this to False for debugging CLEAR_WORKSPACE = True -os.environ["CARGO_TARGET_DIR"] = str(test_crates / "targets/import_hook_file_importer") +os.environ["CARGO_TARGET_DIR"] = str( + test_crates / f"targets/import_hook_file_importer_{MATURIN_TEST_NAME}" +) os.environ["MATURIN_BUILD_DIR"] = str(MATURIN_BUILD_CACHE) diff --git a/tests/run.rs b/tests/run.rs index 0215230f1..d3c3dcdf8 100644 --- a/tests/run.rs +++ b/tests/run.rs @@ -1,17 +1,19 @@ //! To speed up the tests, they are all collected in a single module -use crate::common::import_hook; +use std::path::{Path, PathBuf}; +use std::{env, fs}; + +use indoc::indoc; +use time::macros::datetime; +use which::which; + use common::{ develop, errors, get_python_implementation, handle_result, integration, other, test_python_path, }; -use indoc::indoc; use maturin::pyproject_toml::SdistGenerator; use maturin::Target; -use std::collections::BTreeMap; -use std::path::{Path, PathBuf}; -use std::{env, fs}; -use time::macros::datetime; -use which::which; + +use crate::common::import_hook; mod common; @@ -766,24 +768,48 @@ fn pyo3_source_date_epoch() { )) } +#[ignore] #[test] fn import_hook_project_importer() { handle_result(import_hook::test_import_hook( + "import_hook_project_importer", + "tests/import_hook/test_project_importer.py", + &vec!["boltons"], + &vec![("MATURIN_TEST_NAME", "ALL")], + true, + )); +} + +#[test] +fn import_hook_project_importer_parallel() { + handle_result(import_hook::test_import_hook_parallel( "import_hook_project_importer", &PathBuf::from("tests/import_hook/test_project_importer.py"), - vec!["boltons"], - BTreeMap::new(), + &vec!["boltons"], + &vec![], true, )); } +#[ignore] #[test] fn import_hook_rust_file_importer() { handle_result(import_hook::test_import_hook( + "import_hook_rust_file_importer", + "tests/import_hook/test_rust_file_importer.py", + &vec![], + &vec![("MATURIN_TEST_NAME", "ALL")], + true, + )); +} + +#[test] +fn import_hook_rust_file_importer_parallel() { + handle_result(import_hook::test_import_hook_parallel( "import_hook_rust_file_importer", &PathBuf::from("tests/import_hook/test_rust_file_importer.py"), - vec![], - BTreeMap::new(), + &vec![], + &vec![], true, )); } @@ -799,12 +825,12 @@ fn import_hook_utilities() { .unwrap(); handle_result(import_hook::test_import_hook( "import_hook_utilities", - &PathBuf::from("tests/import_hook/test_utilities.py"), - vec![], - BTreeMap::from([( + "tests/import_hook/test_utilities.py", + &vec![], + &vec![( "RESOLVED_PACKAGES_PATH", resolved_packages_path.to_str().unwrap(), - )]), + )], true, )); } From 0dcd8b715b23a7342872e90a8c3d5d0fdfaa0483 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 3 Sep 2023 17:12:13 +0100 Subject: [PATCH 21/57] replaced custom FileLock implementation with filelock package --- guide/src/develop.md | 19 ++- maturin/import_hook/_building.py | 27 ++++- maturin/import_hook/_file_lock.py | 192 ------------------------------ pyproject.toml | 3 + tests/common/import_hook.rs | 13 +- 5 files changed, 47 insertions(+), 207 deletions(-) delete mode 100644 maturin/import_hook/_file_lock.py diff --git a/guide/src/develop.md b/guide/src/develop.md index 8756a3a51..13bdf7194 100644 --- a/guide/src/develop.md +++ b/guide/src/develop.md @@ -108,24 +108,35 @@ requires = ["maturin>=1.0,<2.0"] build-backend = "maturin" ``` -Editable installs right now is only useful in mixed Rust/Python project so you -don't have to recompile and reinstall when only Python source code changes. For -example when using pip you can make an editable installation with +Editable installs can be used with mixed Rust/Python projects so you +don't have to recompile and reinstall when only Python source code changes. +They can also be used with mixed and pure projects together with the +[import hook](#import-hook) so that recompilation/re-installation occurs +automatically when Python or Rust source code changes. + +To install a package in editable mode with pip: ```bash +cd my-project pip install -e . ``` -Then Python source code changes will take effect immediately. +Then Python source code changes will take effect immediately because the interpreter looks +for the modules directly in the project source tree. ## Import Hook Starting from v0.12.4, the [Python maturin package](https://pypi.org/project/maturin/) provides a Python import hook to allow quickly build and load a Rust module into Python. +This makes development much more convenient as it brings the workflow of +developing Rust modules closer to the workflow of developing regular python modules. It supports importing editable-installed pure Rust and mixed Rust/Python project layouts as well as importing standalone `.rs` files. +> **Note**: you must install maturin with the import-hook extra to be +> able to use the import hook: `pip install maturin[import-hook]` + ```python from maturin import import_hook diff --git a/maturin/import_hook/_building.py b/maturin/import_hook/_building.py index 08d55973b..f633f4a86 100644 --- a/maturin/import_hook/_building.py +++ b/maturin/import_hook/_building.py @@ -13,10 +13,18 @@ from pathlib import Path from typing import Generator, List, Optional, Tuple -from maturin.import_hook._file_lock import FileLock from maturin.import_hook._logging import logger from maturin.import_hook.settings import MaturinSettings +try: + import filelock +except ImportError: + msg = ( + "could not find 'filelock' package. " + "Try installing maturin with the import-hook extra: pip install maturin[import-hook]" + ) + raise ImportError(msg) from None + @dataclass class BuildStatus: @@ -85,16 +93,27 @@ def __init__( self._build_dir = ( build_dir if build_dir is not None else _get_default_build_dir() ) - self._lock = FileLock.new( - self._build_dir / "lock", timeout_seconds=lock_timeout_seconds + self._lock = filelock.FileLock( + self._build_dir / "lock", timeout=lock_timeout_seconds ) @contextmanager def lock(self) -> Generator[LockedBuildCache, None, None]: - with self._lock: + with _acquire_lock(self._lock): yield LockedBuildCache(self._build_dir) +@contextmanager +def _acquire_lock(lock: filelock.FileLock) -> Generator[None, None, None]: + try: + with lock.acquire(blocking=False): + yield + except filelock.Timeout: + logger.info("waiting on lock %s", lock.lock_file) + with lock.acquire(): + yield + + def _get_default_build_dir() -> Path: build_dir = os.environ.get("MATURIN_BUILD_DIR", None) if build_dir: diff --git a/maturin/import_hook/_file_lock.py b/maturin/import_hook/_file_lock.py deleted file mode 100644 index 196beb4fd..000000000 --- a/maturin/import_hook/_file_lock.py +++ /dev/null @@ -1,192 +0,0 @@ -import contextlib -import errno -import os -import platform -import time -from abc import ABC, abstractmethod -from pathlib import Path -from types import ModuleType, TracebackType -from typing import Optional, Type - -from maturin.import_hook._logging import logger - -fcntl: Optional[ModuleType] = None -with contextlib.suppress(ImportError): - import fcntl - - -msvcrt: Optional[ModuleType] = None -with contextlib.suppress(ImportError): - import msvcrt - - -class LockError(Exception): - pass - - -class FileLock(ABC): - def __init__( - self, path: Path, timeout_seconds: Optional[float], poll_interval: float = 0.05 - ) -> None: - self._path = path - self._timeout_seconds = timeout_seconds - self._poll_interval = poll_interval - self._is_locked = False - - @property - def is_locked(self) -> bool: - return self._is_locked - - def acquire(self) -> None: - if self._is_locked: - msg = f"{type(self).__name__} is not reentrant" - raise LockError(msg) - start = time.time() - first_attempt = True - while True: - self.try_acquire() - if self._is_locked: - return - if first_attempt: - logger.info("waiting on lock %s (%s)", self._path, type(self).__name__) - first_attempt = False - - if ( - self._timeout_seconds is not None - and time.time() - start > self._timeout_seconds - ): - msg = f"failed to acquire lock {self._path} in time" - raise TimeoutError(msg) - else: - time.sleep(self._poll_interval) - - @abstractmethod - def try_acquire(self) -> None: - raise NotImplementedError - - @abstractmethod - def release(self) -> None: - raise NotImplementedError - - def __enter__(self) -> None: - self.acquire() - - def __exit__( - self, - exc_type: Optional[Type[BaseException]], - exc_val: Optional[BaseException], - exc_tb: Optional[TracebackType], - ) -> None: - self.release() - - def __del__(self) -> None: - self.release() - - @staticmethod - def new(path: Path, timeout_seconds: Optional[float]) -> "FileLock": - if os.name == "posix": - if fcntl is None: - return AtomicOpenLock(path, timeout_seconds) - else: - return FcntlFileLock(path, timeout_seconds) - elif platform.platform().lower() == "windows": - return WindowsFileLock(path, timeout_seconds) - else: - return AtomicOpenLock(path, timeout_seconds) - - -class FcntlFileLock(FileLock): - def __init__(self, path: Path, timeout_seconds: Optional[float]) -> None: - super().__init__(path, timeout_seconds) - self._path.parent.mkdir(parents=True, exist_ok=True) - self._fd = os.open(self._path, os.O_WRONLY | os.O_CREAT) - - def __del__(self) -> None: - self.release() - os.close(self._fd) - - def try_acquire(self) -> None: - if self._is_locked: - return - assert fcntl is not None - try: - fcntl.flock(self._fd, fcntl.LOCK_EX | fcntl.LOCK_NB) - except OSError as e: - if e.errno == errno.ENOSYS: - msg = "flock not supported by filesystem" - raise LockError(msg) - else: - self._is_locked = True - - def release(self) -> None: - if self._is_locked: - assert fcntl is not None - # do not remove the lock file to avoid a potential race condition where another - # process opens the file then the file gets unlinked, leaving that process with - # a handle to a dangling file, leading it to believe it holds the lock when it doesn't - fcntl.flock(self._fd, fcntl.LOCK_UN) - self._is_locked = False - - -class WindowsFileLock(FileLock): - def __init__(self, path: Path, timeout_seconds: Optional[float]) -> None: - super().__init__(path, timeout_seconds) - self._fd = os.open(self._path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC) - - def try_acquire(self) -> None: - if self._is_locked: - return - assert msvcrt is not None - try: - msvcrt.locking(self._fd, msvcrt.LK_NBLCK, 1) - except OSError as e: - if e.errno != errno.EACCES: - msg = f"failed to acquire lock: {e}" - raise LockError(msg) - else: - self._is_locked = True - - def release(self) -> None: - if self._is_locked: - assert msvcrt is not None - msvcrt.locking(self._fd, msvcrt.LK_UNLCK, 1) - self._is_locked = False - - -class AtomicOpenLock(FileLock): - """This lock should be supported on all platforms but is not as reliable as it depends - on the filesystem supporting atomic file creation [1]. - - - - [1] https://man7.org/linux/man-pages/man2/open.2.html - """ - - def __init__(self, path: Path, timeout_seconds: Optional[float]) -> None: - super().__init__(path, timeout_seconds) - self._fd: Optional[int] = None - self._is_windows = platform.platform().lower() == "windows" - - def try_acquire(self) -> None: - if self._is_locked: - return - assert self._fd is None - try: - fd = os.open(self._path, os.O_WRONLY | os.O_CREAT | os.O_EXCL) - except OSError as e: - if not ( - e.errno == errno.EEXIST - or (self._is_windows and e.errno == errno.EACCES) - ): - msg = f"failed to acquire lock: {e}" - raise LockError(msg) - else: - self._fd = fd - self._is_locked = True - - def release(self) -> None: - if self._is_locked: - assert self._fd is not None - os.close(self._fd) - self._fd = None - self._is_locked = False - self._path.unlink(missing_ok=True) diff --git a/pyproject.toml b/pyproject.toml index f07cc6148..4ab6eb035 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,9 @@ zig = [ patchelf = [ "patchelf", ] +import-hook = [ + "filelock" +] [project.urls] "Source Code" = "https://github.com/PyO3/maturin" diff --git a/tests/common/import_hook.rs b/tests/common/import_hook.rs index 4fae18585..1ed047714 100644 --- a/tests/common/import_hook.rs +++ b/tests/common/import_hook.rs @@ -22,19 +22,18 @@ pub fn test_import_hook<'a>( let (venv_dir, python) = create_virtualenv(virtualenv_name, Some(python)).unwrap(); - let pip_install_args = vec![vec!["pytest", "uniffi-bindgen", "cffi"]]; - let extras: Vec> = extra_packages.into_iter().map(|name| vec![*name]).collect(); - for args in pip_install_args.iter().chain(&extras) { + let mut packages_to_install = vec!["pytest", "uniffi-bindgen", "cffi", "filelock"]; + packages_to_install.extend(extra_packages); + for package_name in packages_to_install { if verbose { - println!("installing {:?}", &args); + println!("installing {package_name}"); } let status = Command::new(&python) - .args(["-m", "pip", "install", "--disable-pip-version-check"]) - .args(args) + .args(["-m", "pip", "install", "--disable-pip-version-check", package_name]) .status() .unwrap(); if !status.success() { - bail!("failed to install: {:?}", &args); + bail!("failed to install: {package_name}"); } } From 464210896ea1c68b2a2eedb5e0603f44b9a476ea Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 3 Sep 2023 17:13:01 +0100 Subject: [PATCH 22/57] formatting fixes --- maturin/import_hook/project_importer.py | 2 +- maturin/import_hook/rust_file_importer.py | 4 +- tests/common/import_hook.rs | 12 +- tests/import_hook/common.py | 8 +- tests/import_hook/test_project_importer.py | 2 +- tests/import_hook/test_rust_file_importer.py | 18 +-- tests/import_hook/test_utilities.py | 144 +++---------------- tests/run.rs | 20 +-- 8 files changed, 59 insertions(+), 151 deletions(-) diff --git a/maturin/import_hook/project_importer.py b/maturin/import_hook/project_importer.py index 6d3ef440b..e9a1eead3 100644 --- a/maturin/import_hook/project_importer.py +++ b/maturin/import_hook/project_importer.py @@ -75,7 +75,7 @@ def __init__( ) def get_settings(self, module_path: str, source_path: Path) -> MaturinSettings: - """this method can be overridden in subclasses to customize settings for specific projects""" + """This method can be overridden in subclasses to customize settings for specific projects.""" return ( self._settings if self._settings is not None else MaturinSettings.default() ) diff --git a/maturin/import_hook/rust_file_importer.py b/maturin/import_hook/rust_file_importer.py index 350ce16de..f258e173a 100644 --- a/maturin/import_hook/rust_file_importer.py +++ b/maturin/import_hook/rust_file_importer.py @@ -46,7 +46,7 @@ def __init__( self._show_warnings = show_warnings def get_settings(self, module_path: str, source_path: Path) -> MaturinSettings: - """this method can be overridden in subclasses to customize settings for specific projects""" + """This method can be overridden in subclasses to customize settings for specific projects.""" return ( self._settings if self._settings is not None else MaturinSettings.default() ) @@ -58,7 +58,7 @@ def generate_project_for_single_rust_file( rust_file: Path, settings: MaturinSettings, ) -> Path: - """this method can be overridden in subclasses to customize project generation""" + """This method can be overridden in subclasses to customize project generation.""" if project_dir.exists(): shutil.rmtree(project_dir) diff --git a/tests/common/import_hook.rs b/tests/common/import_hook.rs index 1ed047714..91ff7f0d1 100644 --- a/tests/common/import_hook.rs +++ b/tests/common/import_hook.rs @@ -8,7 +8,7 @@ use std::path::{Path, PathBuf}; use std::process::Command; use std::{env, fs, str, thread}; -pub fn test_import_hook<'a>( +pub fn test_import_hook( virtualenv_name: &str, test_specifier: &str, extra_packages: &[&str], @@ -29,7 +29,13 @@ pub fn test_import_hook<'a>( println!("installing {package_name}"); } let status = Command::new(&python) - .args(["-m", "pip", "install", "--disable-pip-version-check", package_name]) + .args([ + "-m", + "pip", + "install", + "--disable-pip-version-check", + package_name, + ]) .status() .unwrap(); if !status.success() { @@ -94,7 +100,7 @@ pub fn test_import_hook_parallel( test_import_hook( &virtualenv_name, &test_specifier, - &extra_packages, + extra_packages, &extra_envs_this_test, verbose, ) diff --git a/tests/import_hook/common.py b/tests/import_hook/common.py index a65ef48c4..f0fddc147 100644 --- a/tests/import_hook/common.py +++ b/tests/import_hook/common.py @@ -1,3 +1,4 @@ +import itertools import os import re import shutil @@ -6,9 +7,8 @@ import sys import tempfile import time -import itertools from pathlib import Path -from typing import List, Optional, Tuple, Iterable +from typing import Iterable, List, Optional, Tuple from maturin.import_hook.project_importer import _fix_direct_url, _load_dist_info @@ -211,7 +211,7 @@ def get_project_copy(project_dir: Path, output_path: Path) -> Path: def _get_relative_files_tracked_by_git(root: Path) -> Iterable[Path]: - """this is used to ignore built artifacts to create a clean copy""" + """This is used to ignore built artifacts to create a clean copy.""" output = subprocess.check_output( ["git", "ls-tree", "--name-only", "-z", "-r", "HEAD"], cwd=root ) @@ -246,7 +246,7 @@ def create_project_from_blank_template( def remove_ansii_escape_characters(text: str) -> str: - """Remove escape characters (eg used to color terminal output) from the given string + """Remove escape characters (eg used to color terminal output) from the given string. based on: https://stackoverflow.com/a/14693789 """ diff --git a/tests/import_hook/test_project_importer.py b/tests/import_hook/test_project_importer.py index 27095d782..51e52ede0 100644 --- a/tests/import_hook/test_project_importer.py +++ b/tests/import_hook/test_project_importer.py @@ -17,13 +17,13 @@ is_installed_correctly, log, mixed_test_crate_names, + remove_ansii_escape_characters, run_python, run_python_code, script_dir, test_crates, uninstall, with_underscores, - remove_ansii_escape_characters, ) """ diff --git a/tests/import_hook/test_rust_file_importer.py b/tests/import_hook/test_rust_file_importer.py index 940931025..eb6a77e09 100644 --- a/tests/import_hook/test_rust_file_importer.py +++ b/tests/import_hook/test_rust_file_importer.py @@ -3,16 +3,16 @@ import re import shutil from pathlib import Path -from typing import Tuple, Generator +from typing import Generator, Tuple import pytest from .common import ( log, + remove_ansii_escape_characters, run_python, script_dir, test_crates, - remove_ansii_escape_characters, ) """ @@ -52,7 +52,7 @@ def workspace(tmp_path: Path) -> Generator[Path, None, None]: def test_absolute_import(workspace: Path) -> None: - """test imports of the form `import ab.cd.ef`""" + """Test imports of the form `import ab.cd.ef`.""" _clear_build_cache() helper_path = script_dir / "rust_file_import/absolute_import_helper.py" @@ -71,7 +71,7 @@ def test_absolute_import(workspace: Path) -> None: def test_relative_import() -> None: - """test imports of the form `from .ab import cd`""" + """Test imports of the form `from .ab import cd`.""" _clear_build_cache() output1, duration1 = run_python( @@ -92,7 +92,7 @@ def test_relative_import() -> None: def test_top_level_import(workspace: Path) -> None: - """test imports of the form `import ab`""" + """Test imports of the form `import ab`.""" _clear_build_cache() helper_path = script_dir / "rust_file_import/packages/top_level_import_helper.py" @@ -111,7 +111,7 @@ def test_top_level_import(workspace: Path) -> None: def test_multiple_imports(workspace: Path) -> None: - """test importing the same rs file multiple times by different names in the same session""" + """Test importing the same rs file multiple times by different names in the same session.""" _clear_build_cache() helper_path = script_dir / "rust_file_import/multiple_import_helper.py" @@ -123,7 +123,7 @@ def test_multiple_imports(workspace: Path) -> None: def test_concurrent_import() -> None: - """test multiple processes attempting to import the same modules at the same time""" + """Test multiple processes attempting to import the same modules at the same time.""" _clear_build_cache() args = { "args": ["rust_file_import/concurrent_import_helper.py"], @@ -166,7 +166,7 @@ def test_concurrent_import() -> None: def test_rebuild_on_change(workspace: Path) -> None: - """test that modules are rebuilt if they are edited""" + """Test that modules are rebuilt if they are edited.""" _clear_build_cache() script_path = workspace / "my_script.rs" @@ -196,7 +196,7 @@ def test_rebuild_on_change(workspace: Path) -> None: def test_rebuild_on_settings_change(workspace: Path) -> None: - """test that modules are rebuilt if the settings (eg maturin flags) used by the import hook changes""" + """Test that modules are rebuilt if the settings (eg maturin flags) used by the import hook changes.""" _clear_build_cache() script_path = workspace / "my_script.rs" diff --git a/tests/import_hook/test_utilities.py b/tests/import_hook/test_utilities.py index 77783a97e..9e0c3e2f2 100644 --- a/tests/import_hook/test_utilities.py +++ b/tests/import_hook/test_utilities.py @@ -1,11 +1,7 @@ import json import os -import random import shutil -import string -import threading import time -from concurrent.futures import ProcessPoolExecutor from operator import itemgetter from pathlib import Path from typing import Any, Dict, List, Optional @@ -14,14 +10,14 @@ from maturin.import_hook import MaturinSettings from maturin.import_hook._building import BuildCache, BuildStatus -from maturin.import_hook._file_lock import AtomicOpenLock, FileLock from maturin.import_hook._resolve_project import ProjectResolveError, _resolve_project from maturin.import_hook.project_importer import ( _get_installed_package_mtime, _get_project_mtime, _load_dist_info, ) -from maturin.import_hook.settings import MaturinDevelopSettings, MaturinBuildSettings +from maturin.import_hook.settings import MaturinBuildSettings, MaturinDevelopSettings + from .common import log, test_crates # set this to be able to run these tests without going through run.rs each time @@ -60,27 +56,27 @@ def test_settings() -> None: ) # fmt: off assert settings.to_args() == [ - '--release', - '--strip', - '--quiet', - '--jobs', '1', - '--profile', 'profile1', - '--features', 'feature1,feature2', - '--all-features', - '--no-default-features', - '--target', 'target1', - '--ignore-rust-version', - '--color', 'always', - '--frozen', - '--locked', - '--offline', - '--config', 'key1=value1', - '--config', 'key2=value2', - '-Z', 'unstable1', - '-Z', 'unstable2', - '-vv', - 'flag1', - 'flag2', + "--release", + "--strip", + "--quiet", + "--jobs", "1", + "--profile", "profile1", + "--features", "feature1,feature2", + "--all-features", + "--no-default-features", + "--target", "target1", + "--ignore-rust-version", + "--color", "always", + "--frozen", + "--locked", + "--offline", + "--config", "key1=value1", + "--config", "key2=value2", + "-Z", "unstable1", + "-Z", "unstable2", + "-vv", + "flag1", + "flag2", ] # fmt: on @@ -113,100 +109,6 @@ def test_settings() -> None: ] -class TestFileLock: - @staticmethod - def _create_lock(path: Path, timeout_seconds: float, fallback: bool) -> FileLock: - if fallback: - return AtomicOpenLock(path, timeout_seconds=timeout_seconds) - else: - return FileLock.new(path, timeout_seconds=timeout_seconds) - - @staticmethod - def _random_string(size: int = 1000) -> str: - return "".join(random.choice(string.ascii_lowercase) for _ in range(size)) - - @staticmethod - def _unlocked_worker(work_dir: Path) -> str: - path = work_dir / "my_file.txt" - data = TestFileLock._random_string() - for _ in range(10): - path.write_text(data) - time.sleep(0.001) - assert path.read_text() == data - return "SUCCESS" - - @staticmethod - def _locked_worker(work_dir: Path, use_fallback_lock: bool) -> str: - path = work_dir / "my_file.txt" - lock = TestFileLock._create_lock(work_dir / "lock", 10, use_fallback_lock) - data = TestFileLock._random_string() - for _ in range(10): - with lock: - path.write_text(data) - time.sleep(0.001) - assert path.read_text() == data - return "SUCCESS" - - @pytest.mark.parametrize("use_fallback_lock", [False, True]) - def test_lock_unlock(self, tmp_path: Path, use_fallback_lock: bool) -> None: - lock = self._create_lock(tmp_path / "lock", 5, use_fallback_lock) - - assert not lock.is_locked - for _i in range(2): - with lock: - assert lock.is_locked - assert not lock.is_locked - - @pytest.mark.parametrize("use_fallback_lock", [False, True]) - def test_timeout(self, tmp_path: Path, use_fallback_lock: bool) -> None: - lock_path = tmp_path / "lock" - lock1 = self._create_lock(lock_path, 5, use_fallback_lock) - with lock1: - lock2 = self._create_lock(lock_path, 0.1, use_fallback_lock) - with pytest.raises(TimeoutError): - lock2.acquire() - - @pytest.mark.parametrize("use_fallback_lock", [False, True]) - def test_waiting(self, tmp_path: Path, use_fallback_lock: bool) -> None: - lock_path = tmp_path / "lock" - lock1 = self._create_lock(lock_path, 5, use_fallback_lock) - lock2 = self._create_lock(lock_path, 5, use_fallback_lock) - - lock1.acquire() - t = threading.Timer(0.2, lock1.release) - t.start() - lock2.acquire() - lock2.release() - - @pytest.mark.parametrize("use_fallback_lock", [False, True]) - def test_concurrent_access(self, tmp_path: Path, use_fallback_lock: bool) -> None: - num_workers = 25 - num_threads = 4 - - working_dir = tmp_path / "unlocked" - working_dir.mkdir() - with ProcessPoolExecutor(max_workers=num_threads) as executor: - futures = [ - executor.submit(TestFileLock._unlocked_worker, working_dir) - for _ in range(num_workers) - ] - with pytest.raises(AssertionError): - for f in futures: - f.result() - - working_dir = tmp_path / "locked" - working_dir.mkdir() - with ProcessPoolExecutor(max_workers=num_threads) as executor: - futures = [ - executor.submit( - TestFileLock._locked_worker, working_dir, use_fallback_lock - ) - for _ in range(num_workers) - ] - for f in futures: - assert f.result() == "SUCCESS" - - class TestGetProjectMtime: def test_missing_extension(self, tmp_path: Path) -> None: assert _get_project_mtime(tmp_path, [], tmp_path / "missing", set()) is None diff --git a/tests/run.rs b/tests/run.rs index d3c3dcdf8..6476fabaf 100644 --- a/tests/run.rs +++ b/tests/run.rs @@ -774,8 +774,8 @@ fn import_hook_project_importer() { handle_result(import_hook::test_import_hook( "import_hook_project_importer", "tests/import_hook/test_project_importer.py", - &vec!["boltons"], - &vec![("MATURIN_TEST_NAME", "ALL")], + &["boltons"], + &[("MATURIN_TEST_NAME", "ALL")], true, )); } @@ -785,8 +785,8 @@ fn import_hook_project_importer_parallel() { handle_result(import_hook::test_import_hook_parallel( "import_hook_project_importer", &PathBuf::from("tests/import_hook/test_project_importer.py"), - &vec!["boltons"], - &vec![], + &["boltons"], + &[], true, )); } @@ -797,8 +797,8 @@ fn import_hook_rust_file_importer() { handle_result(import_hook::test_import_hook( "import_hook_rust_file_importer", "tests/import_hook/test_rust_file_importer.py", - &vec![], - &vec![("MATURIN_TEST_NAME", "ALL")], + &[], + &[("MATURIN_TEST_NAME", "ALL")], true, )); } @@ -808,8 +808,8 @@ fn import_hook_rust_file_importer_parallel() { handle_result(import_hook::test_import_hook_parallel( "import_hook_rust_file_importer", &PathBuf::from("tests/import_hook/test_rust_file_importer.py"), - &vec![], - &vec![], + &[], + &[], true, )); } @@ -826,8 +826,8 @@ fn import_hook_utilities() { handle_result(import_hook::test_import_hook( "import_hook_utilities", "tests/import_hook/test_utilities.py", - &vec![], - &vec![( + &[], + &[( "RESOLVED_PACKAGES_PATH", resolved_packages_path.to_str().unwrap(), )], From f761aa2bd87a663a934607b3e1a67e6cc9fa3f0a Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 3 Sep 2023 21:25:34 +0100 Subject: [PATCH 23/57] run mypy checks on the python scripts in tests/ as well --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 5c422829a..1e28bc5a8 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -67,7 +67,7 @@ jobs: with: python-version: '3.x' - run: pip install mypy - - run: mypy maturin + - run: mypy maturin/ tests/ spellcheck: name: Spellcheck From ad0cd5044eca836238c29174b14092ee230d2e2a Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 3 Sep 2023 21:48:49 +0100 Subject: [PATCH 24/57] fixed python 3.7 compatibility --- maturin/import_hook/_building.py | 2 +- maturin/import_hook/_resolve_project.py | 2 +- maturin/import_hook/project_importer.py | 6 +++--- tests/import_hook/common.py | 6 +++--- tests/run.rs | 3 +-- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/maturin/import_hook/_building.py b/maturin/import_hook/_building.py index f633f4a86..58b1e1ad3 100644 --- a/maturin/import_hook/_building.py +++ b/maturin/import_hook/_building.py @@ -192,7 +192,7 @@ def develop_build_project( return output -def run_maturin(args: list[str]) -> Tuple[bool, str]: +def run_maturin(args: List[str]) -> Tuple[bool, str]: maturin_path = shutil.which("maturin") if maturin_path is None: msg = "maturin not found in the PATH" diff --git a/maturin/import_hook/_resolve_project.py b/maturin/import_hook/_resolve_project.py index 60d56939a..2b061cc63 100644 --- a/maturin/import_hook/_resolve_project.py +++ b/maturin/import_hook/_resolve_project.py @@ -197,7 +197,7 @@ def _resolve_module_name( def _get_immediate_path_dependencies( project_dir: Path, cargo: Dict[str, Any] -) -> list[Path]: +) -> List[Path]: path_dependencies = [] for dependency in cargo.get("dependencies", {}).values(): if isinstance(dependency, dict): diff --git a/maturin/import_hook/project_importer.py b/maturin/import_hook/project_importer.py index e9a1eead3..8aaf7f4c3 100644 --- a/maturin/import_hook/project_importer.py +++ b/maturin/import_hook/project_importer.py @@ -11,7 +11,7 @@ from importlib.machinery import ModuleSpec, PathFinder from pathlib import Path from types import ModuleType -from typing import Iterable, Optional, Sequence, Set, Tuple, Union +from typing import Iterable, Optional, Sequence, Set, Tuple, Union, List from maturin.import_hook._building import ( BuildCache, @@ -439,7 +439,7 @@ def _get_installed_package_mtime( def _get_project_mtime( project_dir: Path, - all_path_dependencies: list[Path], + all_path_dependencies: List[Path], installed_package_root: Path, excluded_dir_names: Set[str], ) -> Optional[float]: @@ -463,7 +463,7 @@ def _get_project_mtime( def _package_is_up_to_date( project_dir: Path, - all_path_dependencies: list[Path], + all_path_dependencies: List[Path], installed_package_root: Path, installed_package_mtime: float, excluded_dir_names: Set[str], diff --git a/tests/import_hook/common.py b/tests/import_hook/common.py index f0fddc147..2486e3521 100644 --- a/tests/import_hook/common.py +++ b/tests/import_hook/common.py @@ -40,7 +40,7 @@ def with_underscores(project_name: str) -> str: return project_name.replace("-", "_") -def all_test_crate_names() -> list[str]: +def all_test_crate_names() -> List[str]: return sorted( p.name for p in test_crates.iterdir() @@ -50,7 +50,7 @@ def all_test_crate_names() -> list[str]: ) -def mixed_test_crate_names() -> list[str]: +def mixed_test_crate_names() -> List[str]: return [name for name in all_test_crate_names() if "mixed" in name] @@ -115,7 +115,7 @@ def run_python_code( *, args: Optional[List[str]] = None, cwd: Optional[Path] = None, - python_path: Optional[list[Path]] = None, + python_path: Optional[List[Path]] = None, quiet: bool = False, expect_error: bool = False, ) -> Tuple[str, float]: diff --git a/tests/run.rs b/tests/run.rs index 84d8933c7..3bdc53780 100644 --- a/tests/run.rs +++ b/tests/run.rs @@ -3,14 +3,13 @@ use std::path::{Path, PathBuf}; use std::{env, fs}; -use indoc::indoc; +use expect_test::expect; use time::macros::datetime; use which::which; use common::{ develop, errors, get_python_implementation, handle_result, integration, other, test_python_path, }; -use expect_test::expect; use maturin::pyproject_toml::SdistGenerator; use maturin::Target; From 95671b50d9c0018488dc6d0a768b6d90a5072c7e Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Tue, 24 Oct 2023 20:49:25 +0100 Subject: [PATCH 25/57] added notes on running the tests --- tests/README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 tests/README.md diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 000000000..eda6ccb6d --- /dev/null +++ b/tests/README.md @@ -0,0 +1,16 @@ +# Test Running Notes +- `virtualenv` is required to be in the PATH when running the tests +- intermediate test artifacts are written to `test-crates/targets` and `test-crates/venvs`. + keeping them may speed up running the tests again, but you may also remove them once the tests have finished. +- the import hook tests cannot easily be run outside the test runner. +- to run a single import hook test, modify the test runner in `run.rs` to specify a single test instead of a whole module. For example: + +```rust +handle_result(import_hook::test_import_hook( + "import_hook_rust_file_importer", + "tests/import_hook/test_rust_file_importer.py::test_multiple_imports", // <-- + &[], + &[("MATURIN_TEST_NAME", "ALL")], + true, +)); +``` From b4da2db91a6f1302a33c25a0f898b1a8fa9c4f39 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Tue, 24 Oct 2023 20:50:27 +0100 Subject: [PATCH 26/57] added utility to print more details when an error occurs in a worker process --- tests/import_hook/common.py | 22 +++++++++++++++++++- tests/import_hook/test_project_importer.py | 12 ++++++++--- tests/import_hook/test_rust_file_importer.py | 12 ++++++++--- 3 files changed, 39 insertions(+), 7 deletions(-) diff --git a/tests/import_hook/common.py b/tests/import_hook/common.py index 2486e3521..16bcbd0ce 100644 --- a/tests/import_hook/common.py +++ b/tests/import_hook/common.py @@ -7,8 +7,9 @@ import sys import tempfile import time +from contextlib import contextmanager from pathlib import Path -from typing import Iterable, List, Optional, Tuple +from typing import Iterable, List, Optional, Tuple, Generator from maturin.import_hook.project_importer import _fix_direct_url, _load_dist_info @@ -251,3 +252,22 @@ def remove_ansii_escape_characters(text: str) -> str: based on: https://stackoverflow.com/a/14693789 """ return re.sub(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", text) + + +@contextmanager +def handle_worker_process_error() -> Generator[None, None, None]: + """For some reason, catching and printing the exception output inside the + worker process does not appear in the pytest output, so catch and print in the main process + """ + try: + yield + except subprocess.CalledProcessError as e: + stdout = None if e.stdout is None else e.stdout.decode() + stderr = None if e.stderr is None else e.stderr.decode() + print("Error in worker process") + print("-" * 50) + print(f"Stdout:\n{stdout}") + print("-" * 50) + print(f"Stderr:\n{stderr}") + print("-" * 50) + raise diff --git a/tests/import_hook/test_project_importer.py b/tests/import_hook/test_project_importer.py index 51e52ede0..62f7bf9a3 100644 --- a/tests/import_hook/test_project_importer.py +++ b/tests/import_hook/test_project_importer.py @@ -24,6 +24,7 @@ test_crates, uninstall, with_underscores, + handle_worker_process_error, ) """ @@ -396,9 +397,14 @@ def test_concurrent_import(workspace: Path, initially_mixed: bool, mixed: bool) p2 = pool.apply_async(run_python_code, kwds=args) p3 = pool.apply_async(run_python_code, kwds=args) - output_1, duration_1 = p1.get() - output_2, duration_2 = p2.get() - output_3, duration_3 = p3.get() + with handle_worker_process_error(): + output_1, duration_1 = p1.get() + + with handle_worker_process_error(): + output_2, duration_2 = p2.get() + + with handle_worker_process_error(): + output_3, duration_3 = p3.get() log("output 1") log(output_1) diff --git a/tests/import_hook/test_rust_file_importer.py b/tests/import_hook/test_rust_file_importer.py index eb6a77e09..bbd3d344b 100644 --- a/tests/import_hook/test_rust_file_importer.py +++ b/tests/import_hook/test_rust_file_importer.py @@ -13,6 +13,7 @@ run_python, script_dir, test_crates, + handle_worker_process_error, ) """ @@ -136,9 +137,14 @@ def test_concurrent_import() -> None: p2 = pool.apply_async(run_python, kwds=args) p3 = pool.apply_async(run_python, kwds=args) - output_1, duration_1 = p1.get() - output_2, duration_2 = p2.get() - output_3, duration_3 = p3.get() + with handle_worker_process_error(): + output_1, duration_1 = p1.get() + + with handle_worker_process_error(): + output_2, duration_2 = p2.get() + + with handle_worker_process_error(): + output_3, duration_3 = p3.get() log("output 1") log(output_1) From a2155bc72d91152fbd03682ed74608c6f8fa9344 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Tue, 24 Oct 2023 21:04:35 +0100 Subject: [PATCH 27/57] provide more output of which tests passed --- tests/common/import_hook.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/common/import_hook.rs b/tests/common/import_hook.rs index 91ff7f0d1..e14ae78a8 100644 --- a/tests/common/import_hook.rs +++ b/tests/common/import_hook.rs @@ -106,10 +106,11 @@ pub fn test_import_hook_parallel( ) .unwrap() }); - handles.push(handle); + handles.push((function_name, handle)); } - for handle in handles { + for (function_name, handle) in handles { handle.join().unwrap(); + println!("test {function_name}: passed") } }); Ok(()) From 88153df6706f83a4bfb9bdbc97fb8b4cd0d5b293 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sat, 18 Nov 2023 15:38:38 +0000 Subject: [PATCH 28/57] linting --- maturin/import_hook/_building.py | 46 ++---- maturin/import_hook/_resolve_project.py | 27 +--- maturin/import_hook/project_importer.py | 89 +++--------- maturin/import_hook/rust_file_importer.py | 56 ++------ tests/import_hook/common.py | 49 ++----- tests/import_hook/test_project_importer.py | 141 +++++-------------- tests/import_hook/test_rust_file_importer.py | 49 ++----- tests/import_hook/test_utilities.py | 47 ++----- 8 files changed, 122 insertions(+), 382 deletions(-) diff --git a/maturin/import_hook/_building.py b/maturin/import_hook/_building.py index 58b1e1ad3..834e9e702 100644 --- a/maturin/import_hook/_building.py +++ b/maturin/import_hook/_building.py @@ -87,15 +87,9 @@ def tmp_project_dir(self, project_path: Path, module_name: str) -> Path: class BuildCache: - def __init__( - self, build_dir: Optional[Path], lock_timeout_seconds: Optional[float] - ) -> None: - self._build_dir = ( - build_dir if build_dir is not None else _get_default_build_dir() - ) - self._lock = filelock.FileLock( - self._build_dir / "lock", timeout=lock_timeout_seconds - ) + def __init__(self, build_dir: Optional[Path], lock_timeout_seconds: Optional[float]) -> None: + self._build_dir = build_dir if build_dir is not None else _get_default_build_dir() + self._lock = filelock.FileLock(self._build_dir / "lock", timeout=lock_timeout_seconds) @contextmanager def lock(self) -> Generator[LockedBuildCache, None, None]: @@ -133,16 +127,10 @@ def _get_cache_dir() -> Path: return Path("~/Library/Caches").expanduser() else: xdg_cache_dir = os.environ.get("XDG_CACHE_HOME", None) - return ( - Path(xdg_cache_dir) if xdg_cache_dir else Path("~/.cache").expanduser() - ) + return Path(xdg_cache_dir) if xdg_cache_dir else Path("~/.cache").expanduser() elif platform.platform().lower() == "windows": local_app_data = os.environ.get("LOCALAPPDATA", None) - return ( - Path(local_app_data) - if local_app_data - else Path(r"~\AppData\Local").expanduser() - ) + return Path(local_app_data) if local_app_data else Path(r"~\AppData\Local").expanduser() else: logger.warning("unknown OS. defaulting to ~/.cache as the cache directory") return Path("~/.cache").expanduser() @@ -179,13 +167,9 @@ def develop_build_project( settings: MaturinSettings, ) -> str: if "develop" not in settings.supported_commands(): - msg = ( - f'provided {type(settings).__name__} does not support the "develop" command' - ) + msg = f'provided {type(settings).__name__} does not support the "develop" command' raise ImportError(msg) - success, output = run_maturin( - ["develop", "--manifest-path", str(manifest_path), *settings.to_args()] - ) + success, output = run_maturin(["develop", "--manifest-path", str(manifest_path), *settings.to_args()]) if not success: msg = "Failed to build package with maturin" raise ImportError(msg) @@ -203,9 +187,7 @@ def run_maturin(args: List[str]) -> Tuple[bool, str]: result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output = result.stdout.decode() if result.returncode != 0: - logger.error( - f'command "{subprocess.list2cmdline(command)}" returned non-zero exit status: {result.returncode}' - ) + logger.error(f'command "{subprocess.list2cmdline(command)}" returned non-zero exit status: {result.returncode}') logger.error("maturin output:\n%s", output) return False, output if logger.isEnabledFor(logging.DEBUG): @@ -217,9 +199,7 @@ def run_maturin(args: List[str]) -> Tuple[bool, str]: return True, output -def build_unpacked_wheel( - manifest_path: Path, output_dir: Path, settings: MaturinSettings -) -> str: +def build_unpacked_wheel(manifest_path: Path, output_dir: Path, settings: MaturinSettings) -> str: if output_dir.exists(): shutil.rmtree(output_dir) output = build_wheel(manifest_path, output_dir, settings) @@ -234,15 +214,11 @@ def build_unpacked_wheel( def _find_single_file(dir_path: Path, extension: Optional[str]) -> Optional[Path]: if dir_path.exists(): - candidate_files = [ - p for p in dir_path.iterdir() if extension is None or p.suffix == extension - ] + candidate_files = [p for p in dir_path.iterdir() if extension is None or p.suffix == extension] else: candidate_files = [] return candidate_files[0] if len(candidate_files) == 1 else None def maturin_output_has_warnings(output: str) -> bool: - return ( - re.search(r"`.*` \((lib|bin)\) generated [0-9]+ warnings?", output) is not None - ) + return re.search(r"`.*` \((lib|bin)\) generated [0-9]+ warnings?", output) is not None diff --git a/maturin/import_hook/_resolve_project.py b/maturin/import_hook/_resolve_project.py index 2b061cc63..5c483390d 100644 --- a/maturin/import_hook/_resolve_project.py +++ b/maturin/import_hook/_resolve_project.py @@ -23,9 +23,7 @@ def find_cargo_manifest(project_dir: Path) -> Optional[Path]: def is_maybe_maturin_project(project_dir: Path) -> bool: """note: this function does not check if this really is a maturin project for simplicity.""" - return (project_dir / "pyproject.toml").exists() and find_cargo_manifest( - project_dir - ) is not None + return (project_dir / "pyproject.toml").exists() and find_cargo_manifest(project_dir) is not None class ProjectResolver: @@ -78,9 +76,7 @@ def is_mixed(self) -> bool: @property def all_path_dependencies(self) -> List[Path]: if self._all_path_dependencies is None: - self._all_path_dependencies = _find_all_path_dependencies( - self.immediate_path_dependencies - ) + self._all_path_dependencies = _find_all_path_dependencies(self.immediate_path_dependencies) return self._all_path_dependencies @@ -134,9 +130,7 @@ def _resolve_project(project_dir: Path) -> MaturinProject: extension_module_dir: Optional[Path] python_module: Optional[Path] - python_module, extension_module_dir, extension_module_name = _resolve_rust_module( - python_dir, module_full_name - ) + python_module, extension_module_dir, extension_module_name = _resolve_rust_module(python_dir, module_full_name) immediate_path_dependencies = _get_immediate_path_dependencies(project_dir, cargo) if not python_module.exists(): @@ -171,9 +165,7 @@ def _resolve_rust_module(python_dir: Path, module_name: str) -> Tuple[Path, Path return python_module, extension_module_dir, extension_module_name -def _resolve_module_name( - pyproject: Dict[str, Any], cargo: Dict[str, Any] -) -> Optional[str]: +def _resolve_module_name(pyproject: Dict[str, Any], cargo: Dict[str, Any]) -> Optional[str]: """This follows the same logic as project_layout.rs (ProjectResolver::resolve). Precedence: @@ -195,9 +187,7 @@ def _resolve_module_name( return cargo.get("package", {}).get("name", None) -def _get_immediate_path_dependencies( - project_dir: Path, cargo: Dict[str, Any] -) -> List[Path]: +def _get_immediate_path_dependencies(project_dir: Path, cargo: Dict[str, Any]) -> List[Path]: path_dependencies = [] for dependency in cargo.get("dependencies", {}).values(): if isinstance(dependency, dict): @@ -218,14 +208,11 @@ def _resolve_py_root(project_dir: Path, pyproject: Dict[str, Any]) -> Path: rust_cargo_toml_found = (project_dir / "rust/Cargo.toml").exists() - python_packages = ( - pyproject.get("tool", {}).get("maturin", {}).get("python-packages", []) - ) + python_packages = pyproject.get("tool", {}).get("maturin", {}).get("python-packages", []) package_name = project_name.replace("-", "_") python_src_found = any( - (project_dir / p / "__init__.py").is_file() - for p in itertools.chain((f"src/{package_name}/",), python_packages) + (project_dir / p / "__init__.py").is_file() for p in itertools.chain((f"src/{package_name}/",), python_packages) ) if rust_cargo_toml_found and python_src_found: return project_dir / "src" diff --git a/maturin/import_hook/project_importer.py b/maturin/import_hook/project_importer.py index 8aaf7f4c3..320eb9466 100644 --- a/maturin/import_hook/project_importer.py +++ b/maturin/import_hook/project_importer.py @@ -68,17 +68,11 @@ def __init__( self._install_new_packages = install_new_packages self._force_rebuild = force_rebuild self._show_warnings = show_warnings - self._excluded_dir_names = ( - DEFAULT_EXCLUDED_DIR_NAMES - if excluded_dir_names is None - else excluded_dir_names - ) + self._excluded_dir_names = DEFAULT_EXCLUDED_DIR_NAMES if excluded_dir_names is None else excluded_dir_names def get_settings(self, module_path: str, source_path: Path) -> MaturinSettings: """This method can be overridden in subclasses to customize settings for specific projects.""" - return ( - self._settings if self._settings is not None else MaturinSettings.default() - ) + return self._settings if self._settings is not None else MaturinSettings.default() def find_spec( self, @@ -111,8 +105,7 @@ def find_spec( logger.debug('found project linked by dist-info: "%s"', project_dir) if not is_editable and not self._install_new_packages: logger.debug( - "package not installed in editable-mode " - "and install_new_packages=False. not rebuilding" + "package not installed in editable-mode " "and install_new_packages=False. not rebuilding" ) else: spec, rebuilt = self._rebuild_project(package_name, project_dir) @@ -133,9 +126,7 @@ def find_spec( if spec is not None: duration = time.perf_counter() - start if rebuilt: - logger.info( - 'rebuilt and loaded package "%s" in %.3fs', package_name, duration - ) + logger.info('rebuilt and loaded package "%s" in %.3fs', package_name, duration) else: logger.debug('loaded package "%s" in %.3fs', package_name, duration) return spec @@ -161,12 +152,9 @@ def _rebuild_project( ) return None, False - if not self._install_new_packages and not _is_editable_installed_package( - project_dir, package_name - ): + if not self._install_new_packages and not _is_editable_installed_package(project_dir, package_name): logger.debug( - 'package "%s" is not already installed and ' - "install_new_packages=False. Not importing", + 'package "%s" is not already installed and ' "install_new_packages=False. Not importing", package_name, ) return None, False @@ -180,15 +168,11 @@ def _rebuild_project( ) if spec is not None: return spec, False - logger.debug( - 'package "%s" will be rebuilt because: %s', package_name, reason - ) + logger.debug('package "%s" will be rebuilt because: %s', package_name, reason) logger.info('building "%s"', package_name) start = time.perf_counter() - maturin_output = develop_build_project( - resolved.cargo_manifest_path, settings - ) + maturin_output = develop_build_project(resolved.cargo_manifest_path, settings) _fix_direct_url(project_dir, package_name) logger.debug( 'compiled project "%s" in %.3fs', @@ -208,15 +192,11 @@ def _rebuild_project( if installed_package_root is None: logger.error("could not get installed package root") else: - mtime = _get_installed_package_mtime( - installed_package_root, self._excluded_dir_names - ) + mtime = _get_installed_package_mtime(installed_package_root, self._excluded_dir_names) if mtime is None: logger.error("could not get installed package mtime") else: - build_status = BuildStatus( - mtime, project_dir, settings.to_args(), maturin_output - ) + build_status = BuildStatus(mtime, project_dir, settings.to_args(), maturin_output) build_cache.store_build_status(build_status) return spec, True @@ -245,9 +225,7 @@ def _get_spec_for_up_to_date_package( if installed_package_root is None: return None, "could not find installed package root" - installed_package_mtime = _get_installed_package_mtime( - installed_package_root, self._excluded_dir_names - ) + installed_package_mtime = _get_installed_package_mtime(installed_package_root, self._excluded_dir_names) if installed_package_mtime is None: return None, "could not get installed package mtime" @@ -272,18 +250,12 @@ def _get_spec_for_up_to_date_package( logger.debug('package up to date: "%s" ("%s")', package_name, spec.origin) - if self._show_warnings and maturin_output_has_warnings( - build_status.maturin_output - ): - self._log_build_warnings( - package_name, build_status.maturin_output, is_fresh=False - ) + if self._show_warnings and maturin_output_has_warnings(build_status.maturin_output): + self._log_build_warnings(package_name, build_status.maturin_output, is_fresh=False) return spec, None - def _log_build_warnings( - self, module_path: str, maturin_output: str, is_fresh: bool - ) -> None: + def _log_build_warnings(self, module_path: str, maturin_output: str, is_fresh: bool) -> None: prefix = "" if is_fresh else "the last " message = '%sbuild of "%s" succeeded with warnings:\n%s' if self._show_warnings: @@ -392,9 +364,7 @@ def _fix_direct_url(project_dir: Path, package_name: str) -> None: return -def _find_installed_package_root( - resolved: MaturinProject, package_spec: ModuleSpec -) -> Optional[Path]: +def _find_installed_package_root(resolved: MaturinProject, package_spec: ModuleSpec) -> Optional[Path]: """Find the root of the files that change each time the project is rebuilt: - for mixed projects: the root directory or file of the extension module inside the source tree - for pure projects: the root directory of the installed package. @@ -404,9 +374,7 @@ def _find_installed_package_root( resolved.extension_module_dir, resolved.module_name, require=False ) if installed_package_root is None: - logger.debug( - 'no extension module found in "%s"', resolved.extension_module_dir - ) + logger.debug('no extension module found in "%s"', resolved.extension_module_dir) return installed_package_root elif package_spec.origin is not None: return Path(package_spec.origin).parent @@ -415,16 +383,12 @@ def _find_installed_package_root( return None -def _get_installed_package_mtime( - installed_package_root: Path, excluded_dir_names: Set[str] -) -> Optional[float]: +def _get_installed_package_mtime(installed_package_root: Path, excluded_dir_names: Set[str]) -> Optional[float]: if installed_package_root.is_dir(): try: return min( path.stat().st_mtime - for path in _get_files_in_dirs( - (installed_package_root,), excluded_dir_names, set() - ) + for path in _get_files_in_dirs((installed_package_root,), excluded_dir_names, set()) ) except ValueError: logger.debug('no installed files found in "%s"', installed_package_root) @@ -468,9 +432,7 @@ def _package_is_up_to_date( installed_package_mtime: float, excluded_dir_names: Set[str], ) -> bool: - project_mtime = _get_project_mtime( - project_dir, all_path_dependencies, installed_package_root, excluded_dir_names - ) + project_mtime = _get_project_mtime(project_dir, all_path_dependencies, installed_package_root, excluded_dir_names) if project_mtime is None: return False @@ -483,9 +445,7 @@ def _package_is_up_to_date( return installed_package_mtime >= project_mtime -def _find_extension_module( - dir_path: Path, module_name: str, *, require: bool = False -) -> Optional[Path]: +def _find_extension_module(dir_path: Path, module_name: str, *, require: bool = False) -> Optional[Path]: if (dir_path / module_name / "__init__.py").exists(): return dir_path / module_name @@ -508,13 +468,8 @@ def _get_files_in_dirs( for dir_path in dir_paths: for path in dir_path.iterdir(): if path.is_dir(): - if ( - path.name not in excluded_dir_names - and path not in excluded_dir_paths - ): - yield from _get_files_in_dirs( - (path,), excluded_dir_names, excluded_dir_paths - ) + if path.name not in excluded_dir_names and path not in excluded_dir_paths: + yield from _get_files_in_dirs((path,), excluded_dir_names, excluded_dir_paths) else: yield path diff --git a/maturin/import_hook/rust_file_importer.py b/maturin/import_hook/rust_file_importer.py index f258e173a..3014402e2 100644 --- a/maturin/import_hook/rust_file_importer.py +++ b/maturin/import_hook/rust_file_importer.py @@ -47,9 +47,7 @@ def __init__( def get_settings(self, module_path: str, source_path: Path) -> MaturinSettings: """This method can be overridden in subclasses to customize settings for specific projects.""" - return ( - self._settings if self._settings is not None else MaturinSettings.default() - ) + return self._settings if self._settings is not None else MaturinSettings.default() @staticmethod def generate_project_for_single_rust_file( @@ -68,9 +66,7 @@ def generate_project_for_single_rust_file( raise ImportError(msg) if settings.features is not None: - available_features = [ - feature for feature in settings.features if "/" not in feature - ] + available_features = [feature for feature in settings.features if "/" not in feature] cargo_manifest = project_dir / "Cargo.toml" cargo_manifest.write_text( "{}\n[features]\n{}".format( @@ -110,18 +106,14 @@ def find_spec( for search_path in search_paths: single_rust_file_path = search_path / f"{module_name}.rs" if single_rust_file_path.is_file(): - spec, rebuilt = self._import_rust_file( - fullname, module_name, single_rust_file_path - ) + spec, rebuilt = self._import_rust_file(fullname, module_name, single_rust_file_path) if spec is not None: break if spec is not None: duration = time.perf_counter() - start if rebuilt: - logger.info( - 'rebuilt and loaded module "%s" in %.3fs', fullname, duration - ) + logger.info('rebuilt and loaded module "%s" in %.3fs', fullname, duration) else: logger.debug('loaded module "%s" in %.3fs', fullname, duration) return spec @@ -153,9 +145,7 @@ def _import_rust_file( ) manifest_path = find_cargo_manifest(project_dir) if manifest_path is None: - msg = ( - f"cargo manifest not found in the project generated for {file_path}" - ) + msg = f"cargo manifest not found in the project generated for {file_path}" raise ImportError(msg) maturin_output = build_unpacked_wheel(manifest_path, dist_dir, settings) @@ -167,13 +157,9 @@ def _import_rust_file( if self._show_warnings and maturin_output_has_warnings(maturin_output): self._log_build_warnings(module_path, maturin_output, is_fresh=True) - extension_module_path = _find_extension_module( - dist_dir / module_name, module_name, require=True - ) + extension_module_path = _find_extension_module(dist_dir / module_name, module_name, require=True) if extension_module_path is None: - logger.error( - 'cannot find extension module for "%s" after rebuild', module_path - ) + logger.error('cannot find extension module for "%s" after rebuild', module_path) return None, True build_status = BuildStatus( extension_module_path.stat().st_mtime, @@ -203,9 +189,7 @@ def _get_spec_for_up_to_date_extension_module( if self._force_rebuild: return None, "forcing rebuild" - extension_module_path = _find_extension_module( - search_dir, module_name, require=False - ) + extension_module_path = _find_extension_module(search_dir, module_name, require=False) if extension_module_path is None: return None, "already built module not found" @@ -229,18 +213,12 @@ def _get_spec_for_up_to_date_extension_module( logger.debug('module up to date: "%s" (%s)', module_path, spec.origin) - if self._show_warnings and maturin_output_has_warnings( - build_status.maturin_output - ): - self._log_build_warnings( - module_path, build_status.maturin_output, is_fresh=False - ) + if self._show_warnings and maturin_output_has_warnings(build_status.maturin_output): + self._log_build_warnings(module_path, build_status.maturin_output, is_fresh=False) return spec, None - def _log_build_warnings( - self, module_path: str, maturin_output: str, is_fresh: bool - ) -> None: + def _log_build_warnings(self, module_path: str, maturin_output: str, is_fresh: bool) -> None: prefix = "" if is_fresh else "the last " message = '%sbuild of "%s" succeeded with warnings:\n%s' if self._show_warnings: @@ -249,9 +227,7 @@ def _log_build_warnings( logger.debug(message, prefix, module_path, maturin_output) -def _find_extension_module( - dir_path: Path, module_name: str, *, require: bool = False -) -> Optional[Path]: +def _find_extension_module(dir_path: Path, module_name: str, *, require: bool = False) -> Optional[Path]: # the suffixes include the platform tag and file extension eg '.cpython-311-x86_64-linux-gnu.so' for suffix in importlib.machinery.EXTENSION_SUFFIXES: extension_path = dir_path / f"{module_name}{suffix}" @@ -263,12 +239,8 @@ def _find_extension_module( return None -def _get_spec_for_extension_module( - module_path: str, extension_module_path: Path -) -> Optional[ModuleSpec]: - return importlib.util.spec_from_loader( - module_path, ExtensionFileLoader(module_path, str(extension_module_path)) - ) +def _get_spec_for_extension_module(module_path: str, extension_module_path: Path) -> Optional[ModuleSpec]: + return importlib.util.spec_from_loader(module_path, ExtensionFileLoader(module_path, str(extension_module_path))) IMPORTER: Optional[MaturinRustFileImporter] = None diff --git a/tests/import_hook/common.py b/tests/import_hook/common.py index 16bcbd0ce..999f52cd2 100644 --- a/tests/import_hook/common.py +++ b/tests/import_hook/common.py @@ -45,8 +45,7 @@ def all_test_crate_names() -> List[str]: return sorted( p.name for p in test_crates.iterdir() - if (p / "check_installed/check_installed.py").exists() - and (p / "pyproject.toml").exists() + if (p / "check_installed/check_installed.py").exists() and (p / "pyproject.toml").exists() if p.name not in EXCLUDED_PROJECTS ) @@ -67,9 +66,7 @@ def run_python( env = os.environ if python_path is not None: - env["PYTHONPATH"] = os.pathsep.join( - str(p) for p in itertools.chain(python_path, [maturin_dir]) - ) + env["PYTHONPATH"] = os.pathsep.join(str(p) for p in itertools.chain(python_path, [maturin_dir])) else: env["PYTHONPATH"] = str(maturin_dir) @@ -145,9 +142,7 @@ def log(message: str) -> None: def uninstall(project_name: str) -> None: log(f"uninstalling {project_name}") - subprocess.check_call( - [sys.executable, "-m", "pip", "uninstall", "-y", project_name] - ) + subprocess.check_call([sys.executable, "-m", "pip", "uninstall", "-y", project_name]) def install_editable(project_dir: Path) -> None: @@ -167,14 +162,10 @@ def install_non_editable(project_dir: Path) -> None: def _is_installed_as_pth(project_name: str) -> bool: package_name = with_underscores(project_name) - return any( - (Path(path) / f"{package_name}.pth").exists() for path in site.getsitepackages() - ) + return any((Path(path) / f"{package_name}.pth").exists() for path in site.getsitepackages()) -def _is_installed_editable_with_direct_url( - project_name: str, project_dir: Path -) -> bool: +def _is_installed_editable_with_direct_url(project_name: str, project_dir: Path) -> bool: package_name = with_underscores(project_name) for path in site.getsitepackages(): linked_path, is_editable = _load_dist_info(Path(path), package_name) @@ -183,19 +174,13 @@ def _is_installed_editable_with_direct_url( log(f'project "{project_name}" is installed but not in editable mode') return is_editable else: - log( - f'found linked path "{linked_path}" for project "{project_name}". Expected "{project_dir}"' - ) + log(f'found linked path "{linked_path}" for project "{project_name}". Expected "{project_dir}"') return False -def is_installed_correctly( - project_name: str, project_dir: Path, is_mixed: bool -) -> bool: +def is_installed_correctly(project_name: str, project_dir: Path, is_mixed: bool) -> bool: installed_as_pth = _is_installed_as_pth(project_name) - installed_editable_with_direct_url = _is_installed_editable_with_direct_url( - project_name, project_dir - ) + installed_editable_with_direct_url = _is_installed_editable_with_direct_url(project_name, project_dir) log( f"checking if {project_name} is installed correctly. " f"{is_mixed=}, {installed_as_pth=} {installed_editable_with_direct_url=}" @@ -213,18 +198,14 @@ def get_project_copy(project_dir: Path, output_path: Path) -> Path: def _get_relative_files_tracked_by_git(root: Path) -> Iterable[Path]: """This is used to ignore built artifacts to create a clean copy.""" - output = subprocess.check_output( - ["git", "ls-tree", "--name-only", "-z", "-r", "HEAD"], cwd=root - ) + output = subprocess.check_output(["git", "ls-tree", "--name-only", "-z", "-r", "HEAD"], cwd=root) for relative_path_bytes in output.split(b"\x00"): relative_path = Path(os.fsdecode(relative_path_bytes)) if (root / relative_path).is_file(): yield relative_path -def create_project_from_blank_template( - project_name: str, output_path: Path, *, mixed: bool -) -> Path: +def create_project_from_blank_template(project_name: str, output_path: Path, *, mixed: bool) -> Path: project_dir = get_project_copy(script_dir / "blank-project", output_path) project_name = project_name.replace("_", "-") package_name = project_name.replace("-", "_") @@ -233,16 +214,10 @@ def create_project_from_blank_template( project_dir / "Cargo.toml", project_dir / "src/lib.rs", ]: - path.write_text( - path.read_text() - .replace("blank-project", project_name) - .replace("blank_project", package_name) - ) + path.write_text(path.read_text().replace("blank-project", project_name).replace("blank_project", package_name)) if mixed: (project_dir / package_name).mkdir() - (project_dir / package_name / "__init__.py").write_text( - f"from .{package_name} import *" - ) + (project_dir / package_name / "__init__.py").write_text(f"from .{package_name} import *") return project_dir diff --git a/tests/import_hook/test_project_importer.py b/tests/import_hook/test_project_importer.py index 62f7bf9a3..8ff12136f 100644 --- a/tests/import_hook/test_project_importer.py +++ b/tests/import_hook/test_project_importer.py @@ -34,17 +34,12 @@ """ MATURIN_TEST_NAME = os.environ["MATURIN_TEST_NAME"] -MATURIN_BUILD_CACHE = ( - test_crates - / f"targets/import_hook_project_importer_build_cache_{MATURIN_TEST_NAME}" -) +MATURIN_BUILD_CACHE = test_crates / f"targets/import_hook_project_importer_build_cache_{MATURIN_TEST_NAME}" # the CI does not have enough space to keep the outputs. # When running locally you may set this to False for debugging CLEAR_WORKSPACE = True -os.environ["CARGO_TARGET_DIR"] = str( - test_crates / f"targets/import_hook_project_importer_{MATURIN_TEST_NAME}" -) +os.environ["CARGO_TARGET_DIR"] = str(test_crates / f"targets/import_hook_project_importer_{MATURIN_TEST_NAME}") os.environ["MATURIN_BUILD_DIR"] = str(MATURIN_BUILD_CACHE) @@ -86,9 +81,7 @@ def test_install_from_script_inside(workspace: Path, project_name: str) -> None: check_installed_dir = project_dir / "check_installed" check_installed_path = check_installed_dir / "check_installed.py" - check_installed_path.write_text( - f"{IMPORT_HOOK_HEADER}\n\n{check_installed_path.read_text()}" - ) + check_installed_path.write_text(f"{IMPORT_HOOK_HEADER}\n\n{check_installed_path.read_text()}") empty_dir = workspace / "empty" empty_dir.mkdir() @@ -135,9 +128,7 @@ def test_do_not_install_from_script_inside(workspace: Path, project_name: str) - empty_dir = workspace / "empty" empty_dir.mkdir() - output1, _ = run_python( - [str(check_installed_path)], cwd=empty_dir, expect_error=True, quiet=True - ) + output1, _ = run_python([str(check_installed_path)], cwd=empty_dir, expect_error=True, quiet=True) assert ( f'package "{with_underscores(project_name)}" is not already ' f"installed and install_new_packages=False. Not importing" @@ -149,10 +140,7 @@ def test_do_not_install_from_script_inside(workspace: Path, project_name: str) - output2, _ = run_python([str(check_installed_path)], cwd=empty_dir) assert "SUCCESS" in output2 - assert ( - f'package "{with_underscores(project_name)}" will be rebuilt because: no build status found' - in output2 - ) + assert f'package "{with_underscores(project_name)}" will be rebuilt because: no build status found' in output2 assert _rebuilt_message(project_name) in output2 output3, _ = run_python([str(check_installed_path)], cwd=empty_dir) @@ -162,9 +150,7 @@ def test_do_not_install_from_script_inside(workspace: Path, project_name: str) - @pytest.mark.parametrize("project_name", ["pyo3-mixed", "pyo3-pure"]) -def test_do_not_rebuild_if_installed_non_editable( - workspace: Path, project_name: str -) -> None: +def test_do_not_rebuild_if_installed_non_editable(workspace: Path, project_name: str) -> None: """This test ensures that if a maturin project is installed in non-editable mode then the import hook will not rebuild it or re-install it in editable mode. """ @@ -198,20 +184,14 @@ def test_do_not_rebuild_if_installed_non_editable( assert "SUCCESS" in output1 assert "install_new_packages=False" in output1 assert f'found project linked by dist-info: "{project_dir}"' in output1 - assert ( - "package not installed in editable-mode and install_new_packages=False. not rebuilding" - in output1 - ) + assert "package not installed in editable-mode and install_new_packages=False. not rebuilding" in output1 # when inside the project, will detect the project above output2, _ = run_python(["check_installed.py"], cwd=check_installed_dir) assert "SUCCESS" in output2 assert "install_new_packages=False" in output2 assert "found project above the search path:" in output2 - assert ( - "package not installed in editable-mode and install_new_packages=False. not rebuilding" - in output2 - ) + assert "package not installed in editable-mode and install_new_packages=False. not rebuilding" in output2 output3, _ = run_python( ["check_installed.py", "INSTALL_NEW"], @@ -233,9 +213,7 @@ def test_do_not_rebuild_if_installed_non_editable( # path dependencies tested separately sorted(set(all_test_crate_names()) - {"pyo3-mixed-with-path-dep"}), ) -def test_import_editable_installed_rebuild( - workspace: Path, project_name: str, initially_mixed: bool -) -> None: +def test_import_editable_installed_rebuild(workspace: Path, project_name: str, initially_mixed: bool) -> None: """This test ensures that an editable installed project is rebuilt when necessary if the import hook is active. This applies to mixed projects (which are installed as .pth files into site-packages when installed in editable mode) as well as pure projects (which are copied to site-packages @@ -247,13 +225,9 @@ def test_import_editable_installed_rebuild( _clear_build_cache() uninstall(project_name) - check_installed = ( - test_crates / project_name / "check_installed/check_installed.py" - ).read_text() + check_installed = (test_crates / project_name / "check_installed/check_installed.py").read_text() - project_dir = create_project_from_blank_template( - project_name, workspace / project_name, mixed=initially_mixed - ) + project_dir = create_project_from_blank_template(project_name, workspace / project_name, mixed=initially_mixed) log(f"installing blank project as {project_name}") @@ -262,11 +236,7 @@ def test_import_editable_installed_rebuild( # without the import hook the installation test is expected to fail because the project should not be installed yet output0, _ = run_python_code(check_installed, quiet=True, expect_error=True) - assert ( - "AttributeError" in output0 - or "ImportError" in output0 - or "ModuleNotFoundError" in output0 - ) + assert "AttributeError" in output0 or "ImportError" in output0 or "ModuleNotFoundError" in output0 check_installed = f"{IMPORT_HOOK_HEADER}\n\n{check_installed}" @@ -296,9 +266,7 @@ def test_import_editable_installed_rebuild( # path dependencies tested separately sorted(set(mixed_test_crate_names()) - {"pyo3-mixed-with-path-dep"}), ) -def test_import_editable_installed_mixed_missing( - workspace: Path, project_name: str -) -> None: +def test_import_editable_installed_mixed_missing(workspace: Path, project_name: str) -> None: """This test ensures that editable installed mixed projects are rebuilt if they are imported and their artifacts are missing. @@ -312,18 +280,14 @@ def test_import_editable_installed_mixed_missing( # making a copy because editable installation may write files into the project directory project_dir = get_project_copy(test_crates / project_name, workspace / project_name) - project_backup_dir = get_project_copy( - test_crates / project_name, workspace / f"backup_{project_name}" - ) + project_backup_dir = get_project_copy(test_crates / project_name, workspace / f"backup_{project_name}") install_editable(project_dir) assert is_installed_correctly(project_name, project_dir, "mixed" in project_name) check_installed = test_crates / project_name / "check_installed/check_installed.py" - log( - "checking that check_installed works without the import hook right after installing" - ) + log("checking that check_installed works without the import hook right after installing") output0, _ = run_python_code(check_installed.read_text()) assert "SUCCESS" in output0 @@ -378,9 +342,7 @@ def test_concurrent_import(workspace: Path, initially_mixed: bool, mixed: bool) check_installed_with_hook = f"{IMPORT_HOOK_HEADER}\n\n{check_installed}" - project_dir = create_project_from_blank_template( - project_name, workspace / project_name, mixed=initially_mixed - ) + project_dir = create_project_from_blank_template(project_name, workspace / project_name, mixed=initially_mixed) log(f"initially mixed: {initially_mixed}, mixed: {mixed}") log(f"installing blank project as {project_name}") @@ -446,12 +408,8 @@ def test_import_multiple_projects(workspace: Path) -> None: uninstall("pyo3-mixed") uninstall("pyo3-pure") - mixed_dir = create_project_from_blank_template( - "pyo3-mixed", workspace / "pyo3-mixed", mixed=True - ) - pure_dir = create_project_from_blank_template( - "pyo3-pure", workspace / "pyo3-pure", mixed=False - ) + mixed_dir = create_project_from_blank_template("pyo3-mixed", workspace / "pyo3-mixed", mixed=True) + pure_dir = create_project_from_blank_template("pyo3-pure", workspace / "pyo3-pure", mixed=False) install_editable(mixed_dir) assert is_installed_correctly("pyo3-mixed", mixed_dir, True) @@ -499,9 +457,7 @@ def test_rebuild_on_change_to_path_dependency(workspace: Path) -> None: project_dir = get_project_copy(test_crates / project_name, workspace / project_name) get_project_copy(test_crates / "some_path_dep", workspace / "some_path_dep") - transitive_dep_dir = get_project_copy( - test_crates / "transitive_path_dep", workspace / "transitive_path_dep" - ) + transitive_dep_dir = get_project_copy(test_crates / "transitive_path_dep", workspace / "transitive_path_dep") install_editable(project_dir) assert is_installed_correctly(project_name, project_dir, True) @@ -522,9 +478,7 @@ def test_rebuild_on_change_to_path_dependency(workspace: Path) -> None: assert "21 is half 63: False" in output1 transitive_dep_lib = transitive_dep_dir / "src/lib.rs" - transitive_dep_lib.write_text( - transitive_dep_lib.read_text().replace("x + y == sum", "x + x + y == sum") - ) + transitive_dep_lib.write_text(transitive_dep_lib.read_text().replace("x + y == sum", "x + x + y == sum")) output2, duration2 = run_python_code(check_installed) assert "21 is half 42: False" in output2 @@ -541,16 +495,10 @@ def test_rebuild_on_settings_change(workspace: Path, is_mixed: bool) -> None: _clear_build_cache() uninstall("my-script") - project_dir = create_project_from_blank_template( - "my-script", workspace / "my-script", mixed=is_mixed - ) - shutil.copy( - script_dir / "rust_file_import/my_script_3.rs", project_dir / "src/lib.rs" - ) + project_dir = create_project_from_blank_template("my-script", workspace / "my-script", mixed=is_mixed) + shutil.copy(script_dir / "rust_file_import/my_script_3.rs", project_dir / "src/lib.rs") manifest_path = project_dir / "Cargo.toml" - manifest_path.write_text( - f"{manifest_path.read_text()}\n[features]\nlarge_number = []\n" - ) + manifest_path.write_text(f"{manifest_path.read_text()}\n[features]\nlarge_number = []\n") install_editable(project_dir) assert is_installed_correctly("my-script", project_dir, is_mixed) @@ -561,9 +509,7 @@ def test_rebuild_on_settings_change(workspace: Path, is_mixed: bool) -> None: assert "building with default settings" in output1 assert "get_num = 10" in output1 assert "SUCCESS" in output1 - assert ( - 'package "my_script" will be rebuilt because: no build status found' in output1 - ) + assert 'package "my_script" will be rebuilt because: no build status found' in output1 output2, _ = run_python([str(helper_path)], cwd=workspace) assert "get_num = 10" in output2 @@ -573,8 +519,7 @@ def test_rebuild_on_settings_change(workspace: Path, is_mixed: bool) -> None: output3, _ = run_python([str(helper_path), "LARGE_NUMBER"], cwd=workspace) assert "building with large_number feature enabled" in output3 assert ( - 'package "my_script" will be rebuilt because: ' - "current maturin args do not match the previous build" + 'package "my_script" will be rebuilt because: current maturin args do not match the previous build' ) in output3 assert "get_num = 100" in output3 assert "SUCCESS" in output3 @@ -612,18 +557,12 @@ class TestLogging: def _create_clean_project(tmp_dir: Path, is_mixed: bool) -> Path: _clear_build_cache() uninstall("test-project") - project_dir = create_project_from_blank_template( - "test-project", tmp_dir / "test-project", mixed=is_mixed - ) + project_dir = create_project_from_blank_template("test-project", tmp_dir / "test-project", mixed=is_mixed) install_editable(project_dir) assert is_installed_correctly("test-project", project_dir, is_mixed) lib_path = project_dir / "src/lib.rs" - lib_src = ( - lib_path.read_text() - .replace("_m:", "m:") - .replace("Ok(())", 'm.add("value", 10)?;Ok(())') - ) + lib_src = lib_path.read_text().replace("_m:", "m:").replace("Ok(())", 'm.add("value", 10)?;Ok(())') lib_path.write_text(lib_src) return project_dir @@ -684,11 +623,7 @@ def test_default_compile_warning(self, workspace: Path, is_mixed: bool) -> None: """ project_dir = self._create_clean_project(workspace / "project", is_mixed) lib_path = project_dir / "src/lib.rs" - lib_path.write_text( - lib_path.read_text().replace( - "Ok(())", "#[warn(unused_variables)]{let x = 12;}; Ok(())" - ) - ) + lib_path.write_text(lib_path.read_text().replace("Ok(())", "#[warn(unused_variables)]{let x = 12;}; Ok(())")) output1, _ = run_python_code(self.loader_script) output1 = remove_ansii_escape_characters(output1) @@ -702,9 +637,7 @@ def test_default_compile_warning(self, workspace: Path, is_mixed: bool) -> None: "value 10\n" "SUCCESS\n" ) - assert ( - re.fullmatch(pattern, output1, flags=re.MULTILINE | re.DOTALL) is not None - ) + assert re.fullmatch(pattern, output1, flags=re.MULTILINE | re.DOTALL) is not None output2, _ = run_python_code(self.loader_script) output2 = remove_ansii_escape_characters(output2) @@ -716,14 +649,10 @@ def test_default_compile_warning(self, workspace: Path, is_mixed: bool) -> None: "value 10\n" "SUCCESS\n" ) - assert ( - re.fullmatch(pattern, output2, flags=re.MULTILINE | re.DOTALL) is not None - ) + assert re.fullmatch(pattern, output2, flags=re.MULTILINE | re.DOTALL) is not None @pytest.mark.parametrize("is_mixed", [False, True]) - def test_reset_logger_without_configuring( - self, workspace: Path, is_mixed: bool - ) -> None: + def test_reset_logger_without_configuring(self, workspace: Path, is_mixed: bool) -> None: """If reset_logger is called then by default logging level INFO is not printed (because the messages are handled by the root logger). """ @@ -732,15 +661,11 @@ def test_reset_logger_without_configuring( assert output == "value 10\nSUCCESS\n" @pytest.mark.parametrize("is_mixed", [False, True]) - def test_successful_compilation_but_not_valid( - self, workspace: Path, is_mixed: bool - ) -> None: + def test_successful_compilation_but_not_valid(self, workspace: Path, is_mixed: bool) -> None: """If the project compiles but does not import correctly an ImportError is raised.""" project_dir = self._create_clean_project(workspace / "project", is_mixed) lib_path = project_dir / "src/lib.rs" - lib_path.write_text( - lib_path.read_text().replace("test_project", "test_project_new_name") - ) + lib_path.write_text(lib_path.read_text().replace("test_project", "test_project_new_name")) output, _ = run_python_code(self.loader_script, quiet=True) pattern = ( diff --git a/tests/import_hook/test_rust_file_importer.py b/tests/import_hook/test_rust_file_importer.py index bbd3d344b..8b9339be4 100644 --- a/tests/import_hook/test_rust_file_importer.py +++ b/tests/import_hook/test_rust_file_importer.py @@ -23,16 +23,12 @@ """ MATURIN_TEST_NAME = os.environ["MATURIN_TEST_NAME"] -MATURIN_BUILD_CACHE = ( - test_crates / f"targets/import_hook_file_importer_build_cache_{MATURIN_TEST_NAME}" -) +MATURIN_BUILD_CACHE = test_crates / f"targets/import_hook_file_importer_build_cache_{MATURIN_TEST_NAME}" # the CI does not have enough space to keep the outputs. # When running locally you may set this to False for debugging CLEAR_WORKSPACE = True -os.environ["CARGO_TARGET_DIR"] = str( - test_crates / f"targets/import_hook_file_importer_{MATURIN_TEST_NAME}" -) +os.environ["CARGO_TARGET_DIR"] = str(test_crates / f"targets/import_hook_file_importer_{MATURIN_TEST_NAME}") os.environ["MATURIN_BUILD_DIR"] = str(MATURIN_BUILD_CACHE) @@ -75,16 +71,12 @@ def test_relative_import() -> None: """Test imports of the form `from .ab import cd`.""" _clear_build_cache() - output1, duration1 = run_python( - ["-m", "rust_file_import.relative_import_helper"], cwd=script_dir - ) + output1, duration1 = run_python(["-m", "rust_file_import.relative_import_helper"], cwd=script_dir) assert "SUCCESS" in output1 assert "module up to date" not in output1 assert "creating project for" in output1 - output2, duration2 = run_python( - ["-m", "rust_file_import.relative_import_helper"], cwd=script_dir - ) + output2, duration2 = run_python(["-m", "rust_file_import.relative_import_helper"], cwd=script_dir) assert "SUCCESS" in output2 assert "module up to date" in output2 assert "creating project for" not in output2 @@ -176,9 +168,7 @@ def test_rebuild_on_change(workspace: Path) -> None: _clear_build_cache() script_path = workspace / "my_script.rs" - helper_path = shutil.copy( - script_dir / "rust_file_import/rebuild_on_change_helper.py", workspace - ) + helper_path = shutil.copy(script_dir / "rust_file_import/rebuild_on_change_helper.py", workspace) shutil.copy(script_dir / "rust_file_import/my_script_1.rs", script_path) @@ -206,9 +196,7 @@ def test_rebuild_on_settings_change(workspace: Path) -> None: _clear_build_cache() script_path = workspace / "my_script.rs" - helper_path = shutil.copy( - script_dir / "rust_file_import/rebuild_on_settings_change_helper.py", workspace - ) + helper_path = shutil.copy(script_dir / "rust_file_import/rebuild_on_settings_change_helper.py", workspace) shutil.copy(script_dir / "rust_file_import/my_script_3.rs", script_path) @@ -277,12 +265,7 @@ def test_default_rebuild(self, workspace: Path) -> None: rs_path, py_path = self._create_clean_package(workspace / "package") output, _ = run_python([str(py_path)], workspace) - pattern = ( - 'building "my_script"\n' - 'rebuilt and loaded module "my_script" in [0-9.]+s\n' - "get_num 10\n" - "SUCCESS\n" - ) + pattern = 'building "my_script"\nrebuilt and loaded module "my_script" in [0-9.]+s\nget_num 10\nSUCCESS\n' assert re.fullmatch(pattern, output, flags=re.MULTILINE) is not None def test_default_up_to_date(self, workspace: Path) -> None: @@ -319,11 +302,7 @@ def test_default_compile_warning(self, workspace: Path) -> None: built, the warnings will be printed again. """ rs_path, py_path = self._create_clean_package(workspace / "package") - rs_path.write_text( - rs_path.read_text().replace( - "10", "#[warn(unused_variables)]{let x = 12;}; 20" - ) - ) + rs_path.write_text(rs_path.read_text().replace("10", "#[warn(unused_variables)]{let x = 12;}; 20")) output1, _ = run_python([str(py_path)], workspace) output1 = remove_ansii_escape_characters(output1) @@ -337,9 +316,7 @@ def test_default_compile_warning(self, workspace: Path) -> None: "get_num 20\n" "SUCCESS\n" ) - assert ( - re.fullmatch(pattern, output1, flags=re.MULTILINE | re.DOTALL) is not None - ) + assert re.fullmatch(pattern, output1, flags=re.MULTILINE | re.DOTALL) is not None output2, _ = run_python([str(py_path)], workspace) output2 = remove_ansii_escape_characters(output2) @@ -351,9 +328,7 @@ def test_default_compile_warning(self, workspace: Path) -> None: "get_num 20\n" "SUCCESS\n" ) - assert ( - re.fullmatch(pattern, output2, flags=re.MULTILINE | re.DOTALL) is not None - ) + assert re.fullmatch(pattern, output2, flags=re.MULTILINE | re.DOTALL) is not None def test_reset_logger_without_configuring(self, workspace: Path) -> None: """If reset_logger is called then by default logging level INFO is not printed @@ -366,9 +341,7 @@ def test_reset_logger_without_configuring(self, workspace: Path) -> None: def test_successful_compilation_but_not_valid(self, workspace: Path) -> None: """If the script compiles but does not import correctly an ImportError is raised.""" rs_path, py_path = self._create_clean_package(workspace / "package") - rs_path.write_text( - rs_path.read_text().replace("my_script", "my_script_new_name") - ) + rs_path.write_text(rs_path.read_text().replace("my_script", "my_script_new_name")) output, _ = run_python([str(py_path)], workspace, quiet=True) pattern = ( 'building "my_script"\n' diff --git a/tests/import_hook/test_utilities.py b/tests/import_hook/test_utilities.py index 9e0c3e2f2..76d2a4952 100644 --- a/tests/import_hook/test_utilities.py +++ b/tests/import_hook/test_utilities.py @@ -80,9 +80,7 @@ def test_settings() -> None: ] # fmt: on - build_settings = MaturinBuildSettings( - skip_auditwheel=True, zig=True, color=False, rustc_flags=["flag1", "flag2"] - ) + build_settings = MaturinBuildSettings(skip_auditwheel=True, zig=True, color=False, rustc_flags=["flag1", "flag2"]) assert build_settings.to_args() == [ "--skip-auditwheel", "--zig", @@ -118,9 +116,7 @@ def test_missing_extension(self, tmp_path: Path) -> None: def test_missing_path_dep(self, tmp_path: Path) -> None: (tmp_path / "extension").touch() - project_mtime = _get_project_mtime( - tmp_path, [tmp_path / "missing"], tmp_path / "extension", set() - ) + project_mtime = _get_project_mtime(tmp_path, [tmp_path / "missing"], tmp_path / "extension", set()) assert project_mtime is None def test_simple(self, tmp_path: Path) -> None: @@ -129,9 +125,7 @@ def test_simple(self, tmp_path: Path) -> None: (src_dir / "source_file.rs").touch() _small_sleep() (tmp_path / "extension_module").touch() - project_mtime = _get_project_mtime( - tmp_path, [], tmp_path / "extension_module", set() - ) + project_mtime = _get_project_mtime(tmp_path, [], tmp_path / "extension_module", set()) assert project_mtime == (tmp_path / "extension_module").stat().st_mtime (tmp_path / "extension_module").unlink() @@ -140,14 +134,10 @@ def test_simple(self, tmp_path: Path) -> None: # if the extension module is a directory then it should be excluded from the project mtime # calculation as it may contain pycache files that are generated after installation - project_mtime = _get_project_mtime( - tmp_path, [], tmp_path / "extension_module", set() - ) + project_mtime = _get_project_mtime(tmp_path, [], tmp_path / "extension_module", set()) assert project_mtime == (src_dir / "source_file.rs").stat().st_mtime - project_mtime = _get_project_mtime( - tmp_path, [], tmp_path / "extension_module", {"src"} - ) + project_mtime = _get_project_mtime(tmp_path, [], tmp_path / "extension_module", {"src"}) assert project_mtime is None def test_simple_path_dep(self, tmp_path: Path) -> None: @@ -163,15 +153,11 @@ def test_simple_path_dep(self, tmp_path: Path) -> None: _small_sleep() (project_b / "source").touch() - project_mtime = _get_project_mtime( - project_a, [project_b], extension_module, set() - ) + project_mtime = _get_project_mtime(project_a, [project_b], extension_module, set()) assert project_mtime == (project_b / "source").stat().st_mtime extension_module.touch() - project_mtime = _get_project_mtime( - project_a, [project_b], extension_module, set() - ) + project_mtime = _get_project_mtime(project_a, [project_b], extension_module, set()) assert project_mtime == (project_a / "extension").stat().st_mtime def test_extension_module_dir_with_some_newer(self, tmp_path: Path) -> None: @@ -213,14 +199,9 @@ def test_extension_module_dir_with_newer_pycache(self, tmp_path: Path) -> None: extension_mtime = _get_installed_package_mtime(extension_path, set()) assert extension_mtime == extension_path.stat().st_mtime project_mtime = _get_project_mtime(tmp_path, [], extension_path, set()) - assert ( - project_mtime - == (mixed_src_dir / "__pycache__/some_cache.pyc").stat().st_mtime - ) - - project_mtime = _get_project_mtime( - tmp_path, [], extension_path, {"__pycache__"} - ) + assert project_mtime == (mixed_src_dir / "__pycache__/some_cache.pyc").stat().st_mtime + + project_mtime = _get_project_mtime(tmp_path, [], extension_path, {"__pycache__"}) assert project_mtime == extension_path.stat().st_mtime def test_extension_outside_project_source(self, tmp_path: Path) -> None: @@ -274,9 +255,7 @@ def test_resolve_project(project_name: str) -> None: "cargo_manifest_path": _optional_path_to_str(resolved.cargo_manifest_path), "python_dir": _optional_path_to_str(resolved.python_dir), "python_module": _optional_path_to_str(resolved.python_module), - "extension_module_dir": _optional_path_to_str( - resolved.extension_module_dir - ), + "extension_module_dir": _optional_path_to_str(resolved.extension_module_dir), "module_full_name": resolved.module_full_name, } log("calculated:") @@ -313,9 +292,7 @@ def test_load_dist_info(tmp_path: Path) -> None: '{"dir_info": {"editable": true}, "url": "file:///tmp/some%20directory/foo"}' ) - linked_path, is_editable = _load_dist_info( - tmp_path, "package_foo", require_project_target=False - ) + linked_path, is_editable = _load_dist_info(tmp_path, "package_foo", require_project_target=False) assert linked_path == Path("/tmp/some directory/foo") assert is_editable From 07af8b8cb4718f2a6efd2305b3bcdb8e33d04963 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Fri, 22 Dec 2023 12:05:34 +0000 Subject: [PATCH 29/57] improved package resolving for tests to better match maturin cli behaviour --- tests/common/import_hook.rs | 39 +++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/tests/common/import_hook.rs b/tests/common/import_hook.rs index e14ae78a8..b9b217391 100644 --- a/tests/common/import_hook.rs +++ b/tests/common/import_hook.rs @@ -151,20 +151,30 @@ pub fn resolve_all_packages() -> Result { Ok(serde_json::to_string(&Value::Object(resolved_packages))?) } +struct TemporaryChdir { + old_dir: PathBuf, +} + +impl TemporaryChdir { + pub fn chdir(new_cwd: &Path) -> std::io::Result { + let old_dir = env::current_dir()?; + match env::set_current_dir(new_cwd) { + Ok(()) => Ok(Self { old_dir }), + Err(e) => Err(e), + } + } +} + +impl Drop for TemporaryChdir { + fn drop(&mut self) { + env::set_current_dir(&self.old_dir).unwrap(); + } +} + fn resolve_package(project_root: &Path) -> Result { - let manifest_path = if project_root.join("Cargo.toml").exists() { - project_root.join("Cargo.toml") - } else { - project_root.join("rust").join("Cargo.toml") - }; + let _cwd = TemporaryChdir::chdir(project_root)?; - let build_options = BuildOptions { - cargo: CargoOptions { - manifest_path: Some(manifest_path.to_owned()), - ..Default::default() - }, - ..Default::default() - }; + let build_options: BuildOptions = Default::default(); let build_context = build_options.into_build_context(false, false, false)?; let extension_module_dir = if build_context.project_layout.python_module.is_some() { Some(build_context.project_layout.rust_module) @@ -180,3 +190,8 @@ fn resolve_package(project_root: &Path) -> Result { "extension_module_dir": extension_module_dir, })) } + +pub fn debug_print_resolved_package(package_path: &Path) { + let resolved = resolve_package(package_path).unwrap_or(Value::Null); + println!("{}", serde_json::to_string_pretty(&resolved).unwrap()); +} From c51474492656bec708de8033cce5cc937e86d275 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Fri, 22 Dec 2023 12:06:27 +0000 Subject: [PATCH 30/57] added some notes --- tests/README.md | 13 +++++++++++++ tests/import_hook/test_project_importer.py | 1 + 2 files changed, 14 insertions(+) diff --git a/tests/README.md b/tests/README.md index eda6ccb6d..650786967 100644 --- a/tests/README.md +++ b/tests/README.md @@ -14,3 +14,16 @@ handle_result(import_hook::test_import_hook( true, )); ``` + +## Debugging The Import Hook +- if an individual package is failing to import for some reason + - configure the logging level to get more information from the import hook. + - create a script that calls `maturin.import_hook.install()` and run the script in a debugger and step into the import hook source code +- to debug the rust implementation of resolving projects, create and run a test like so. Run the test in a debugger or add print statements. + +```rust +#[test] +fn test_resolve_package() { + debug_print_resolved_package(&Path::new("test-crates/pyo3-mixed-workspace")); +} +``` \ No newline at end of file diff --git a/tests/import_hook/test_project_importer.py b/tests/import_hook/test_project_importer.py index 8ff12136f..6f6ef7696 100644 --- a/tests/import_hook/test_project_importer.py +++ b/tests/import_hook/test_project_importer.py @@ -33,6 +33,7 @@ which provides a clean virtual environment for these tests to use. """ +# when running in parallel this environment variable is set to the name of the function being tested by the runner. MATURIN_TEST_NAME = os.environ["MATURIN_TEST_NAME"] MATURIN_BUILD_CACHE = test_crates / f"targets/import_hook_project_importer_build_cache_{MATURIN_TEST_NAME}" # the CI does not have enough space to keep the outputs. From f1cbb2f5467409a7344e236b6128932c4bb25829 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Wed, 27 Dec 2023 14:20:30 +0000 Subject: [PATCH 31/57] improved cargo manifest detection --- maturin/import_hook/_resolve_project.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/maturin/import_hook/_resolve_project.py b/maturin/import_hook/_resolve_project.py index 5c483390d..3671a0c0c 100644 --- a/maturin/import_hook/_resolve_project.py +++ b/maturin/import_hook/_resolve_project.py @@ -12,6 +12,14 @@ def find_cargo_manifest(project_dir: Path) -> Optional[Path]: + pyproject_path = project_dir / "pyproject.toml" + if pyproject_path.exists(): + with pyproject_path.open("rb") as f: + pyproject = tomllib.load(f) + relative_manifest_path = pyproject.get("tool", {}).get("maturin", {}).get("manifest-path", None) + if relative_manifest_path is not None: + return project_dir / relative_manifest_path + manifest_path = project_dir / "Cargo.toml" if manifest_path.exists(): return manifest_path From d2029ce0e6832c21220f18ce894e7b77d6a4e5b1 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Wed, 27 Dec 2023 16:54:36 +0000 Subject: [PATCH 32/57] resolve path dependencies relative to the manifest dir instead of the project dir --- maturin/import_hook/_resolve_project.py | 16 ++++++++-------- maturin/import_hook/project_importer.py | 8 ++++++-- tests/README.md | 7 +++++-- tests/common/import_hook.rs | 2 +- 4 files changed, 20 insertions(+), 13 deletions(-) diff --git a/maturin/import_hook/_resolve_project.py b/maturin/import_hook/_resolve_project.py index 3671a0c0c..211b8a7e9 100644 --- a/maturin/import_hook/_resolve_project.py +++ b/maturin/import_hook/_resolve_project.py @@ -94,15 +94,15 @@ def _find_all_path_dependencies(immediate_path_dependencies: List[Path]) -> List all_path_dependencies = set() to_search = immediate_path_dependencies.copy() while to_search: - project_dir = to_search.pop() - if project_dir in all_path_dependencies: + dependency_project_dir = to_search.pop() + if dependency_project_dir in all_path_dependencies: continue - all_path_dependencies.add(project_dir) - manifest_path = project_dir / "Cargo.toml" + all_path_dependencies.add(dependency_project_dir) + manifest_path = dependency_project_dir / "Cargo.toml" if manifest_path.exists(): with manifest_path.open("rb") as f: cargo = tomllib.load(f) - to_search.extend(_get_immediate_path_dependencies(project_dir, cargo)) + to_search.extend(_get_immediate_path_dependencies(dependency_project_dir, cargo)) return sorted(all_path_dependencies) @@ -139,7 +139,7 @@ def _resolve_project(project_dir: Path) -> MaturinProject: extension_module_dir: Optional[Path] python_module: Optional[Path] python_module, extension_module_dir, extension_module_name = _resolve_rust_module(python_dir, module_full_name) - immediate_path_dependencies = _get_immediate_path_dependencies(project_dir, cargo) + immediate_path_dependencies = _get_immediate_path_dependencies(manifest_path.parent, cargo) if not python_module.exists(): extension_module_dir = None @@ -195,13 +195,13 @@ def _resolve_module_name(pyproject: Dict[str, Any], cargo: Dict[str, Any]) -> Op return cargo.get("package", {}).get("name", None) -def _get_immediate_path_dependencies(project_dir: Path, cargo: Dict[str, Any]) -> List[Path]: +def _get_immediate_path_dependencies(manifest_dir_path: Path, cargo: Dict[str, Any]) -> List[Path]: path_dependencies = [] for dependency in cargo.get("dependencies", {}).values(): if isinstance(dependency, dict): relative_path = dependency.get("path", None) if relative_path is not None: - path_dependencies.append((project_dir / relative_path).resolve()) + path_dependencies.append((manifest_dir_path / relative_path).resolve()) return path_dependencies diff --git a/maturin/import_hook/project_importer.py b/maturin/import_hook/project_importer.py index 320eb9466..8a1b90b99 100644 --- a/maturin/import_hook/project_importer.py +++ b/maturin/import_hook/project_importer.py @@ -407,6 +407,7 @@ def _get_project_mtime( installed_package_root: Path, excluded_dir_names: Set[str], ) -> Optional[float]: + """get the mtime of the last modified file of the given project or any of its local dependencies""" excluded_dirs = set() if installed_package_root.is_dir(): excluded_dirs.add(installed_package_root) @@ -420,8 +421,11 @@ def _get_project_mtime( excluded_dirs, ) ) - except (FileNotFoundError, ValueError): - logger.debug("error getting project mtime") + except FileNotFoundError as e: + logger.debug("error getting project mtime: %r (%s)", e, e.filename) + return None + except ValueError as e: + logger.debug("error getting project mtime: %r", e) return None diff --git a/tests/README.md b/tests/README.md index 650786967..24c7f93cd 100644 --- a/tests/README.md +++ b/tests/README.md @@ -4,12 +4,15 @@ keeping them may speed up running the tests again, but you may also remove them once the tests have finished. - the import hook tests cannot easily be run outside the test runner. - to run a single import hook test, modify the test runner in `run.rs` to specify a single test instead of a whole module. For example: +- set `CLEAR_WORKSPACE=False` if you want to inspect the output after a test has run +- include "debugpy" in the list of packages if you want to use a debugger such as vscode to place breakpoints and debug: + - `import debugpy; debugpy.listen(5678); debugpy.wait_for_client(); debugpy.breakpoint()` ```rust handle_result(import_hook::test_import_hook( "import_hook_rust_file_importer", "tests/import_hook/test_rust_file_importer.py::test_multiple_imports", // <-- - &[], + &["boltons", "debugpy"], // <-- &[("MATURIN_TEST_NAME", "ALL")], true, )); @@ -26,4 +29,4 @@ handle_result(import_hook::test_import_hook( fn test_resolve_package() { debug_print_resolved_package(&Path::new("test-crates/pyo3-mixed-workspace")); } -``` \ No newline at end of file +``` diff --git a/tests/common/import_hook.rs b/tests/common/import_hook.rs index b9b217391..72947354d 100644 --- a/tests/common/import_hook.rs +++ b/tests/common/import_hook.rs @@ -1,6 +1,6 @@ use crate::common::{create_virtualenv, test_python_path}; use anyhow::{bail, Result}; -use maturin::{BuildOptions, CargoOptions, Target}; +use maturin::{BuildOptions, Target}; use regex::RegexBuilder; use serde_json; use serde_json::{json, Value}; From b760c6133291b1b35bbd8d46240d44850711c428 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Wed, 27 Dec 2023 20:01:44 +0000 Subject: [PATCH 33/57] allow not using the debug utility function --- tests/common/import_hook.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/common/import_hook.rs b/tests/common/import_hook.rs index 72947354d..56d335e47 100644 --- a/tests/common/import_hook.rs +++ b/tests/common/import_hook.rs @@ -191,6 +191,7 @@ fn resolve_package(project_root: &Path) -> Result { })) } +#[allow(unused)] pub fn debug_print_resolved_package(package_path: &Path) { let resolved = resolve_package(package_path).unwrap_or(Value::Null); println!("{}", serde_json::to_string_pretty(&resolved).unwrap()); From db520fd685c168daaf23f5908c157940513ad594 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 31 Dec 2023 17:02:08 +0000 Subject: [PATCH 34/57] better concurrent python runner making output easier to debug --- maturin/import_hook.py | 0 tests/import_hook/common.py | 56 +++++++++++++------- tests/import_hook/test_project_importer.py | 39 +++++--------- tests/import_hook/test_rust_file_importer.py | 36 ++++--------- 4 files changed, 58 insertions(+), 73 deletions(-) delete mode 100644 maturin/import_hook.py diff --git a/maturin/import_hook.py b/maturin/import_hook.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/import_hook/common.py b/tests/import_hook/common.py index 999f52cd2..eca2752e7 100644 --- a/tests/import_hook/common.py +++ b/tests/import_hook/common.py @@ -1,4 +1,5 @@ import itertools +import multiprocessing import os import re import shutil @@ -7,9 +8,9 @@ import sys import tempfile import time -from contextlib import contextmanager +from dataclasses import dataclass from pathlib import Path -from typing import Iterable, List, Optional, Tuple, Generator +from typing import Iterable, List, Optional, Tuple, Any from maturin.import_hook.project_importer import _fix_direct_url, _load_dist_info @@ -229,20 +230,37 @@ def remove_ansii_escape_characters(text: str) -> str: return re.sub(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", text) -@contextmanager -def handle_worker_process_error() -> Generator[None, None, None]: - """For some reason, catching and printing the exception output inside the - worker process does not appear in the pytest output, so catch and print in the main process - """ - try: - yield - except subprocess.CalledProcessError as e: - stdout = None if e.stdout is None else e.stdout.decode() - stderr = None if e.stderr is None else e.stderr.decode() - print("Error in worker process") - print("-" * 50) - print(f"Stdout:\n{stdout}") - print("-" * 50) - print(f"Stderr:\n{stderr}") - print("-" * 50) - raise +@dataclass +class PythonProcessOutput: + output: str + duration: Optional[float] + success: bool + + +def run_concurrent_python(num: int, args: dict[str, Any]) -> list[PythonProcessOutput]: + outputs: list[PythonProcessOutput] = [] + with multiprocessing.Pool(processes=num) as pool: + processes = [] + for i in range(num): + processes.append(pool.apply_async(run_python, kwds=args)) + + for i, proc in enumerate(processes): + try: + output, duration = proc.get() + except subprocess.CalledProcessError as e: + stdout = "None" if e.stdout is None else e.stdout.decode() + stderr = "None" if e.stderr is None else e.stderr.decode() + output = "\n".join(["-" * 50, "Stdout:", stdout, "Stderr:", stderr, "-" * 50]) + success = False + duration = None + else: + success = True + outputs.append(PythonProcessOutput(output, duration, success)) + + for i, o in enumerate(outputs): + log(f"# Subprocess {i}") + log(f"success: {o.success}") + log(f"duration: {o.duration}") + log(f"output:\n{o.output}") + + return outputs diff --git a/tests/import_hook/test_project_importer.py b/tests/import_hook/test_project_importer.py index 6f6ef7696..4c2bbb881 100644 --- a/tests/import_hook/test_project_importer.py +++ b/tests/import_hook/test_project_importer.py @@ -1,4 +1,3 @@ -import multiprocessing import os import re import shutil @@ -24,7 +23,7 @@ test_crates, uninstall, with_underscores, - handle_worker_process_error, + run_concurrent_python, ) """ @@ -341,7 +340,11 @@ def test_concurrent_import(workspace: Path, initially_mixed: bool, mixed: bool) _clear_build_cache() uninstall(project_name) - check_installed_with_hook = f"{IMPORT_HOOK_HEADER}\n\n{check_installed}" + # increase default timeout as under heavy load on a weak machine + # the workers may be waiting on the locks for a long time. + assert "import_hook.install()" in IMPORT_HOOK_HEADER + header = IMPORT_HOOK_HEADER.replace("import_hook.install()", "import_hook.install(lock_timeout_seconds=5 * 60)") + check_installed_with_hook = f"{header}\n\n{check_installed}" project_dir = create_project_from_blank_template(project_name, workspace / project_name, mixed=initially_mixed) @@ -355,40 +358,22 @@ def test_concurrent_import(workspace: Path, initially_mixed: bool, mixed: bool) get_project_copy(test_crates / project_name, project_dir) args = {"python_script": check_installed_with_hook, "quiet": True} - with multiprocessing.Pool(processes=3) as pool: - p1 = pool.apply_async(run_python_code, kwds=args) - p2 = pool.apply_async(run_python_code, kwds=args) - p3 = pool.apply_async(run_python_code, kwds=args) - with handle_worker_process_error(): - output_1, duration_1 = p1.get() - - with handle_worker_process_error(): - output_2, duration_2 = p2.get() - - with handle_worker_process_error(): - output_3, duration_3 = p3.get() - - log("output 1") - log(output_1) - log("output 2") - log(output_2) - log("output 3") - log(output_3) + outputs = run_concurrent_python(3, args) num_compilations = 0 num_up_to_date = 0 num_waiting = 0 - for output in [output_1, output_2, output_3]: - assert "SUCCESS" in output + for output in outputs: + assert "SUCCESS" in output.output - if "waiting on lock" in output: + if "waiting on lock" in output.output: num_waiting += 1 - if _up_to_date_message(project_name) in output: + if _up_to_date_message(project_name) in output.output: num_up_to_date += 1 - if _rebuilt_message(project_name) in output: + if _rebuilt_message(project_name) in output.output: num_compilations += 1 assert num_compilations == 1 diff --git a/tests/import_hook/test_rust_file_importer.py b/tests/import_hook/test_rust_file_importer.py index 8b9339be4..63dc816f5 100644 --- a/tests/import_hook/test_rust_file_importer.py +++ b/tests/import_hook/test_rust_file_importer.py @@ -1,4 +1,3 @@ -import multiprocessing import os import re import shutil @@ -13,7 +12,7 @@ run_python, script_dir, test_crates, - handle_worker_process_error, + run_concurrent_python, ) """ @@ -124,38 +123,21 @@ def test_concurrent_import() -> None: "quiet": True, } - with multiprocessing.Pool(processes=3) as pool: - p1 = pool.apply_async(run_python, kwds=args) - p2 = pool.apply_async(run_python, kwds=args) - p3 = pool.apply_async(run_python, kwds=args) + outputs = run_concurrent_python(3, args) - with handle_worker_process_error(): - output_1, duration_1 = p1.get() - - with handle_worker_process_error(): - output_2, duration_2 = p2.get() - - with handle_worker_process_error(): - output_3, duration_3 = p3.get() - - log("output 1") - log(output_1) - log("output 2") - log(output_2) - log("output 3") - log(output_3) + assert all(o.success for o in outputs) num_compilations = 0 num_up_to_date = 0 num_waiting = 0 - for output in [output_1, output_2, output_3]: - assert "SUCCESS" in output - assert "importing rust file" in output - if "waiting on lock" in output: + for output in outputs: + assert "SUCCESS" in output.output + assert "importing rust file" in output.output + if "waiting on lock" in output.output: num_waiting += 1 - if "creating project for" in output: + if "creating project for" in output.output: num_compilations += 1 - if "module up to date" in output: + if "module up to date" in output.output: num_up_to_date += 1 assert num_compilations == 1 From b79330e3fbfb6e74d2944b70d15c0937e9144368 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 31 Dec 2023 17:05:25 +0000 Subject: [PATCH 35/57] increase lock timeout for concurrent tests when running tests in parallel on a weak machine the concurrent test worker subprocesses will be waiting on the first compilation to finish. the other tests should not have any contested locks so the extra timeout is only important for the concurrent tests. --- maturin/import_hook/__init__.py | 3 ++- maturin/import_hook/_building.py | 18 +++++++++++++----- .../concurrent_import_helper.py | 4 +++- tests/import_hook/test_project_importer.py | 5 +++-- 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/maturin/import_hook/__init__.py b/maturin/import_hook/__init__.py index 64991d31b..dcc6aec4e 100644 --- a/maturin/import_hook/__init__.py +++ b/maturin/import_hook/__init__.py @@ -41,7 +41,8 @@ def install( and so whether the extension module needs to be rebuilt :param lock_timeout_seconds: a lock is required to prevent projects from being built concurrently. - If the lock is not released before this timeout is reached the import hook stops waiting and aborts + If the lock is not released before this timeout is reached the import hook stops waiting and aborts. + A value of None means that the import hook will wait for the lock indefinitely. :param show_warnings: whether to show compilation warnings diff --git a/maturin/import_hook/_building.py b/maturin/import_hook/_building.py index 834e9e702..ccebf3502 100644 --- a/maturin/import_hook/_building.py +++ b/maturin/import_hook/_building.py @@ -100,12 +100,20 @@ def lock(self) -> Generator[LockedBuildCache, None, None]: @contextmanager def _acquire_lock(lock: filelock.FileLock) -> Generator[None, None, None]: try: - with lock.acquire(blocking=False): - yield + try: + with lock.acquire(blocking=False): + yield + except filelock.Timeout: + logger.info("waiting on lock %s", lock.lock_file) + with lock.acquire(): + yield except filelock.Timeout: - logger.info("waiting on lock %s", lock.lock_file) - with lock.acquire(): - yield + raise TimeoutError( + f'Acquiring lock "{lock.lock_file}" timed out after {lock.timeout} seconds. ' + f"If the project is still compiling and needs more time you can increase the " + f"timeout using the lock_timeout_seconds argument to import_hook.install() " + f"(or set to None to wait indefinitely)" + ) from None def _get_default_build_dir() -> Path: diff --git a/tests/import_hook/rust_file_import/concurrent_import_helper.py b/tests/import_hook/rust_file_import/concurrent_import_helper.py index 8c8043209..9040e0bf2 100644 --- a/tests/import_hook/rust_file_import/concurrent_import_helper.py +++ b/tests/import_hook/rust_file_import/concurrent_import_helper.py @@ -6,7 +6,9 @@ from maturin import import_hook import_hook.reset_logger() -import_hook.install() +# increase default timeout as under heavy load on a weak machine +# the workers may be waiting on the locks for a long time. +import_hook.install(lock_timeout_seconds=10 * 60) import packages.my_rust_module diff --git a/tests/import_hook/test_project_importer.py b/tests/import_hook/test_project_importer.py index 4c2bbb881..9f7f22bdd 100644 --- a/tests/import_hook/test_project_importer.py +++ b/tests/import_hook/test_project_importer.py @@ -342,8 +342,9 @@ def test_concurrent_import(workspace: Path, initially_mixed: bool, mixed: bool) # increase default timeout as under heavy load on a weak machine # the workers may be waiting on the locks for a long time. - assert "import_hook.install()" in IMPORT_HOOK_HEADER - header = IMPORT_HOOK_HEADER.replace("import_hook.install()", "import_hook.install(lock_timeout_seconds=5 * 60)") + original_call = "import_hook.install()" + assert original_call in IMPORT_HOOK_HEADER + header = IMPORT_HOOK_HEADER.replace(original_call, "import_hook.install(lock_timeout_seconds=10 * 60)") check_installed_with_hook = f"{header}\n\n{check_installed}" project_dir = create_project_from_blank_template(project_name, workspace / project_name, mixed=initially_mixed) From de4c31a694c5e8cbd00ae35d242a67a0cc14b30d Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 31 Dec 2023 17:14:42 +0000 Subject: [PATCH 36/57] fixed linting issues --- src/auditwheel/mod.rs | 2 +- src/build_context.rs | 6 +++--- src/build_options.rs | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/auditwheel/mod.rs b/src/auditwheel/mod.rs index c9faf392e..6d9f314db 100644 --- a/src/auditwheel/mod.rs +++ b/src/auditwheel/mod.rs @@ -7,5 +7,5 @@ mod repair; pub use audit::*; pub use platform_tag::PlatformTag; -pub use policy::{Policy, MANYLINUX_POLICIES, MUSLLINUX_POLICIES}; +pub use policy::Policy; pub use repair::find_external_libs; diff --git a/src/build_context.rs b/src/build_context.rs index 9e331603b..87419ea7c 100644 --- a/src/build_context.rs +++ b/src/build_context.rs @@ -323,7 +323,7 @@ impl BuildContext { ); } - let tag = others.get(0).or_else(|| musllinux.get(0)).copied(); + let tag = others.first().or_else(|| musllinux.first()).copied(); get_policy_and_libs(artifact, tag, &self.target, allow_linking_libpython) } @@ -694,7 +694,7 @@ impl BuildContext { let mut wheels = Vec::new(); // On windows, we have picked an interpreter to set the location of python.lib, // otherwise it's none - let python_interpreter = interpreters.get(0); + let python_interpreter = interpreters.first(); let artifact = self.compile_cdylib( python_interpreter, Some(&self.project_layout.extension_name), @@ -822,7 +822,7 @@ impl BuildContext { .context("Failed to build a native library through cargo")?; let error_msg = "Cargo didn't build a cdylib. Did you miss crate-type = [\"cdylib\"] \ in the lib section of your Cargo.toml?"; - let artifacts = artifacts.get(0).context(error_msg)?; + let artifacts = artifacts.first().context(error_msg)?; let mut artifact = artifacts .get("cdylib") diff --git a/src/build_options.rs b/src/build_options.rs index d05c2be84..3d0762e4b 100644 --- a/src/build_options.rs +++ b/src/build_options.rs @@ -383,7 +383,7 @@ impl BuildOptions { InterpreterConfig::from_pyo3_config(config_file.as_ref(), target) .context("Invalid PYO3_CONFIG_FILE")?; Ok(vec![PythonInterpreter::from_config(interpreter_config)]) - } else if let Some(interp) = interpreters.get(0) { + } else if let Some(interp) = interpreters.first() { eprintln!("🐍 Using {interp} to generate to link bindings (With abi3, an interpreter is only required on windows)"); Ok(interpreters) } else if generate_import_lib { From 6f9c05f67eee6f502fa0a1332a25a14233a8dc9e Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 31 Dec 2023 17:33:14 +0000 Subject: [PATCH 37/57] pass function to call to concurrent runner utility --- tests/import_hook/common.py | 10 ++++++---- tests/import_hook/test_project_importer.py | 2 +- tests/import_hook/test_rust_file_importer.py | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/import_hook/common.py b/tests/import_hook/common.py index eca2752e7..75853dea9 100644 --- a/tests/import_hook/common.py +++ b/tests/import_hook/common.py @@ -10,7 +10,7 @@ import time from dataclasses import dataclass from pathlib import Path -from typing import Iterable, List, Optional, Tuple, Any +from typing import Iterable, List, Optional, Tuple, Callable, Any, Dict from maturin.import_hook.project_importer import _fix_direct_url, _load_dist_info @@ -237,12 +237,14 @@ class PythonProcessOutput: success: bool -def run_concurrent_python(num: int, args: dict[str, Any]) -> list[PythonProcessOutput]: - outputs: list[PythonProcessOutput] = [] +def run_concurrent_python( + num: int, func: Callable[..., Tuple[str, float]], args: Dict[str, Any] +) -> List[PythonProcessOutput]: + outputs: List[PythonProcessOutput] = [] with multiprocessing.Pool(processes=num) as pool: processes = [] for i in range(num): - processes.append(pool.apply_async(run_python, kwds=args)) + processes.append(pool.apply_async(func, kwds=args)) for i, proc in enumerate(processes): try: diff --git a/tests/import_hook/test_project_importer.py b/tests/import_hook/test_project_importer.py index 9f7f22bdd..3d1ccc904 100644 --- a/tests/import_hook/test_project_importer.py +++ b/tests/import_hook/test_project_importer.py @@ -360,7 +360,7 @@ def test_concurrent_import(workspace: Path, initially_mixed: bool, mixed: bool) args = {"python_script": check_installed_with_hook, "quiet": True} - outputs = run_concurrent_python(3, args) + outputs = run_concurrent_python(3, run_python_code, args) num_compilations = 0 num_up_to_date = 0 diff --git a/tests/import_hook/test_rust_file_importer.py b/tests/import_hook/test_rust_file_importer.py index 63dc816f5..b8ccadaec 100644 --- a/tests/import_hook/test_rust_file_importer.py +++ b/tests/import_hook/test_rust_file_importer.py @@ -123,7 +123,7 @@ def test_concurrent_import() -> None: "quiet": True, } - outputs = run_concurrent_python(3, args) + outputs = run_concurrent_python(3, run_python, args) assert all(o.success for o in outputs) From 1e56fd254b0f8b9815180869a8abf1dc6e96762d Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Mon, 1 Jan 2024 11:15:47 +0000 Subject: [PATCH 38/57] run import hook tests in serial on Cirrus CI --- tests/run.rs | 72 +++++++++++++++++++++++++--------------------------- 1 file changed, 34 insertions(+), 38 deletions(-) diff --git a/tests/run.rs b/tests/run.rs index 7b2006233..ba855b973 100644 --- a/tests/run.rs +++ b/tests/run.rs @@ -831,50 +831,46 @@ fn pyo3_source_date_epoch() { )) } -#[ignore] #[test] fn import_hook_project_importer() { - handle_result(import_hook::test_import_hook( - "import_hook_project_importer", - "tests/import_hook/test_project_importer.py", - &["boltons"], - &[("MATURIN_TEST_NAME", "ALL")], - true, - )); -} - -#[test] -fn import_hook_project_importer_parallel() { - handle_result(import_hook::test_import_hook_parallel( - "import_hook_project_importer", - &PathBuf::from("tests/import_hook/test_project_importer.py"), - &["boltons"], - &[], - true, - )); + if env::var("CIRRUS_CI").is_ok() { + handle_result(import_hook::test_import_hook( + "import_hook_project_importer", + "tests/import_hook/test_project_importer.py", + &["boltons"], + &[("MATURIN_TEST_NAME", "ALL")], + true, + )); + } else { + handle_result(import_hook::test_import_hook_parallel( + "import_hook_project_importer", + &PathBuf::from("tests/import_hook/test_project_importer.py"), + &["boltons"], + &[], + true, + )); + } } -#[ignore] #[test] fn import_hook_rust_file_importer() { - handle_result(import_hook::test_import_hook( - "import_hook_rust_file_importer", - "tests/import_hook/test_rust_file_importer.py", - &[], - &[("MATURIN_TEST_NAME", "ALL")], - true, - )); -} - -#[test] -fn import_hook_rust_file_importer_parallel() { - handle_result(import_hook::test_import_hook_parallel( - "import_hook_rust_file_importer", - &PathBuf::from("tests/import_hook/test_rust_file_importer.py"), - &[], - &[], - true, - )); + if env::var("CIRRUS_CI").is_ok() { + handle_result(import_hook::test_import_hook( + "import_hook_rust_file_importer", + "tests/import_hook/test_rust_file_importer.py", + &[], + &[("MATURIN_TEST_NAME", "ALL")], + true, + )); + } else { + handle_result(import_hook::test_import_hook_parallel( + "import_hook_rust_file_importer", + &PathBuf::from("tests/import_hook/test_rust_file_importer.py"), + &[], + &[], + true, + )); + } } #[test] From 5b546043282ea432572b7e5c4eb2a8f0b9452b54 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Mon, 1 Jan 2024 11:21:58 +0000 Subject: [PATCH 39/57] normalize newline in python output --- tests/import_hook/common.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/import_hook/common.py b/tests/import_hook/common.py index 75853dea9..7c2513772 100644 --- a/tests/import_hook/common.py +++ b/tests/import_hook/common.py @@ -100,6 +100,8 @@ def run_python( raise duration = time.perf_counter() - start + output = output.replace("\r\n", "\n") + if verbose and not quiet: print("-" * 40) print(subprocess.list2cmdline(cmd)) From 1eca5d929c9f1f313b7ad28bf957e7186ce44f4d Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Mon, 1 Jan 2024 11:44:19 +0000 Subject: [PATCH 40/57] check_match utility to give more informative error messages --- tests/import_hook/common.py | 6 ++++++ tests/import_hook/test_project_importer.py | 11 ++++++----- tests/import_hook/test_rust_file_importer.py | 11 ++++++----- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/tests/import_hook/common.py b/tests/import_hook/common.py index 7c2513772..093cebeb8 100644 --- a/tests/import_hook/common.py +++ b/tests/import_hook/common.py @@ -232,6 +232,12 @@ def remove_ansii_escape_characters(text: str) -> str: return re.sub(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", text) +def check_match(text: str, pattern: str, *, flags: int = 0) -> None: + assert ( + re.fullmatch(pattern, text, flags=flags) is not None + ), f'text does not match pattern:\npattern: "{pattern}"\ntext:\n{text}' + + @dataclass class PythonProcessOutput: output: str diff --git a/tests/import_hook/test_project_importer.py b/tests/import_hook/test_project_importer.py index 3d1ccc904..42e1c1a22 100644 --- a/tests/import_hook/test_project_importer.py +++ b/tests/import_hook/test_project_importer.py @@ -24,6 +24,7 @@ uninstall, with_underscores, run_concurrent_python, + check_match, ) """ @@ -568,7 +569,7 @@ def test_default_rebuild(self, workspace: Path, is_mixed: bool) -> None: "value 10\n" "SUCCESS\n" ) - assert re.fullmatch(pattern, output, flags=re.MULTILINE) is not None + check_match(output, pattern, flags=re.MULTILINE) @pytest.mark.parametrize("is_mixed", [False, True]) def test_default_up_to_date(self, workspace: Path, is_mixed: bool) -> None: @@ -600,7 +601,7 @@ def test_default_compile_error(self, workspace: Path, is_mixed: bool) -> None: ".*" "caught ImportError: Failed to build package with maturin\n" ) - assert re.fullmatch(pattern, output, flags=re.MULTILINE | re.DOTALL) is not None + check_match(output, pattern, flags=re.MULTILINE | re.DOTALL) @pytest.mark.parametrize("is_mixed", [False, True]) def test_default_compile_warning(self, workspace: Path, is_mixed: bool) -> None: @@ -624,7 +625,7 @@ def test_default_compile_warning(self, workspace: Path, is_mixed: bool) -> None: "value 10\n" "SUCCESS\n" ) - assert re.fullmatch(pattern, output1, flags=re.MULTILINE | re.DOTALL) is not None + check_match(output1, pattern, flags=re.MULTILINE | re.DOTALL) output2, _ = run_python_code(self.loader_script) output2 = remove_ansii_escape_characters(output2) @@ -636,7 +637,7 @@ def test_default_compile_warning(self, workspace: Path, is_mixed: bool) -> None: "value 10\n" "SUCCESS\n" ) - assert re.fullmatch(pattern, output2, flags=re.MULTILINE | re.DOTALL) is not None + check_match(output2, pattern, flags=re.MULTILINE | re.DOTALL) @pytest.mark.parametrize("is_mixed", [False, True]) def test_reset_logger_without_configuring(self, workspace: Path, is_mixed: bool) -> None: @@ -660,7 +661,7 @@ def test_successful_compilation_but_not_valid(self, workspace: Path, is_mixed: b 'rebuilt and loaded package "test_project" in [0-9.]+s\n' "caught ImportError: dynamic module does not define module export function \\(PyInit_test_project\\)\n" ) - assert re.fullmatch(pattern, output, flags=re.MULTILINE) is not None + check_match(output, pattern, flags=re.MULTILINE) def _up_to_date_message(project_name: str) -> str: diff --git a/tests/import_hook/test_rust_file_importer.py b/tests/import_hook/test_rust_file_importer.py index b8ccadaec..9be719012 100644 --- a/tests/import_hook/test_rust_file_importer.py +++ b/tests/import_hook/test_rust_file_importer.py @@ -13,6 +13,7 @@ script_dir, test_crates, run_concurrent_python, + check_match, ) """ @@ -248,7 +249,7 @@ def test_default_rebuild(self, workspace: Path) -> None: output, _ = run_python([str(py_path)], workspace) pattern = 'building "my_script"\nrebuilt and loaded module "my_script" in [0-9.]+s\nget_num 10\nSUCCESS\n' - assert re.fullmatch(pattern, output, flags=re.MULTILINE) is not None + check_match(output, pattern, flags=re.MULTILINE) def test_default_up_to_date(self, workspace: Path) -> None: """By default, when the module is up-to-date nothing is printed.""" @@ -276,7 +277,7 @@ def test_default_compile_error(self, workspace: Path) -> None: ".*" "caught ImportError: Failed to build wheel with maturin\n" ) - assert re.fullmatch(pattern, output, flags=re.MULTILINE | re.DOTALL) is not None + check_match(output, pattern, flags=re.MULTILINE | re.DOTALL) def test_default_compile_warning(self, workspace: Path) -> None: """If compilation succeeds with warnings then the output of maturin is printed. @@ -298,7 +299,7 @@ def test_default_compile_warning(self, workspace: Path) -> None: "get_num 20\n" "SUCCESS\n" ) - assert re.fullmatch(pattern, output1, flags=re.MULTILINE | re.DOTALL) is not None + check_match(output1, pattern, flags=re.MULTILINE | re.DOTALL) output2, _ = run_python([str(py_path)], workspace) output2 = remove_ansii_escape_characters(output2) @@ -310,7 +311,7 @@ def test_default_compile_warning(self, workspace: Path) -> None: "get_num 20\n" "SUCCESS\n" ) - assert re.fullmatch(pattern, output2, flags=re.MULTILINE | re.DOTALL) is not None + check_match(output2, pattern, flags=re.MULTILINE | re.DOTALL) def test_reset_logger_without_configuring(self, workspace: Path) -> None: """If reset_logger is called then by default logging level INFO is not printed @@ -330,4 +331,4 @@ def test_successful_compilation_but_not_valid(self, workspace: Path) -> None: 'rebuilt and loaded module "my_script" in [0-9.]+s\n' "caught ImportError: dynamic module does not define module export function \\(PyInit_my_script\\)\n" ) - assert re.fullmatch(pattern, output, flags=re.MULTILINE) is not None + check_match(output, pattern, flags=re.MULTILINE) From 5f3becb5b17f8940e506b918403b84f88706d1b1 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Mon, 1 Jan 2024 15:46:36 +0000 Subject: [PATCH 41/57] add support for pypy error message --- tests/import_hook/test_project_importer.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/import_hook/test_project_importer.py b/tests/import_hook/test_project_importer.py index 42e1c1a22..926549721 100644 --- a/tests/import_hook/test_project_importer.py +++ b/tests/import_hook/test_project_importer.py @@ -1,4 +1,5 @@ import os +import platform import re import shutil from pathlib import Path @@ -656,10 +657,18 @@ def test_successful_compilation_but_not_valid(self, workspace: Path, is_mixed: b lib_path.write_text(lib_path.read_text().replace("test_project", "test_project_new_name")) output, _ = run_python_code(self.loader_script, quiet=True) + + if platform.python_implementation() == "CPython": + error_message = "dynamic module does not define module export function \\(PyInit_test_project\\)" + elif platform.python_implementation() == "PyPy": + error_message = "function _cffi_pypyinit_test_project or PyInit_test_project not found in library .*" + else: + raise NotImplementedError(platform.python_implementation()) + pattern = ( 'building "test_project"\n' 'rebuilt and loaded package "test_project" in [0-9.]+s\n' - "caught ImportError: dynamic module does not define module export function \\(PyInit_test_project\\)\n" + f"caught ImportError: {error_message}\n" ) check_match(output, pattern, flags=re.MULTILINE) From fb42e94644e1de6b809a88b789cc1a8ddbf4332a Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Mon, 1 Jan 2024 16:46:53 +0000 Subject: [PATCH 42/57] disabled pip version check --- tests/import_hook/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/import_hook/common.py b/tests/import_hook/common.py index 093cebeb8..80328311a 100644 --- a/tests/import_hook/common.py +++ b/tests/import_hook/common.py @@ -145,7 +145,7 @@ def log(message: str) -> None: def uninstall(project_name: str) -> None: log(f"uninstalling {project_name}") - subprocess.check_call([sys.executable, "-m", "pip", "uninstall", "-y", project_name]) + subprocess.check_call([sys.executable, "-m", "pip", "uninstall", "--disable-pip-version-check", "-y", project_name]) def install_editable(project_dir: Path) -> None: @@ -160,7 +160,7 @@ def install_editable(project_dir: Path) -> None: def install_non_editable(project_dir: Path) -> None: log(f"installing {project_dir.name} in non-editable mode") - subprocess.check_call([sys.executable, "-m", "pip", "install", str(project_dir)]) + subprocess.check_call([sys.executable, "-m", "pip", "install", "--disable-pip-version-check", str(project_dir)]) def _is_installed_as_pth(project_name: str) -> bool: From 1ee950ac5b0001463e38becc4401d415cfaa8d59 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Mon, 1 Jan 2024 17:17:10 +0000 Subject: [PATCH 43/57] skip import hook tests on CIRRUS_CI --- tests/run.rs | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/tests/run.rs b/tests/run.rs index ba855b973..24c75c2d4 100644 --- a/tests/run.rs +++ b/tests/run.rs @@ -834,19 +834,21 @@ fn pyo3_source_date_epoch() { #[test] fn import_hook_project_importer() { if env::var("CIRRUS_CI").is_ok() { - handle_result(import_hook::test_import_hook( + // skip + } else if env::var("GITHUB_ACTIONS").is_ok() { + handle_result(import_hook::test_import_hook_parallel( "import_hook_project_importer", - "tests/import_hook/test_project_importer.py", + &PathBuf::from("tests/import_hook/test_project_importer.py"), &["boltons"], - &[("MATURIN_TEST_NAME", "ALL")], + &[], true, )); } else { - handle_result(import_hook::test_import_hook_parallel( + handle_result(import_hook::test_import_hook( "import_hook_project_importer", - &PathBuf::from("tests/import_hook/test_project_importer.py"), + "tests/import_hook/test_project_importer.py", &["boltons"], - &[], + &[("MATURIN_TEST_NAME", "ALL")], true, )); } @@ -855,19 +857,21 @@ fn import_hook_project_importer() { #[test] fn import_hook_rust_file_importer() { if env::var("CIRRUS_CI").is_ok() { - handle_result(import_hook::test_import_hook( + // skip + } else if env::var("GITHUB_ACTIONS").is_ok() { + handle_result(import_hook::test_import_hook_parallel( "import_hook_rust_file_importer", - "tests/import_hook/test_rust_file_importer.py", + &PathBuf::from("tests/import_hook/test_rust_file_importer.py"), + &[], &[], - &[("MATURIN_TEST_NAME", "ALL")], true, )); } else { - handle_result(import_hook::test_import_hook_parallel( + handle_result(import_hook::test_import_hook( "import_hook_rust_file_importer", - &PathBuf::from("tests/import_hook/test_rust_file_importer.py"), - &[], + "tests/import_hook/test_rust_file_importer.py", &[], + &[("MATURIN_TEST_NAME", "ALL")], true, )); } From 6213b0b1e07af3c1786b632bdc50f6ff65382286 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Tue, 2 Jan 2024 21:45:52 +0000 Subject: [PATCH 44/57] improved logging in case of run_python failure --- tests/import_hook/common.py | 30 ++++++++++------- tests/import_hook/test_project_importer.py | 38 +++++++++++----------- 2 files changed, 38 insertions(+), 30 deletions(-) diff --git a/tests/import_hook/common.py b/tests/import_hook/common.py index 80328311a..49ae84283 100644 --- a/tests/import_hook/common.py +++ b/tests/import_hook/common.py @@ -16,7 +16,6 @@ verbose = True - script_dir = Path(__file__).resolve().parent maturin_dir = script_dir.parent.parent test_crates = maturin_dir / "test-crates" @@ -30,7 +29,6 @@ import_hook.install() """ - EXCLUDED_PROJECTS = { "hello-world", # not imported as a python module (subprocess only) "license-test", # not imported as a python module (subprocess only) @@ -88,16 +86,18 @@ def run_python( message = "\n".join( [ "-" * 40, - "ERROR:", + "Called Process Error:", subprocess.list2cmdline(cmd), - "", + "Output:", output, "-" * 40, ] ) - print(message, file=sys.stderr) + print(message) if not expect_error: - raise + # re-raising the CalledProcessError would cause + # unnecessary output since we are already printing it above + raise RuntimeError("run_python failed") from None duration = time.perf_counter() - start output = output.replace("\r\n", "\n") @@ -176,18 +176,26 @@ def _is_installed_editable_with_direct_url(project_name: str, project_dir: Path) if not is_editable: log(f'project "{project_name}" is installed but not in editable mode') return is_editable - else: + elif linked_path is not None: log(f'found linked path "{linked_path}" for project "{project_name}". Expected "{project_dir}"') + return False return False -def is_installed_correctly(project_name: str, project_dir: Path, is_mixed: bool) -> bool: +def is_editable_installed_correctly(project_name: str, project_dir: Path, is_mixed: bool) -> bool: + log(f"checking if {project_name} is installed correctly.") installed_as_pth = _is_installed_as_pth(project_name) installed_editable_with_direct_url = _is_installed_editable_with_direct_url(project_name, project_dir) - log( - f"checking if {project_name} is installed correctly. " - f"{is_mixed=}, {installed_as_pth=} {installed_editable_with_direct_url=}" + log(f"{is_mixed=}, {installed_as_pth=} {installed_editable_with_direct_url=}") + + proc = subprocess.run( + [sys.executable, "-m", "pip", "show", "--disable-pip-version-check", "-f", project_name], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, ) + output = "None" if proc.stdout is None else proc.stdout.decode() + log(f"pip output (returned {proc.returncode}):\n{output}") return installed_editable_with_direct_url and (installed_as_pth == is_mixed) diff --git a/tests/import_hook/test_project_importer.py b/tests/import_hook/test_project_importer.py index 926549721..10ed37453 100644 --- a/tests/import_hook/test_project_importer.py +++ b/tests/import_hook/test_project_importer.py @@ -14,7 +14,7 @@ get_project_copy, install_editable, install_non_editable, - is_installed_correctly, + is_editable_installed_correctly, log, mixed_test_crate_names, remove_ansii_escape_characters, @@ -93,7 +93,7 @@ def test_install_from_script_inside(workspace: Path, project_name: str) -> None: assert _rebuilt_message(project_name) in output1 assert _up_to_date_message(project_name) not in output1 - assert is_installed_correctly(project_name, project_dir, "mixed" in project_name) + assert is_editable_installed_correctly(project_name, project_dir, "mixed" in project_name) output2, duration2 = run_python([str(check_installed_path)], cwd=empty_dir) assert "SUCCESS" in output2 @@ -102,7 +102,7 @@ def test_install_from_script_inside(workspace: Path, project_name: str) -> None: assert duration2 < duration1 - assert is_installed_correctly(project_name, project_dir, "mixed" in project_name) + assert is_editable_installed_correctly(project_name, project_dir, "mixed" in project_name) @pytest.mark.parametrize("project_name", ["pyo3-mixed", "pyo3-pure"]) @@ -138,7 +138,7 @@ def test_do_not_install_from_script_inside(workspace: Path, project_name: str) - assert "SUCCESS" not in output1 install_editable(project_dir) - assert is_installed_correctly(project_name, project_dir, "mixed" in project_name) + assert is_editable_installed_correctly(project_name, project_dir, "mixed" in project_name) output2, _ = run_python([str(check_installed_path)], cwd=empty_dir) assert "SUCCESS" in output2 @@ -234,7 +234,7 @@ def test_import_editable_installed_rebuild(workspace: Path, project_name: str, i log(f"installing blank project as {project_name}") install_editable(project_dir) - assert is_installed_correctly(project_name, project_dir, initially_mixed) + assert is_editable_installed_correctly(project_name, project_dir, initially_mixed) # without the import hook the installation test is expected to fail because the project should not be installed yet output0, _ = run_python_code(check_installed, quiet=True, expect_error=True) @@ -251,7 +251,7 @@ def test_import_editable_installed_rebuild(workspace: Path, project_name: str, i assert _rebuilt_message(project_name) in output1 assert _up_to_date_message(project_name) not in output1 - assert is_installed_correctly(project_name, project_dir, "mixed" in project_name) + assert is_editable_installed_correctly(project_name, project_dir, "mixed" in project_name) output2, duration2 = run_python_code(check_installed) assert "SUCCESS" in output2 @@ -260,7 +260,7 @@ def test_import_editable_installed_rebuild(workspace: Path, project_name: str, i assert duration2 < duration1 - assert is_installed_correctly(project_name, project_dir, "mixed" in project_name) + assert is_editable_installed_correctly(project_name, project_dir, "mixed" in project_name) @pytest.mark.parametrize( @@ -285,7 +285,7 @@ def test_import_editable_installed_mixed_missing(workspace: Path, project_name: project_backup_dir = get_project_copy(test_crates / project_name, workspace / f"backup_{project_name}") install_editable(project_dir) - assert is_installed_correctly(project_name, project_dir, "mixed" in project_name) + assert is_editable_installed_correctly(project_name, project_dir, "mixed" in project_name) check_installed = test_crates / project_name / "check_installed/check_installed.py" @@ -312,7 +312,7 @@ def test_import_editable_installed_mixed_missing(workspace: Path, project_name: assert duration2 < duration1 - assert is_installed_correctly(project_name, project_dir, True) + assert is_editable_installed_correctly(project_name, project_dir, True) @pytest.mark.parametrize("mixed", [False, True]) @@ -355,7 +355,7 @@ def test_concurrent_import(workspace: Path, initially_mixed: bool, mixed: bool) log(f"installing blank project as {project_name}") install_editable(project_dir) - assert is_installed_correctly(project_name, project_dir, initially_mixed) + assert is_editable_installed_correctly(project_name, project_dir, initially_mixed) shutil.rmtree(project_dir) get_project_copy(test_crates / project_name, project_dir) @@ -383,7 +383,7 @@ def test_concurrent_import(workspace: Path, initially_mixed: bool, mixed: bool) assert num_up_to_date == 2 assert num_waiting == 2 - assert is_installed_correctly(project_name, project_dir, mixed) + assert is_editable_installed_correctly(project_name, project_dir, mixed) def test_import_multiple_projects(workspace: Path) -> None: @@ -401,9 +401,9 @@ def test_import_multiple_projects(workspace: Path) -> None: pure_dir = create_project_from_blank_template("pyo3-pure", workspace / "pyo3-pure", mixed=False) install_editable(mixed_dir) - assert is_installed_correctly("pyo3-mixed", mixed_dir, True) + assert is_editable_installed_correctly("pyo3-mixed", mixed_dir, True) install_editable(pure_dir) - assert is_installed_correctly("pyo3-pure", pure_dir, False) + assert is_editable_installed_correctly("pyo3-pure", pure_dir, False) shutil.rmtree(mixed_dir) shutil.rmtree(pure_dir) @@ -432,8 +432,8 @@ def test_import_multiple_projects(workspace: Path) -> None: assert duration2 < duration1 - assert is_installed_correctly("pyo3-mixed", mixed_dir, True) - assert is_installed_correctly("pyo3-pure", pure_dir, False) + assert is_editable_installed_correctly("pyo3-mixed", mixed_dir, True) + assert is_editable_installed_correctly("pyo3-pure", pure_dir, False) def test_rebuild_on_change_to_path_dependency(workspace: Path) -> None: @@ -449,7 +449,7 @@ def test_rebuild_on_change_to_path_dependency(workspace: Path) -> None: transitive_dep_dir = get_project_copy(test_crates / "transitive_path_dep", workspace / "transitive_path_dep") install_editable(project_dir) - assert is_installed_correctly(project_name, project_dir, True) + assert is_editable_installed_correctly(project_name, project_dir, True) check_installed = f""" {IMPORT_HOOK_HEADER} @@ -473,7 +473,7 @@ def test_rebuild_on_change_to_path_dependency(workspace: Path) -> None: assert "21 is half 42: False" in output2 assert "21 is half 63: True" in output2 - assert is_installed_correctly(project_name, project_dir, True) + assert is_editable_installed_correctly(project_name, project_dir, True) @pytest.mark.parametrize("is_mixed", [False, True]) @@ -490,7 +490,7 @@ def test_rebuild_on_settings_change(workspace: Path, is_mixed: bool) -> None: manifest_path.write_text(f"{manifest_path.read_text()}\n[features]\nlarge_number = []\n") install_editable(project_dir) - assert is_installed_correctly("my-script", project_dir, is_mixed) + assert is_editable_installed_correctly("my-script", project_dir, is_mixed) helper_path = script_dir / "rust_file_import/rebuild_on_settings_change_helper.py" @@ -548,7 +548,7 @@ def _create_clean_project(tmp_dir: Path, is_mixed: bool) -> Path: uninstall("test-project") project_dir = create_project_from_blank_template("test-project", tmp_dir / "test-project", mixed=is_mixed) install_editable(project_dir) - assert is_installed_correctly("test-project", project_dir, is_mixed) + assert is_editable_installed_correctly("test-project", project_dir, is_mixed) lib_path = project_dir / "src/lib.rs" lib_src = lib_path.read_text().replace("_m:", "m:").replace("Ok(())", 'm.add("value", 10)?;Ok(())') From 53d5cd0ca5f250b464e7d5e31f08ce6e7f9070d2 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Tue, 2 Jan 2024 22:18:17 +0000 Subject: [PATCH 45/57] fix handling of file URIs on Windows --- maturin/import_hook/project_importer.py | 14 ++++++++++++-- tests/import_hook/test_utilities.py | 9 +++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/maturin/import_hook/project_importer.py b/maturin/import_hook/project_importer.py index 8a1b90b99..8e0aceef8 100644 --- a/maturin/import_hook/project_importer.py +++ b/maturin/import_hook/project_importer.py @@ -4,10 +4,12 @@ import json import logging import math +import os import site import sys import time import urllib.parse +import urllib.request from importlib.machinery import ModuleSpec, PathFinder from pathlib import Path from types import ModuleType @@ -330,13 +332,21 @@ def _load_dist_info( prefix = "file://" if not url.startswith(prefix): return None, is_editable - linked_path = Path(urllib.parse.unquote(url[len(prefix) :])) + linked_path = _uri_to_path(url) if not require_project_target or is_maybe_maturin_project(linked_path): return linked_path, is_editable else: return None, is_editable +def _uri_to_path(uri: str) -> Path: + """based on https://stackoverflow.com/a/61922504""" + parsed = urllib.parse.urlparse(uri) + host = "{0}{0}{netloc}{0}".format(os.path.sep, netloc=parsed.netloc) + path = urllib.request.url2pathname(urllib.parse.unquote(parsed.path)) + return Path(os.path.normpath(os.path.join(host, path))) + + def _fix_direct_url(project_dir: Path, package_name: str) -> None: """Seemingly due to a bug, installing with `pip install -e` will write the correct entry into `direct_url.json` to point at the project directory, but calling `maturin develop` does not currently write this value correctly. @@ -352,7 +362,7 @@ def _fix_direct_url(project_dir: Path, package_name: str) -> None: direct_url = json.load(f) except OSError: continue - url = f"file://{urllib.parse.quote(str(project_dir))}" + url = project_dir.as_uri() if direct_url.get("url") != url: logger.debug("fixing direct_url.json for package %s", package_name) logger.debug('"%s" -> "%s"', direct_url.get("url"), url) diff --git a/tests/import_hook/test_utilities.py b/tests/import_hook/test_utilities.py index 76d2a4952..e68717bee 100644 --- a/tests/import_hook/test_utilities.py +++ b/tests/import_hook/test_utilities.py @@ -1,5 +1,6 @@ import json import os +import platform import shutil import time from operator import itemgetter @@ -15,6 +16,7 @@ _get_installed_package_mtime, _get_project_mtime, _load_dist_info, + _uri_to_path, ) from maturin.import_hook.settings import MaturinBuildSettings, MaturinDevelopSettings @@ -285,6 +287,13 @@ def test_build_cache(tmp_path: Path) -> None: assert locked_cache.get_build_status(tmp_path / "source1") == status1b +def test_uri_to_path() -> None: + if platform.platform().lower() == "windows": + assert _uri_to_path("file:///C:/abc/d%20e%20f") == Path(r"C:\abc\d e f") + else: + assert _uri_to_path("file:///abc/d%20e%20f") == Path("/abc/d e f") + + def test_load_dist_info(tmp_path: Path) -> None: dist_info = tmp_path / "package_foo-1.0.0.dist-info" dist_info.mkdir(parents=True) From b51976ecc632fa05c91976e589a67834628bcdd1 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Wed, 3 Jan 2024 00:22:36 +0000 Subject: [PATCH 46/57] fix setting PATH environment on windows --- tests/common/import_hook.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/common/import_hook.rs b/tests/common/import_hook.rs index 56d335e47..447ddf452 100644 --- a/tests/common/import_hook.rs +++ b/tests/common/import_hook.rs @@ -45,7 +45,10 @@ pub fn test_import_hook( let path = env::var_os("PATH").unwrap(); let mut paths = env::split_paths(&path).collect::>(); - paths.insert(0, venv_dir.join("bin")); + paths.insert( + 0, + venv_dir.join(if cfg!(windows) { "Scripts" } else { "bin" }), + ); paths.insert( 0, Path::new(env!("CARGO_BIN_EXE_maturin")) From 437230039390972688db373243fd6cfcf759eae2 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Wed, 3 Jan 2024 19:45:42 +0000 Subject: [PATCH 47/57] improved OS detection --- maturin/import_hook/_building.py | 16 ++++++++-------- tests/import_hook/test_utilities.py | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/maturin/import_hook/_building.py b/maturin/import_hook/_building.py index ccebf3502..eef104899 100644 --- a/maturin/import_hook/_building.py +++ b/maturin/import_hook/_building.py @@ -130,17 +130,17 @@ def _get_default_build_dir() -> Path: def _get_cache_dir() -> Path: - if os.name == "posix": - if sys.platform == "darwin": - return Path("~/Library/Caches").expanduser() - else: - xdg_cache_dir = os.environ.get("XDG_CACHE_HOME", None) - return Path(xdg_cache_dir) if xdg_cache_dir else Path("~/.cache").expanduser() - elif platform.platform().lower() == "windows": + os_name = platform.system() + if os_name == "Linux": + xdg_cache_dir = os.environ.get("XDG_CACHE_HOME", None) + return Path(xdg_cache_dir) if xdg_cache_dir else Path("~/.cache").expanduser() + elif os_name == "Darwin": + return Path("~/Library/Caches").expanduser() + elif os_name == "Windows": local_app_data = os.environ.get("LOCALAPPDATA", None) return Path(local_app_data) if local_app_data else Path(r"~\AppData\Local").expanduser() else: - logger.warning("unknown OS. defaulting to ~/.cache as the cache directory") + logger.warning("unknown OS: %s. defaulting to ~/.cache as the cache directory", os_name) return Path("~/.cache").expanduser() diff --git a/tests/import_hook/test_utilities.py b/tests/import_hook/test_utilities.py index e68717bee..e4050af81 100644 --- a/tests/import_hook/test_utilities.py +++ b/tests/import_hook/test_utilities.py @@ -288,7 +288,7 @@ def test_build_cache(tmp_path: Path) -> None: def test_uri_to_path() -> None: - if platform.platform().lower() == "windows": + if platform.system() == "Windows": assert _uri_to_path("file:///C:/abc/d%20e%20f") == Path(r"C:\abc\d e f") else: assert _uri_to_path("file:///abc/d%20e%20f") == Path("/abc/d e f") From f582ee45a5c90baa5b9fe6d29294bed0e511b1f1 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Wed, 3 Jan 2024 19:50:06 +0000 Subject: [PATCH 48/57] fixed expected error message for pypy --- tests/import_hook/common.py | 10 ++++++++++ tests/import_hook/test_project_importer.py | 18 ++++-------------- tests/import_hook/test_rust_file_importer.py | 3 ++- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/tests/import_hook/common.py b/tests/import_hook/common.py index 49ae84283..4ca5570fa 100644 --- a/tests/import_hook/common.py +++ b/tests/import_hook/common.py @@ -8,6 +8,7 @@ import sys import tempfile import time +import platform from dataclasses import dataclass from pathlib import Path from typing import Iterable, List, Optional, Tuple, Callable, Any, Dict @@ -246,6 +247,15 @@ def check_match(text: str, pattern: str, *, flags: int = 0) -> None: ), f'text does not match pattern:\npattern: "{pattern}"\ntext:\n{text}' +def missing_entrypoint_error_message_pattern(name: str) -> str: + if platform.python_implementation() == "CPython": + return f"dynamic module does not define module export function \\(PyInit_{name}\\)" + elif platform.python_implementation() == "PyPy": + return f"function _cffi_pypyinit_{name} or PyInit_{name} not found in library .*" + else: + raise NotImplementedError(platform.python_implementation()) + + @dataclass class PythonProcessOutput: output: str diff --git a/tests/import_hook/test_project_importer.py b/tests/import_hook/test_project_importer.py index 10ed37453..dc1ce8c06 100644 --- a/tests/import_hook/test_project_importer.py +++ b/tests/import_hook/test_project_importer.py @@ -1,5 +1,4 @@ import os -import platform import re import shutil from pathlib import Path @@ -26,6 +25,7 @@ with_underscores, run_concurrent_python, check_match, + missing_entrypoint_error_message_pattern, ) """ @@ -203,10 +203,8 @@ def test_do_not_rebuild_if_installed_non_editable(workspace: Path, project_name: ) assert "SUCCESS" not in output3 assert "install_new_packages=True" in output3 - assert ( - f"ImportError: dynamic module does not define module " - f"export function (PyInit_{with_underscores(project_name)})" - ) in output3 + pattern = f"ImportError: {missing_entrypoint_error_message_pattern(with_underscores(project_name))}" + assert re.search(pattern, output3) is not None @pytest.mark.parametrize("initially_mixed", [False, True]) @@ -657,18 +655,10 @@ def test_successful_compilation_but_not_valid(self, workspace: Path, is_mixed: b lib_path.write_text(lib_path.read_text().replace("test_project", "test_project_new_name")) output, _ = run_python_code(self.loader_script, quiet=True) - - if platform.python_implementation() == "CPython": - error_message = "dynamic module does not define module export function \\(PyInit_test_project\\)" - elif platform.python_implementation() == "PyPy": - error_message = "function _cffi_pypyinit_test_project or PyInit_test_project not found in library .*" - else: - raise NotImplementedError(platform.python_implementation()) - pattern = ( 'building "test_project"\n' 'rebuilt and loaded package "test_project" in [0-9.]+s\n' - f"caught ImportError: {error_message}\n" + f"caught ImportError: {missing_entrypoint_error_message_pattern('test_project')}\n" ) check_match(output, pattern, flags=re.MULTILINE) diff --git a/tests/import_hook/test_rust_file_importer.py b/tests/import_hook/test_rust_file_importer.py index 9be719012..2a17f1022 100644 --- a/tests/import_hook/test_rust_file_importer.py +++ b/tests/import_hook/test_rust_file_importer.py @@ -14,6 +14,7 @@ test_crates, run_concurrent_python, check_match, + missing_entrypoint_error_message_pattern, ) """ @@ -329,6 +330,6 @@ def test_successful_compilation_but_not_valid(self, workspace: Path) -> None: pattern = ( 'building "my_script"\n' 'rebuilt and loaded module "my_script" in [0-9.]+s\n' - "caught ImportError: dynamic module does not define module export function \\(PyInit_my_script\\)\n" + f"caught ImportError: {missing_entrypoint_error_message_pattern('my_script')}\n" ) check_match(output, pattern, flags=re.MULTILINE) From 76b32315ab2ddd04b7251218cdba703f2ed305ba Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Wed, 3 Jan 2024 20:50:55 +0000 Subject: [PATCH 49/57] add windows support to test --- tests/import_hook/test_utilities.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/import_hook/test_utilities.py b/tests/import_hook/test_utilities.py index e4050af81..0f0af0bd1 100644 --- a/tests/import_hook/test_utilities.py +++ b/tests/import_hook/test_utilities.py @@ -297,12 +297,17 @@ def test_uri_to_path() -> None: def test_load_dist_info(tmp_path: Path) -> None: dist_info = tmp_path / "package_foo-1.0.0.dist-info" dist_info.mkdir(parents=True) - (dist_info / "direct_url.json").write_text( - '{"dir_info": {"editable": true}, "url": "file:///tmp/some%20directory/foo"}' - ) + if platform.system() == "Windows": + uri = "file:///C:/some%20directory/foo" + path = Path(r"C:\some directory\foo") + else: + uri = "file:///tmp/some%20directory/foo" + path = Path("/tmp/some directory/foo") + + (dist_info / "direct_url.json").write_text('{"dir_info": {"editable": true}, "url": "' + uri + '"}') linked_path, is_editable = _load_dist_info(tmp_path, "package_foo", require_project_target=False) - assert linked_path == Path("/tmp/some directory/foo") + assert linked_path == path assert is_editable From 584a79969cd0282392c9fa9cd49678ae5ce8ed46 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Thu, 4 Jan 2024 00:01:30 +0000 Subject: [PATCH 50/57] moved fixing of direct_url.json into maturin itself. Also refactored develop.rs --- Cargo.toml | 4 +- maturin/import_hook/project_importer.py | 28 --- src/develop.rs | 258 ++++++++++++++++++------ tests/import_hook/common.py | 4 +- 4 files changed, 197 insertions(+), 97 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e30968034..e0f86af62 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -75,6 +75,7 @@ path-slash = "0.2.1" pep440_rs = { version = "0.3.6", features = ["serde"] } pep508_rs = { version = "0.2.1", features = ["serde"] } time = "0.3.17" +url = "2.5.0" # cli clap = { version = "4.0.0", features = ["derive", "env", "wrap_help", "unstable-styles"] } @@ -104,7 +105,6 @@ rustls = { version = "0.21.9", optional = true } rustls-pemfile = { version = "2.0.0", optional = true } keyring = { version = "2.0.0", default-features = false, features = ["linux-no-secret-service"], optional = true } wild = { version = "2.1.0", optional = true } -url = { version = "2.5.0", optional = true } [dev-dependencies] expect-test = "1.4.1" @@ -124,7 +124,7 @@ log = ["tracing-subscriber"] cli-completion = ["dep:clap_complete_command"] -upload = ["ureq", "multipart", "configparser", "bytesize", "dialoguer/password", "url", "wild", "dep:dirs"] +upload = ["ureq", "multipart", "configparser", "bytesize", "dialoguer/password", "wild", "dep:dirs"] # keyring doesn't support *BSD so it's not enabled in `full` by default password-storage = ["upload", "keyring"] diff --git a/maturin/import_hook/project_importer.py b/maturin/import_hook/project_importer.py index 8e0aceef8..71d4401d2 100644 --- a/maturin/import_hook/project_importer.py +++ b/maturin/import_hook/project_importer.py @@ -175,7 +175,6 @@ def _rebuild_project( logger.info('building "%s"', package_name) start = time.perf_counter() maturin_output = develop_build_project(resolved.cargo_manifest_path, settings) - _fix_direct_url(project_dir, package_name) logger.debug( 'compiled project "%s" in %.3fs', package_name, @@ -347,33 +346,6 @@ def _uri_to_path(uri: str) -> Path: return Path(os.path.normpath(os.path.join(host, path))) -def _fix_direct_url(project_dir: Path, package_name: str) -> None: - """Seemingly due to a bug, installing with `pip install -e` will write the correct entry into `direct_url.json` to - point at the project directory, but calling `maturin develop` does not currently write this value correctly. - """ - logger.debug("fixing direct_url for %s", package_name) - for path in site.getsitepackages(): - dist_info = next(Path(path).glob(f"{package_name}-*.dist-info"), None) - if dist_info is None: - continue - direct_url_path = dist_info / "direct_url.json" - try: - with open(direct_url_path) as f: - direct_url = json.load(f) - except OSError: - continue - url = project_dir.as_uri() - if direct_url.get("url") != url: - logger.debug("fixing direct_url.json for package %s", package_name) - logger.debug('"%s" -> "%s"', direct_url.get("url"), url) - direct_url = {"dir_info": {"editable": True}, "url": url} - try: - with open(direct_url_path, "w") as f: - json.dump(direct_url, f) - except OSError: - return - - def _find_installed_package_root(resolved: MaturinProject, package_spec: ModuleSpec) -> Optional[Path]: """Find the root of the files that change each time the project is rebuilt: - for mixed projects: the root directory or file of the extension module inside the source tree diff --git a/src/develop.rs b/src/develop.rs index f6ed88a4f..ff1de80b5 100644 --- a/src/develop.rs +++ b/src/develop.rs @@ -1,5 +1,6 @@ use crate::build_options::CargoOptions; use crate::target::Arch; +use crate::BuildContext; use crate::BuildOptions; use crate::PlatformTag; use crate::PythonInterpreter; @@ -7,10 +8,13 @@ use crate::Target; use anyhow::{anyhow, bail, Context, Result}; use cargo_options::heading; use pep508_rs::{MarkerExpression, MarkerOperator, MarkerTree, MarkerValue}; +use regex::Regex; +use std::fs; use std::path::Path; use std::path::PathBuf; use std::process::Command; use tempfile::TempDir; +use url::Url; /// Install the crate as module in the current virtualenv #[derive(Debug, clap::Parser)] @@ -72,6 +76,142 @@ fn make_pip_command(python_path: &Path, pip_path: Option<&Path>) -> Command { } } +fn install_dependencies( + build_context: &BuildContext, + extras: &[String], + interpreter: &PythonInterpreter, + pip_path: Option<&Path>, +) -> Result<()> { + if !build_context.metadata21.requires_dist.is_empty() { + let mut args = vec!["install".to_string()]; + args.extend(build_context.metadata21.requires_dist.iter().map(|x| { + let mut pkg = x.clone(); + // Remove extra marker to make it installable with pip + // Keep in sync with `Metadata21::merge_pyproject_toml()`! + for extra in extras { + pkg.marker = pkg.marker.and_then(|marker| -> Option { + match marker.clone() { + MarkerTree::Expression(MarkerExpression { + l_value: MarkerValue::Extra, + operator: MarkerOperator::Equal, + r_value: MarkerValue::QuotedString(extra_value), + }) if &extra_value == extra => None, + MarkerTree::And(and) => match &*and { + [existing, MarkerTree::Expression(MarkerExpression { + l_value: MarkerValue::Extra, + operator: MarkerOperator::Equal, + r_value: MarkerValue::QuotedString(extra_value), + })] if extra_value == extra => Some(existing.clone()), + _ => Some(marker), + }, + _ => Some(marker), + } + }); + } + pkg.to_string() + })); + let status = make_pip_command(&interpreter.executable, pip_path) + .args(&args) + .status() + .context("Failed to run pip install")?; + if !status.success() { + bail!(r#"pip install finished with "{}""#, status) + } + } + Ok(()) +} + +fn pip_install_wheel( + build_context: &BuildContext, + python: &Path, + venv_dir: &Path, + pip_path: Option<&Path>, + wheel_filename: &Path, +) -> Result<()> { + let mut pip_cmd = make_pip_command(&python, pip_path); + let output = pip_cmd + .args(["install", "--no-deps", "--force-reinstall"]) + .arg(dunce::simplified(wheel_filename)) + .output() + .context(format!( + "pip install failed (ran {:?} with {:?})", + pip_cmd.get_program(), + &pip_cmd.get_args().collect::>(), + ))?; + if !output.status.success() { + bail!( + "pip install in {} failed running {:?}: {}\n--- Stdout:\n{}\n--- Stderr:\n{}\n---\n", + venv_dir.display(), + &pip_cmd.get_args().collect::>(), + output.status, + String::from_utf8_lossy(&output.stdout).trim(), + String::from_utf8_lossy(&output.stderr).trim(), + ); + } + if !output.stderr.is_empty() { + eprintln!( + "⚠️ Warning: pip raised a warning running {:?}:\n{}", + &pip_cmd.get_args().collect::>(), + String::from_utf8_lossy(&output.stderr).trim(), + ); + } + fix_direct_url(build_context, python, pip_path)?; + Ok(()) +} + +/// Each editable-installed python package has a direct_url.json file that includes a file:// URL +/// indicating the location of the source code of that project. The maturin import hook uses this +/// URL to locate and rebuild editable-installed projects. +/// +/// When a maturin package is installed using `pip install -e`, pip takes care of writing the +/// correct URL, however when a maturin package is installed with `maturin develop`, the URL is +/// set to the path to the temporary wheel file created during installation. +fn fix_direct_url( + build_context: &BuildContext, + python: &Path, + pip_path: Option<&Path>, +) -> Result<()> { + println!("✏️ Setting installed package as editable"); + let mut pip_cmd = make_pip_command(&python, pip_path); + let output = pip_cmd + .args(["show", "--files"]) + .arg(&build_context.metadata21.name) + .output() + .context(format!( + "pip show failed (ran {:?} with {:?})", + pip_cmd.get_program(), + &pip_cmd.get_args().collect::>(), + ))?; + if let Some(direct_url_path) = parse_direct_url_path(&String::from_utf8_lossy(&output.stdout))? + { + let project_dir = build_context + .pyproject_toml_path + .parent() + .ok_or_else(|| anyhow!("failed to get project directory"))?; + let uri = Url::from_file_path(project_dir) + .map_err(|_| anyhow!("failed to convert project directory to file URL"))?; + let content = format!("{{\"dir_info\": {{\"editable\": true}}, \"url\": \"{uri}\"}}"); + fs::write(direct_url_path, content)?; + } + Ok(()) +} + +fn parse_direct_url_path(pip_show_output: &str) -> Result> { + if let Some(Some(location)) = Regex::new(r"Location: (.*)")? + .captures(pip_show_output) + .map(|c| c.get(1)) + { + if let Some(Some(direct_url_path)) = Regex::new(r" (.*direct_url.json)")? + .captures(pip_show_output) + .map(|c| c.get(1)) + { + let absolute_path = PathBuf::from(location.as_str()).join(direct_url_path.as_str()); + return Ok(Some(absolute_path)); + } + } + Ok(None) +} + /// Installs a crate by compiling it and copying the shared library to site-packages. /// Also adds the dist-info directory to make sure pip and other tools detect the library /// @@ -137,74 +277,18 @@ pub fn develop(develop_options: DevelopOptions, venv_dir: &Path) -> Result<()> { || anyhow!("Expected `python` to be a python interpreter inside a virtualenv ಠ_ಠ"), )?; - // Install dependencies - if !build_context.metadata21.requires_dist.is_empty() { - let mut args = vec!["install".to_string()]; - args.extend(build_context.metadata21.requires_dist.iter().map(|x| { - let mut pkg = x.clone(); - // Remove extra marker to make it installable with pip - // Keep in sync with `Metadata21::merge_pyproject_toml()`! - for extra in &extras { - pkg.marker = pkg.marker.and_then(|marker| -> Option { - match marker.clone() { - MarkerTree::Expression(MarkerExpression { - l_value: MarkerValue::Extra, - operator: MarkerOperator::Equal, - r_value: MarkerValue::QuotedString(extra_value), - }) if &extra_value == extra => None, - MarkerTree::And(and) => match &*and { - [existing, MarkerTree::Expression(MarkerExpression { - l_value: MarkerValue::Extra, - operator: MarkerOperator::Equal, - r_value: MarkerValue::QuotedString(extra_value), - })] if extra_value == extra => Some(existing.clone()), - _ => Some(marker), - }, - _ => Some(marker), - } - }); - } - pkg.to_string() - })); - let status = make_pip_command(&interpreter.executable, pip_path.as_deref()) - .args(&args) - .status() - .context("Failed to run pip install")?; - if !status.success() { - bail!(r#"pip install finished with "{}""#, status) - } - } + install_dependencies(&build_context, &extras, &interpreter, pip_path.as_deref())?; let wheels = build_context.build_wheels()?; if !skip_install { for (filename, _supported_version) in wheels.iter() { - let mut pip_cmd = make_pip_command(&python, pip_path.as_deref()); - let output = pip_cmd - .args(["install", "--no-deps", "--force-reinstall"]) - .arg(dunce::simplified(filename)) - .output() - .context(format!( - "pip install failed (ran {:?} with {:?})", - pip_cmd.get_program(), - &pip_cmd.get_args().collect::>(), - ))?; - if !output.status.success() { - bail!( - "pip install in {} failed running {:?}: {}\n--- Stdout:\n{}\n--- Stderr:\n{}\n---\n", - venv_dir.display(), - &pip_cmd.get_args().collect::>(), - output.status, - String::from_utf8_lossy(&output.stdout).trim(), - String::from_utf8_lossy(&output.stderr).trim(), - ); - } - if !output.stderr.is_empty() { - eprintln!( - "⚠️ Warning: pip raised a warning running {:?}:\n{}", - &pip_cmd.get_args().collect::>(), - String::from_utf8_lossy(&output.stderr).trim(), - ); - } + pip_install_wheel( + &build_context, + &python, + venv_dir, + pip_path.as_deref(), + filename, + )?; eprintln!( "🛠 Installed {}-{}", build_context.metadata21.name, build_context.metadata21.version @@ -214,3 +298,49 @@ pub fn develop(develop_options: DevelopOptions, venv_dir: &Path) -> Result<()> { Ok(()) } + +#[cfg(test)] +mod test { + use std::path::PathBuf; + + use super::parse_direct_url_path; + + #[test] + fn test_parse_direct_url() { + let example_with_direct_url = "\ +Name: my-project +Version: 0.1.0 +Location: /foo bar/venv/lib/pythonABC/site-packages +Editable project location: /tmp/temporary.whl +Files: + my_project-0.1.0+abc123de.dist-info/INSTALLER + my_project-0.1.0+abc123de.dist-info/METADATA + my_project-0.1.0+abc123de.dist-info/RECORD + my_project-0.1.0+abc123de.dist-info/REQUESTED + my_project-0.1.0+abc123de.dist-info/WHEEL + my_project-0.1.0+abc123de.dist-info/direct_url.json + my_project-0.1.0+abc123de.dist-info/entry_points.txt + my_project.pth +"; + assert_eq!(parse_direct_url_path(example_with_direct_url).unwrap(), Some(PathBuf::from("/foo bar/venv/lib/pythonABC/site-packages/my_project-0.1.0.dist-info/direct_url.json"))); + + let example_without_direct_url = "\ +Name: my-project +Version: 0.1.0 +Location: /foo bar/venv/lib/pythonABC/site-packages +Files: + my_project-0.1.0+abc123de.dist-info/INSTALLER + my_project-0.1.0+abc123de.dist-info/METADATA + my_project-0.1.0+abc123de.dist-info/RECORD + my_project-0.1.0+abc123de.dist-info/REQUESTED + my_project-0.1.0+abc123de.dist-info/WHEEL + my_project-0.1.0+abc123de.dist-info/entry_points.txt + my_project.pth +"; + + assert_eq!( + parse_direct_url_path(example_without_direct_url).unwrap(), + None + ); + } +} diff --git a/tests/import_hook/common.py b/tests/import_hook/common.py index 4ca5570fa..638a4b44b 100644 --- a/tests/import_hook/common.py +++ b/tests/import_hook/common.py @@ -13,7 +13,7 @@ from pathlib import Path from typing import Iterable, List, Optional, Tuple, Callable, Any, Dict -from maturin.import_hook.project_importer import _fix_direct_url, _load_dist_info +from maturin.import_hook.project_importer import _load_dist_info verbose = True @@ -155,8 +155,6 @@ def install_editable(project_dir: Path) -> None: env = os.environ.copy() env["VIRTUAL_ENV"] = sys.exec_prefix subprocess.check_call(["maturin", "develop"], cwd=project_dir, env=env) - package_name = with_underscores(project_dir.name) - _fix_direct_url(project_dir, package_name) def install_non_editable(project_dir: Path) -> None: From 03e33046f0873a683e9e56c221ff9001ad259dff Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Thu, 4 Jan 2024 00:19:11 +0000 Subject: [PATCH 51/57] changed some notes and added ability to disable with an environment variable --- guide/src/develop.md | 18 +++++++++++++++++- guide/src/environment-variables.md | 5 +++++ maturin/import_hook/__init__.py | 9 ++++++++- 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/guide/src/develop.md b/guide/src/develop.md index f35284ff3..478671562 100644 --- a/guide/src/develop.md +++ b/guide/src/develop.md @@ -129,6 +129,11 @@ To install a package in editable mode with pip: cd my-project pip install -e . ``` +or +```bash +cd my-project +maturin develop +``` Then Python source code changes will take effect immediately because the interpreter looks for the modules directly in the project source tree. @@ -144,7 +149,7 @@ It supports importing editable-installed pure Rust and mixed Rust/Python project layouts as well as importing standalone `.rs` files. > **Note**: you must install maturin with the import-hook extra to be -> able to use the import hook: `pip install maturin[import-hook]` +> able to use the import hook: `pipx install maturin[import-hook]` ```python from maturin import import_hook @@ -196,6 +201,17 @@ into `site-packages/sitecustomize.py` of your development virtual environment enable the hook for every script run by that interpreter without calling `import_hook.install()` in every script, meaning the scripts do not need alteration before deployment. +### Environment +The import hook can be disabled by setting `MATURIN_IMPORT_HOOK_ENABLED=0`. This can be used to disable +the import hook in production if you want to leave calls to `import_hook.install()` in place. + +Build files will be stored in an appropriate place for the current system but can be overridden +by setting `MATURIN_BUILD_DIR`. The precedence for storing build files is: + +* `MATURIN_BUILD_DIR` +* `/maturin_build_cache` +* `/maturin_build_cache` + * e.g. `~/.cache/maturin_build_cache` on POSIX ### Advanced Usage diff --git a/guide/src/environment-variables.md b/guide/src/environment-variables.md index 270bf5419..a5301d325 100644 --- a/guide/src/environment-variables.md +++ b/guide/src/environment-variables.md @@ -15,6 +15,11 @@ See [environment variables Cargo reads](https://doc.rust-lang.org/cargo/referenc * `MATURIN_PYPI_TOKEN`: PyPI token for uploading wheels * `MATURIN_PASSWORD`: PyPI password for uploading wheels +## Import hook environment variables + +* `MATURIN_BUILD_DIR`: Path to a location to cache build files +* `MATURIN_IMPORT_HOOK_ENABLED`: set to `0` to disable calls to `import_hook.install()` + ## `pyo3` environment variables * `PYO3_CROSS_PYTHON_VERSION`: Python version to use for cross compilation diff --git a/maturin/import_hook/__init__.py b/maturin/import_hook/__init__.py index dcc6aec4e..97f65b71f 100644 --- a/maturin/import_hook/__init__.py +++ b/maturin/import_hook/__init__.py @@ -1,8 +1,9 @@ +import os from pathlib import Path from typing import Optional, Set from maturin.import_hook import project_importer, rust_file_importer -from maturin.import_hook._logging import reset_logger +from maturin.import_hook._logging import logger, reset_logger from maturin.import_hook.settings import MaturinSettings __all__ = ["install", "uninstall", "reset_logger"] @@ -22,6 +23,8 @@ def install( ) -> None: """Install import hooks for automatically rebuilding and importing maturin projects or .rs files. + see the guide at for more details + :param enable_project_importer: enable the hook for automatically rebuilding editable installed maturin projects :param enable_rs_file_importer: enable the hook for importing .rs files as though they were regular python modules @@ -47,6 +50,10 @@ def install( :param show_warnings: whether to show compilation warnings """ + if os.environ.get("MATURIN_IMPORT_HOOK_ENABLED") == "0": + logger.info("maturin import hook disabled by environment variable") + return + if enable_rs_file_importer: rust_file_importer.install( settings=settings, From 72d9f32b59b57d3b2a67ccecf85fcc6d1ea7ad18 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Thu, 4 Jan 2024 00:19:58 +0000 Subject: [PATCH 52/57] renamed document to match title --- guide/src/{develop.md => local_development.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename guide/src/{develop.md => local_development.md} (100%) diff --git a/guide/src/develop.md b/guide/src/local_development.md similarity index 100% rename from guide/src/develop.md rename to guide/src/local_development.md From 6d1b3a11c8458b71e4d4c6aeafe504aba9137820 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Thu, 4 Jan 2024 00:28:42 +0000 Subject: [PATCH 53/57] small fixes --- src/develop.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/develop.rs b/src/develop.rs index ff1de80b5..7642e12a5 100644 --- a/src/develop.rs +++ b/src/develop.rs @@ -128,7 +128,7 @@ fn pip_install_wheel( pip_path: Option<&Path>, wheel_filename: &Path, ) -> Result<()> { - let mut pip_cmd = make_pip_command(&python, pip_path); + let mut pip_cmd = make_pip_command(python, pip_path); let output = pip_cmd .args(["install", "--no-deps", "--force-reinstall"]) .arg(dunce::simplified(wheel_filename)) @@ -172,7 +172,7 @@ fn fix_direct_url( pip_path: Option<&Path>, ) -> Result<()> { println!("✏️ Setting installed package as editable"); - let mut pip_cmd = make_pip_command(&python, pip_path); + let mut pip_cmd = make_pip_command(python, pip_path); let output = pip_cmd .args(["show", "--files"]) .arg(&build_context.metadata21.name) @@ -322,7 +322,7 @@ Files: my_project-0.1.0+abc123de.dist-info/entry_points.txt my_project.pth "; - assert_eq!(parse_direct_url_path(example_with_direct_url).unwrap(), Some(PathBuf::from("/foo bar/venv/lib/pythonABC/site-packages/my_project-0.1.0.dist-info/direct_url.json"))); + assert_eq!(parse_direct_url_path(example_with_direct_url).unwrap(), Some(PathBuf::from("/foo bar/venv/lib/pythonABC/site-packages/my_project-0.1.0+abc123de.dist-info/direct_url.json"))); let example_without_direct_url = "\ Name: my-project From 01685a212f42df0a4a708cbee8aa7908691e9ff4 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Thu, 4 Jan 2024 21:49:50 +0000 Subject: [PATCH 54/57] added some documentation --- guide/src/local_development.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/guide/src/local_development.md b/guide/src/local_development.md index 478671562..403bf852f 100644 --- a/guide/src/local_development.md +++ b/guide/src/local_development.md @@ -149,12 +149,13 @@ It supports importing editable-installed pure Rust and mixed Rust/Python project layouts as well as importing standalone `.rs` files. > **Note**: you must install maturin with the import-hook extra to be -> able to use the import hook: `pipx install maturin[import-hook]` +> able to use the import hook: `pip install maturin[import-hook]` ```python from maturin import import_hook -# install the import hook with default settings +# install the import hook with default settings. +# this must be called before importing any maturin project import_hook.install() # when a rust package that is installed in editable mode is imported, @@ -172,6 +173,7 @@ The maturin project importer and the rust file importer can be used separately ```python from maturin.import_hook import rust_file_importer rust_file_importer.install() + from maturin.import_hook import project_importer project_importer.install() ``` @@ -206,7 +208,8 @@ The import hook can be disabled by setting `MATURIN_IMPORT_HOOK_ENABLED=0`. This the import hook in production if you want to leave calls to `import_hook.install()` in place. Build files will be stored in an appropriate place for the current system but can be overridden -by setting `MATURIN_BUILD_DIR`. The precedence for storing build files is: +by setting `MATURIN_BUILD_DIR`. These files can be deleted without causing any issues (unless a build is in progress). +The precedence for storing build files is: * `MATURIN_BUILD_DIR` * `/maturin_build_cache` From 9cc276c219342d49d16a4ba68c5eed7b81c99171 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Thu, 4 Jan 2024 21:50:46 +0000 Subject: [PATCH 55/57] support windows style paths when fixing direct_url.json --- src/develop.rs | 39 +++++++++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/src/develop.rs b/src/develop.rs index 7642e12a5..aed9d2c28 100644 --- a/src/develop.rs +++ b/src/develop.rs @@ -197,7 +197,7 @@ fn fix_direct_url( } fn parse_direct_url_path(pip_show_output: &str) -> Result> { - if let Some(Some(location)) = Regex::new(r"Location: (.*)")? + if let Some(Some(location)) = Regex::new(r"Location: ([^\r\n]*)")? .captures(pip_show_output) .map(|c| c.get(1)) { @@ -205,8 +205,9 @@ fn parse_direct_url_path(pip_show_output: &str) -> Result> { .captures(pip_show_output) .map(|c| c.get(1)) { - let absolute_path = PathBuf::from(location.as_str()).join(direct_url_path.as_str()); - return Ok(Some(absolute_path)); + return Ok(Some( + PathBuf::from(location.as_str()).join(direct_url_path.as_str()), + )); } } Ok(None) @@ -306,6 +307,7 @@ mod test { use super::parse_direct_url_path; #[test] + #[cfg(not(target_os = "windows"))] fn test_parse_direct_url() { let example_with_direct_url = "\ Name: my-project @@ -322,7 +324,11 @@ Files: my_project-0.1.0+abc123de.dist-info/entry_points.txt my_project.pth "; - assert_eq!(parse_direct_url_path(example_with_direct_url).unwrap(), Some(PathBuf::from("/foo bar/venv/lib/pythonABC/site-packages/my_project-0.1.0+abc123de.dist-info/direct_url.json"))); + let expected_path = PathBuf::from("/foo bar/venv/lib/pythonABC/site-packages/my_project-0.1.0+abc123de.dist-info/direct_url.json"); + assert_eq!( + parse_direct_url_path(example_with_direct_url).unwrap(), + Some(expected_path) + ); let example_without_direct_url = "\ Name: my-project @@ -343,4 +349,29 @@ Files: None ); } + + #[test] + #[cfg(target_os = "windows")] + fn test_parse_direct_url_windows() { + let example_with_direct_url_windows = "\ +Name: my-project\r +Version: 0.1.0\r +Location: C:\\foo bar\\venv\\Lib\\site-packages\r +Files:\r + my_project-0.1.0+abc123de.dist-info\\INSTALLER\r + my_project-0.1.0+abc123de.dist-info\\METADATA\r + my_project-0.1.0+abc123de.dist-info\\RECORD\r + my_project-0.1.0+abc123de.dist-info\\REQUESTED\r + my_project-0.1.0+abc123de.dist-info\\WHEEL\r + my_project-0.1.0+abc123de.dist-info\\direct_url.json\r + my_project-0.1.0+abc123de.dist-info\\entry_points.txt\r + my_project.pth\r +"; + + let expected_path = PathBuf::from("C:\\foo bar\\venv\\Lib\\site-packages\\my_project-0.1.0+abc123de.dist-info\\direct_url.json"); + assert_eq!( + parse_direct_url_path(example_with_direct_url_windows).unwrap(), + Some(expected_path) + ); + } } From 7a2302cbe0dbce2bd69cf1def0f924068fd0217d Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Thu, 4 Jan 2024 22:55:57 +0000 Subject: [PATCH 56/57] improved logging in utility tests --- maturin/import_hook/project_importer.py | 12 ++++++++---- tests/common/import_hook.rs | 8 +++++++- tests/import_hook/test_utilities.py | 5 ++++- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/maturin/import_hook/project_importer.py b/maturin/import_hook/project_importer.py index 71d4401d2..b7ab240a6 100644 --- a/maturin/import_hook/project_importer.py +++ b/maturin/import_hook/project_importer.py @@ -395,13 +395,13 @@ def _get_project_mtime( excluded_dirs.add(installed_package_root) try: - return max( - path.stat().st_mtime - for path in _get_files_in_dirs( + latest_path = max( + _get_files_in_dirs( itertools.chain((project_dir,), all_path_dependencies), excluded_dir_names, excluded_dirs, - ) + ), + key=lambda p: p.stat().st_mtime, ) except FileNotFoundError as e: logger.debug("error getting project mtime: %r (%s)", e, e.filename) @@ -409,6 +409,10 @@ def _get_project_mtime( except ValueError as e: logger.debug("error getting project mtime: %r", e) return None + else: + mtime = latest_path.stat().st_mtime + logger.debug("most recently written path: '%s' at %f", latest_path, mtime) + return mtime def _package_is_up_to_date( diff --git a/tests/common/import_hook.rs b/tests/common/import_hook.rs index 447ddf452..f5cc7336c 100644 --- a/tests/common/import_hook.rs +++ b/tests/common/import_hook.rs @@ -59,7 +59,13 @@ pub fn test_import_hook( let path = env::join_paths(paths).unwrap(); let output = Command::new(&python) - .args(["-m", "pytest", test_specifier]) + .args([ + "-m", + "pytest", + "--show-capture=all", + "--log-level=DEBUG", + test_specifier, + ]) .env("PATH", path) .env("VIRTUAL_ENV", venv_dir) .envs(extra_envs.iter().cloned()) diff --git a/tests/import_hook/test_utilities.py b/tests/import_hook/test_utilities.py index 0f0af0bd1..b4b5f9380 100644 --- a/tests/import_hook/test_utilities.py +++ b/tests/import_hook/test_utilities.py @@ -9,7 +9,7 @@ import pytest -from maturin.import_hook import MaturinSettings +from maturin.import_hook import MaturinSettings, reset_logger from maturin.import_hook._building import BuildCache, BuildStatus from maturin.import_hook._resolve_project import ProjectResolveError, _resolve_project from maturin.import_hook.project_importer import ( @@ -31,6 +31,9 @@ os.environ["RESOLVED_PACKAGES_PATH"] = str(SAVED_RESOLVED_PACKAGES_PATH) +reset_logger() + + def test_settings() -> None: assert MaturinSettings().to_args() == [] assert MaturinBuildSettings().to_args() == [] From 18b03d1318bae9b8e7c214dd593dd3933c956b2a Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Thu, 4 Jan 2024 22:56:12 +0000 Subject: [PATCH 57/57] hopefully fix problem causing flaky test --- tests/import_hook/test_utilities.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/import_hook/test_utilities.py b/tests/import_hook/test_utilities.py index b4b5f9380..05c34d2f2 100644 --- a/tests/import_hook/test_utilities.py +++ b/tests/import_hook/test_utilities.py @@ -157,6 +157,7 @@ def test_simple_path_dep(self, tmp_path: Path) -> None: extension_module.touch() _small_sleep() (project_b / "source").touch() + _small_sleep() project_mtime = _get_project_mtime(project_a, [project_b], extension_module, set()) assert project_mtime == (project_b / "source").stat().st_mtime