From 3aea84ddf0968dcd11295fab38ba540c74a4f176 Mon Sep 17 00:00:00 2001 From: "Philipp A." Date: Mon, 18 Nov 2024 20:57:22 +0100 Subject: [PATCH] Add stubs (#38) --- .gitignore | 93 ++++++++----------------------------------- Cargo.lock | 93 +++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 5 ++- build.rs | 32 +++++++++++++++ docs/contributing.md | 10 +++++ pyproject.toml | 5 ++- python/zarrs/py.typed | 0 src/bin/stub_gen.rs | 7 ++++ src/lib.rs | 6 +++ 9 files changed, 170 insertions(+), 81 deletions(-) create mode 100644 build.rs create mode 100644 python/zarrs/py.typed create mode 100644 src/bin/stub_gen.rs diff --git a/.gitignore b/.gitignore index 517f163..1c5b5ab 100644 --- a/.gitignore +++ b/.gitignore @@ -1,81 +1,20 @@ -/target +# IDEs +/.idea/ +/.vscode/ -# Byte-compiled / optimized / DLL files +# Caches +.DS_Store __pycache__/ -.pytest_cache/ -*.py[cod] +/.*cache/ +/.hypothesis/ -# C extensions +# Build *.so - -.hypothesis/ - -# docs generated rst -docs/generated - -# Distribution / packaging -.Python -.venv/ -env/ -bin/ -build/ -develop-eggs/ -dist/ -eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -include/ -man/ -venv/ -*.egg-info/ -.installed.cfg -*.egg - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt -pip-selfcheck.json - -# Unit test / coverage reports -htmlcov/ -.tox/ -.coverage -.cache -nosetests.xml -coverage.xml - -# Translations -*.mo - -# Mr Developer -.mr.developer.cfg -.project -.pydevproject - -# Rope -.ropeproject - -# Django stuff: -*.log -*.pot - -.DS_Store - -# Sphinx documentation -docs/_build/ - -# PyCharm -.idea/ - -# VSCode -.vscode/ - -# Pyenv -.python-version - -# testing -data/ -fixture/ \ No newline at end of file +*.pyi +/target/ +/dist/ +/docs/_build/ + +# Coverage +/.coverage +/coverage.xml diff --git a/Cargo.lock b/Cargo.lock index 49009c2..5b08288 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,6 +14,12 @@ version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "45862d1c77f2228b9e10bc609d5bc203d86ebc9b87ad8d5d5167a6c9abf739d9" +[[package]] +name = "anyhow" +version = "1.0.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775" + [[package]] name = "autocfg" version = "1.4.0" @@ -366,6 +372,12 @@ dependencies = [ "libc", ] +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + [[package]] name = "matrixmultiply" version = "0.3.9" @@ -731,6 +743,34 @@ dependencies = [ "syn", ] +[[package]] +name = "pyo3-stub-gen" +version = "0.6.1" +source = "git+https://github.com/flying-sheep/pyo3-stub-gen.git?branch=py-untyped-array#ab44d97087a9622eabc07e36eb605987f0258a1b" +dependencies = [ + "anyhow", + "inventory", + "itertools", + "log", + "maplit", + "num-complex", + "numpy", + "pyo3", + "pyo3-stub-gen-derive", + "serde", + "toml", +] + +[[package]] +name = "pyo3-stub-gen-derive" +version = "0.6.1" +source = "git+https://github.com/flying-sheep/pyo3-stub-gen.git?branch=py-untyped-array#ab44d97087a9622eabc07e36eb605987f0258a1b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "quanta" version = "0.12.3" @@ -894,6 +934,15 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_spanned" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" +dependencies = [ + "serde", +] + [[package]] name = "shlex" version = "1.3.0" @@ -989,6 +1038,40 @@ dependencies = [ "once_cell", ] +[[package]] +name = "toml" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "triomphe" version = "0.1.11" @@ -1219,6 +1302,15 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "0.6.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" +dependencies = [ + "memchr", +] + [[package]] name = "zarrs" version = "0.18.0-beta.0" @@ -1259,6 +1351,7 @@ dependencies = [ "numpy", "openssl", "pyo3", + "pyo3-stub-gen", "rayon", "rayon_iter_concurrent_limit", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index 674a2d9..ed9615c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,8 +5,8 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [lib] -name = "zarrs" -crate-type = ["cdylib"] +name = "zarrs_python" +crate-type = ["cdylib", "rlib"] [dependencies] pyo3 = "0.22.6" @@ -18,6 +18,7 @@ openssl = { version = "0.10", features = ["vendored"] } numpy = "0.22.1" unsafe_cell_slice = "0.2.0" serde_json = "1.0.128" +pyo3-stub-gen = { version = "0.6.1", git = "https://github.com/flying-sheep/pyo3-stub-gen.git", branch = "py-untyped-array" } [profile.release] lto = true diff --git a/build.rs b/build.rs new file mode 100644 index 0000000..7e64a81 --- /dev/null +++ b/build.rs @@ -0,0 +1,32 @@ +use std::env; +use std::path::PathBuf; +use std::process::Command; + +fn main() -> Result<(), Box> { + maybe_generate_stubs()?; + Ok(()) +} + +/// Generate stubs if we’re compiling the library +/// Returns `true` if stubs were generated +fn maybe_generate_stubs() -> Result> { + if env::var("CARGO_BIN_NAME") != Err(env::VarError::NotPresent) { + return Ok(false); + } + + // Find an existing `stub_gen` binary or exit silently + let Some(bin_path) = ["debug", "release"] + .into_iter() + .filter_map(|mode| { + let p = PathBuf::from(format!("target/{mode}/stub_gen")); + p.exists().then_some(p) + }) + .next() + else { + return Ok(false); + }; + + // If we’re compiling the library, generate stubs first + Command::new(bin_path).spawn()?.wait()?; + Ok(true) +} diff --git a/docs/contributing.md b/docs/contributing.md index a6eacbc..3f7b3e4 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -29,3 +29,13 @@ pytest -n auto ``` for parallelized tests. Most tests have been copied from the `zarr-python` repository with the exception of `test_pipeline.py` which we have written. + +## Type hints + +When authoring Python code, your IDE will not be able to analyze the extension module `zarrs._internal`. +But thanks to [`pyo3-stub-gen`][], we can generate type stubs for it! + +To build the stub generator, run `cargo build --bin stub_gen`. +Afterwards, whenever you `cargo build`, `maturin build` or interact with your editor’s rust language server (e.g. `rust-analyzer`), the type hints will be updated. + +[`pyo3-stub-gen`]: https://github.com/Jij-Inc/pyo3-stub-gen diff --git a/pyproject.toml b/pyproject.toml index 96e0ad5..f63a6a9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,7 @@ classifiers = [ "Programming Language :: Rust", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", + "Typing :: Typed", ] dynamic = ["version"] dependencies = [ @@ -41,9 +42,9 @@ test = [ "requests", "mypy", "hypothesis", - "pytest-xdist" + "pytest-xdist", ] -dev = ["maturin"] +dev = ["maturin", "pip"] doc = ["sphinx>=7.4.6", "myst-parser"] [tool.maturin] diff --git a/python/zarrs/py.typed b/python/zarrs/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/src/bin/stub_gen.rs b/src/bin/stub_gen.rs new file mode 100644 index 0000000..9b43a09 --- /dev/null +++ b/src/bin/stub_gen.rs @@ -0,0 +1,7 @@ +use pyo3_stub_gen::Result; + +fn main() -> Result<()> { + let stub = zarrs_python::stub_info()?; + stub.generate()?; + Ok(()) +} diff --git a/src/lib.rs b/src/lib.rs index 60d43a4..8cad80a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,6 +6,8 @@ use numpy::npyffi::PyArrayObject; use numpy::{IntoPyArray, PyArray1, PyUntypedArray, PyUntypedArrayMethods}; use pyo3::exceptions::{PyRuntimeError, PyTypeError, PyValueError}; use pyo3::prelude::*; +use pyo3_stub_gen::define_stub_info_gatherer; +use pyo3_stub_gen::derive::{gen_stub_pyclass, gen_stub_pymethods}; use rayon::iter::{IntoParallelIterator, ParallelIterator}; use rayon_iter_concurrent_limit::iter_concurrent_limit; use std::borrow::Cow; @@ -37,6 +39,7 @@ trait CodecPipelineStore: Send + Sync { } // TODO: Use a OnceLock for store with get_or_try_init when stabilised? +#[gen_stub_pyclass] #[pyclass] pub struct CodecPipelineImpl { pub(crate) codec_chain: Arc, @@ -234,6 +237,7 @@ impl CodecPipelineImpl { } } +#[gen_stub_pymethods] #[pymethods] impl CodecPipelineImpl { #[pyo3(signature = ( @@ -531,3 +535,5 @@ fn _internal(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; Ok(()) } + +define_stub_info_gatherer!(stub_info);