From d03ba88effe69516eeb97182ad8006d896cbd423 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Mon, 3 Jul 2023 21:20:49 +0100 Subject: [PATCH 01/57] ci: run valgrind and careful with 'CI-build-full' label --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4bbd8aa46c3..ef597ead739 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -254,7 +254,7 @@ jobs: extra-features: "multiple-pymethods" valgrind: - if: ${{ github.event_name != 'pull_request' && github.ref != 'refs/heads/main' }} + if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || (github.event_name != 'pull_request' && github.ref != 'refs/heads/main') }} needs: [fmt] runs-on: ubuntu-latest steps: @@ -274,7 +274,7 @@ jobs: TRYBUILD: overwrite careful: - if: ${{ github.event_name != 'pull_request' && github.ref != 'refs/heads/main' }} + if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || (github.event_name != 'pull_request' && github.ref != 'refs/heads/main') }} needs: [fmt] runs-on: ubuntu-latest steps: From f5a8c25d2c988b58abb11d63f87139e5abb21ce1 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Mon, 3 Jul 2023 21:36:17 +0100 Subject: [PATCH 02/57] add some missing type conversions to the guide --- guide/src/conversions/tables.md | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/guide/src/conversions/tables.md b/guide/src/conversions/tables.md index 911a9869973..01a5c5a3cfc 100644 --- a/guide/src/conversions/tables.md +++ b/guide/src/conversions/tables.md @@ -16,28 +16,31 @@ The table below contains the Python type and the corresponding function argument | `str` | `String`, `Cow`, `&str`, `OsString`, `PathBuf`, `Path` | `&PyString`, `&PyUnicode` | | `bytes` | `Vec`, `&[u8]`, `Cow<[u8]>` | `&PyBytes` | | `bool` | `bool` | `&PyBool` | -| `int` | Any integer type (`i32`, `u32`, `usize`, etc) | `&PyLong` | +| `int` | `i8`, `u8`, `i16`, `u16`, `i32`, `u32`, `i64`, `u64`, `i128`, `u128`, `isize`, `usize`, `num_bigint::BigInt`[^1], `num_bigint::BigUint`[^1] | `&PyLong` | | `float` | `f32`, `f64` | `&PyFloat` | -| `complex` | `num_complex::Complex`[^1] | `&PyComplex` | +| `complex` | `num_complex::Complex`[^2] | `&PyComplex` | | `list[T]` | `Vec` | `&PyList` | -| `dict[K, V]` | `HashMap`, `BTreeMap`, `hashbrown::HashMap`[^2], `indexmap::IndexMap`[^3] | `&PyDict` | +| `dict[K, V]` | `HashMap`, `BTreeMap`, `hashbrown::HashMap`[^3], `indexmap::IndexMap`[^4] | `&PyDict` | | `tuple[T, U]` | `(T, U)`, `Vec` | `&PyTuple` | -| `set[T]` | `HashSet`, `BTreeSet`, `hashbrown::HashSet`[^2] | `&PySet` | -| `frozenset[T]` | `HashSet`, `BTreeSet`, `hashbrown::HashSet`[^2] | `&PyFrozenSet` | +| `set[T]` | `HashSet`, `BTreeSet`, `hashbrown::HashSet`[^3] | `&PySet` | +| `frozenset[T]` | `HashSet`, `BTreeSet`, `hashbrown::HashSet`[^3] | `&PyFrozenSet` | | `bytearray` | `Vec`, `Cow<[u8]>` | `&PyByteArray` | | `slice` | - | `&PySlice` | | `type` | - | `&PyType` | | `module` | - | `&PyModule` | -| `pathlib.Path` | `PathBuf`, `Path` | `&PyString`, `&PyUnicode` | +| `collections.abc.Buffer` | - | `PyBuffer` | | `datetime.datetime` | - | `&PyDateTime` | | `datetime.date` | - | `&PyDate` | | `datetime.time` | - | `&PyTime` | | `datetime.tzinfo` | - | `&PyTzInfo` | | `datetime.timedelta` | - | `&PyDelta` | -| `collections.abc.Buffer` | - | `PyBuffer` | +| `decimal.Decimal` | `rust_decimal::Decimal`[^5] | - | +| `ipaddress.IPv4Address` | `std::net::IpAddr`, `std::net::IpV4Addr` | - | +| `ipaddress.IPv6Address` | `std::net::IpAddr`, `std::net::IpV6Addr` | - | +| `pathlib.Path` | `PathBuf`, `Path` | `&PyString`, `&PyUnicode` | | `typing.Optional[T]` | `Option` | - | | `typing.Sequence[T]` | `Vec` | `&PySequence` | -| `typing.Mapping[K, V]` | `HashMap`, `BTreeMap`, `hashbrown::HashMap`[^2], `indexmap::IndexMap`[^3] | `&PyMapping` | +| `typing.Mapping[K, V]` | `HashMap`, `BTreeMap`, `hashbrown::HashMap`[^3], `indexmap::IndexMap`[^4] | `&PyMapping` | | `typing.Iterator[Any]` | - | `&PyIterator` | | `typing.Union[...]` | See [`#[derive(FromPyObject)]`](traits.html#deriving-a-hrefhttpsdocsrspyo3latestpyo3conversiontraitfrompyobjecthtmlfrompyobjecta-for-enums) | - | @@ -95,8 +98,12 @@ Finally, the following Rust types are also able to convert to Python as return v | `PyRef` | `T` | | `PyRefMut` | `T` | -[^1]: Requires the `num-complex` optional feature. +[^1]: Requires the `num-bigint` optional feature. + +[^2]: Requires the `num-complex` optional feature. + +[^3]: Requires the `hashbrown` optional feature. -[^2]: Requires the `hashbrown` optional feature. +[^4]: Requires the `indexmap` optional feature. -[^3]: Requires the `indexmap` optional feature. +[^5]: Requires the `rust_decimal` optional feature. From 8e588efba786cbbe3b15021e9dd9c616b08cf5de Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Tue, 4 Jul 2023 07:01:11 +0100 Subject: [PATCH 03/57] add PyState_*Module definitions for PyPy --- newsfragments/3295.added.md | 1 + pyo3-ffi/src/pystate.rs | 12 ++++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 newsfragments/3295.added.md diff --git a/newsfragments/3295.added.md b/newsfragments/3295.added.md new file mode 100644 index 00000000000..82987455381 --- /dev/null +++ b/newsfragments/3295.added.md @@ -0,0 +1 @@ +Add FFI definitions `PyState_AddModule`, `PyState_RemoveModule` and `PyState_FindModule` for PyPy 3.9 and up. diff --git a/pyo3-ffi/src/pystate.rs b/pyo3-ffi/src/pystate.rs index 8bae6652d07..d2fd39e497d 100644 --- a/pyo3-ffi/src/pystate.rs +++ b/pyo3-ffi/src/pystate.rs @@ -1,4 +1,4 @@ -#[cfg(not(PyPy))] +#[cfg(any(not(PyPy), Py_3_9))] use crate::moduleobject::PyModuleDef; use crate::object::PyObject; use std::os::raw::c_int; @@ -28,13 +28,17 @@ extern "C" { #[cfg(not(PyPy))] pub fn PyInterpreterState_GetID(arg1: *mut PyInterpreterState) -> i64; - #[cfg(not(PyPy))] + #[cfg(any(not(PyPy), Py_3_9))] // only on PyPy since 3.9 + #[cfg_attr(PyPy, link_name = "PyPyState_AddModule")] pub fn PyState_AddModule(arg1: *mut PyObject, arg2: *mut PyModuleDef) -> c_int; - #[cfg(not(PyPy))] + #[cfg(any(not(PyPy), Py_3_9))] // only on PyPy since 3.9 + #[cfg_attr(PyPy, link_name = "PyPyState_RemoveModule")] pub fn PyState_RemoveModule(arg1: *mut PyModuleDef) -> c_int; - #[cfg(not(PyPy))] + #[cfg(any(not(PyPy), Py_3_9))] // only on PyPy since 3.9 + // only has PyPy prefix since 3.10 + #[cfg_attr(all(PyPy, Py_3_10), link_name = "PyPyState_FindModule")] pub fn PyState_FindModule(arg1: *mut PyModuleDef) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyThreadState_New")] From fd28ca89a3771a3b6acb3c01ea7b0b4035cda0ad Mon Sep 17 00:00:00 2001 From: Grant Slatton Date: Sat, 24 Jun 2023 08:43:37 -0700 Subject: [PATCH 04/57] Fix fixed offset timezone conversion bug. See https://github.com/PyO3/pyo3/issues/3267 --- newsfragments/3269.fixed.md | 1 + src/conversions/chrono.rs | 102 ++++++++++++++++++++++-------------- 2 files changed, 64 insertions(+), 39 deletions(-) create mode 100644 newsfragments/3269.fixed.md diff --git a/newsfragments/3269.fixed.md b/newsfragments/3269.fixed.md new file mode 100644 index 00000000000..0f1e1af7b80 --- /dev/null +++ b/newsfragments/3269.fixed.md @@ -0,0 +1 @@ +Fix timezone conversion bug for FixedOffset datetimes that were being incorrectly converted to and from UTC. diff --git a/src/conversions/chrono.rs b/src/conversions/chrono.rs index 374fc161763..ae43ed610ef 100644 --- a/src/conversions/chrono.rs +++ b/src/conversions/chrono.rs @@ -221,8 +221,8 @@ impl FromPyObject<'_> for NaiveDateTime { impl ToPyObject for DateTime { fn to_object(&self, py: Python<'_>) -> PyObject { - let date = self.naive_utc().date(); - let time = self.naive_utc().time(); + let date = self.naive_local().date(); + let time = self.naive_local().time(); let yy = date.year(); let mm = date.month() as u8; let dd = date.day() as u8; @@ -251,7 +251,7 @@ impl IntoPy for DateTime { impl FromPyObject<'_> for DateTime { fn extract(ob: &PyAny) -> PyResult> { let dt: &PyDateTime = ob.downcast()?; - let ms = dt.get_fold() as u32 * 1_000_000 + dt.get_microsecond(); + let ms = dt.get_microsecond(); let h = dt.get_hour().into(); let m = dt.get_minute().into(); let s = dt.get_second().into(); @@ -266,7 +266,8 @@ impl FromPyObject<'_> for DateTime { NaiveTime::from_hms_micro_opt(h, m, s, ms) .ok_or_else(|| PyValueError::new_err("invalid or out-of-range time"))?, ); - Ok(DateTime::from_utc(dt, tz)) + // `FixedOffset` cannot have ambiguities so we don't have to worry about DST folds and such + Ok(DateTime::from_local(dt, tz)) } } @@ -612,7 +613,7 @@ mod tests { .and_hms_micro_opt(hour, minute, ssecond, ms) .unwrap(); let datetime = - DateTime::::from_utc(datetime, offset).to_object(py); + DateTime::::from_local(datetime, offset).to_object(py); let datetime: &PyDateTime = datetime.extract(py).unwrap(); let py_tz = offset.to_object(py); let py_tz = py_tz.downcast(py).unwrap(); @@ -681,41 +682,36 @@ mod tests { check_utc("fold", 2014, 5, 6, 7, 8, 9, 1_999_999, 999_999, true); check_utc("non fold", 2014, 5, 6, 7, 8, 9, 999_999, 999_999, false); - let check_fixed_offset = - |name: &'static str, year, month, day, hour, minute, second, ms, py_ms, fold| { - Python::with_gil(|py| { - let offset = FixedOffset::east_opt(3600).unwrap(); - let py_tz = offset.to_object(py); - let py_tz = py_tz.downcast(py).unwrap(); - let py_datetime = PyDateTime::new_with_fold( - py, - year, - month as u8, - day as u8, - hour as u8, - minute as u8, - second as u8, - py_ms, - Some(py_tz), - fold, - ) + let check_fixed_offset = |year, month, day, hour, minute, second, ms| { + Python::with_gil(|py| { + let offset = FixedOffset::east_opt(3600).unwrap(); + let py_tz = offset.to_object(py); + let py_tz = py_tz.downcast(py).unwrap(); + let py_datetime = PyDateTime::new_with_fold( + py, + year, + month as u8, + day as u8, + hour as u8, + minute as u8, + second as u8, + ms, + Some(py_tz), + false, // No such thing as fold for fixed offset timezones + ) + .unwrap(); + let py_datetime: DateTime = py_datetime.extract().unwrap(); + let datetime = NaiveDate::from_ymd_opt(year, month, day) + .unwrap() + .and_hms_micro_opt(hour, minute, second, ms) .unwrap(); - let py_datetime: DateTime = py_datetime.extract().unwrap(); - let datetime = NaiveDate::from_ymd_opt(year, month, day) - .unwrap() - .and_hms_micro_opt(hour, minute, second, ms) - .unwrap(); - let datetime = DateTime::::from_utc(datetime, offset); - assert_eq!( - py_datetime, datetime, - "{}: {} != {}", - name, datetime, py_datetime - ); - }) - }; + let datetime = DateTime::::from_local(datetime, offset); - check_fixed_offset("fold", 2014, 5, 6, 7, 8, 9, 1_999_999, 999_999, true); - check_fixed_offset("non fold", 2014, 5, 6, 7, 8, 9, 999_999, 999_999, false); + assert_eq!(py_datetime, datetime, "{} != {}", datetime, py_datetime); + }) + }; + + check_fixed_offset(2014, 5, 6, 7, 8, 9, 999_999); Python::with_gil(|py| { let py_tz = Utc.to_object(py); @@ -850,10 +846,38 @@ mod tests { #[cfg(all(test, not(target_arch = "wasm32")))] mod proptests { use super::*; + use crate::types::IntoPyDict; use proptest::prelude::*; proptest! { + + // Range is limited to 1970 to 2038 due to windows limitations + #[test] + fn test_pyo3_offset_fixed_frompyobject_created_in_python(timestamp in 0..(i32::MAX as i64), timedelta in -86399i32..=86399i32) { + Python::with_gil(|py| { + + let globals = [("datetime", py.import("datetime").unwrap())].into_py_dict(py); + let code = format!("datetime.datetime.fromtimestamp({}).replace(tzinfo=datetime.timezone(datetime.timedelta(seconds={})))", timestamp, timedelta); + let t = py.eval(&code, Some(globals), None).unwrap(); + + // Get ISO 8601 string from python + let py_iso_str = t.call_method0("isoformat").unwrap(); + + // Get ISO 8601 string from rust + let t = t.extract::>().unwrap(); + // Python doesn't print the seconds of the offset if they are 0 + let rust_iso_str = if timedelta % 60 == 0 { + t.format("%Y-%m-%dT%H:%M:%S%:z").to_string() + } else { + t.format("%Y-%m-%dT%H:%M:%S%::z").to_string() + }; + + // They should be equal + assert_eq!(py_iso_str.to_string(), rust_iso_str); + }) + } + #[test] fn test_duration_roundtrip(days in -999999999i64..=999999999i64) { // Test roundtrip convertion rust->python->rust for all allowed @@ -947,7 +971,7 @@ mod tests { hour in 0u32..=24u32, min in 0u32..=60u32, sec in 0u32..=60u32, - micro in 0u32..=2_000_000u32, + micro in 0u32..=1_000_000u32, offset_secs in -86399i32..=86399i32 ) { Python::with_gil(|py| { From 4440d09ca7e25d0771bbb087abc1447c97bf7f8d Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Sun, 25 Jun 2023 20:01:58 +0100 Subject: [PATCH 05/57] add some style guide to Contributing.md --- Contributing.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/Contributing.md b/Contributing.md index c6d29e2d427..75b129edd1d 100644 --- a/Contributing.md +++ b/Contributing.md @@ -111,6 +111,36 @@ To include your changes in the release notes, you should create one (or more) ne - `removed` - for features which have been removed - `fixed` - for "changed" features which were classed as a bugfix +### Style guide + +#### Generic code + +PyO3 has a lot of generic APIs to increase usability. These can come at the cost of generic code bloat. Where reasonable, try to implement a concrete sub-portion of generic functions. There are two forms of this: + +- If the concrete sub-portion doesn't benefit from re-use by other functions, name it `inner` and keep it as a local to the function. +- If the concrete sub-portion is re-used by other functions, preferably name it `_foo` and place it directly below `foo` in the source code (where `foo` is the original generic function). + +#### FFI calls + +PyO3 makes a lot of FFI calls to Python's C API using raw pointers. Where possible try to avoid using pointers-to-temporaries in expressions: + +```rust +// dangerous +pyo3::ffi::Something(name.to_object(py).as_ptr()); + +// because the following refactoring is a use-after-free error: +let name = name.to_object(py).as_ptr(); +pyo3::ffi::Something(name) +``` + +Instead, prefer to bind the safe owned `PyObject` wrapper before passing to ffi functions: + +```rust +let name: PyObject = name.to_object(py); +pyo3::ffi::Something(name.as_ptr()) +// name will automatically be freed when it falls out of scope +``` + ## Python and Rust version support policy PyO3 aims to keep sufficient compatibility to make packaging Python extensions built with PyO3 feasible on most common package managers. From 119d7cb3f86e77c12b22c823783c7cce5d8869ca Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Tue, 27 Jun 2023 08:33:16 +0100 Subject: [PATCH 06/57] prefer inner / _private naming --- pyo3-macros-backend/src/module.rs | 2 +- pyo3-macros-backend/src/pymethod.rs | 2 +- src/buffer.rs | 18 ++++++------ src/impl_/pyfunction.rs | 2 +- src/impl_/pymethods.rs | 2 +- src/impl_/trampoline.rs | 14 +++++----- src/macros.rs | 4 +-- src/pyclass/create_type_object.rs | 12 ++++---- src/types/dict.rs | 43 ++++++++++++++++------------- src/types/frozenset.rs | 4 +-- src/types/set.rs | 7 ++--- tests/ui/traverse.stderr | 4 +-- 12 files changed, 58 insertions(+), 56 deletions(-) diff --git a/pyo3-macros-backend/src/module.rs b/pyo3-macros-backend/src/module.rs index e6db700718f..b1dd100324b 100644 --- a/pyo3-macros-backend/src/module.rs +++ b/pyo3-macros-backend/src/module.rs @@ -121,7 +121,7 @@ pub fn process_functions_in_module( let name = &func.sig.ident; let statements: Vec = syn::parse_quote! { #wrapped_function - #module_name.add_function(#krate::impl_::pyfunction::wrap_pyfunction_impl(&#name::DEF, #module_name)?)?; + #module_name.add_function(#krate::impl_::pyfunction::_wrap_pyfunction(&#name::DEF, #module_name)?)?; }; stmts.extend(statements); } diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index 38c325113f4..75c52e3b053 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -414,7 +414,7 @@ fn impl_traverse_slot(cls: &syn::Type, spec: &FnSpec<'_>) -> syn::Result ::std::os::raw::c_int { - _pyo3::impl_::pymethods::call_traverse_impl::<#cls>(slf, #cls::#rust_fn_ident, visit, arg) + _pyo3::impl_::pymethods::_call_traverse::<#cls>(slf, #cls::#rust_fn_ident, visit, arg) } }; let slot_def = quote! { diff --git a/src/buffer.rs b/src/buffer.rs index d3be9ee6187..79e416ba939 100644 --- a/src/buffer.rs +++ b/src/buffer.rs @@ -469,7 +469,7 @@ impl PyBuffer { /// you can use `::is_compatible_format(buf.format())`. /// Alternatively, `match buffer::ElementType::from_format(buf.format())`. pub fn copy_to_slice(&self, py: Python<'_>, target: &mut [T]) -> PyResult<()> { - self.copy_to_slice_impl(py, target, b'C') + self._copy_to_slice(py, target, b'C') } /// Copies the buffer elements to the specified slice. @@ -482,10 +482,10 @@ impl PyBuffer { /// you can use `::is_compatible_format(buf.format())`. /// Alternatively, `match buffer::ElementType::from_format(buf.format())`. pub fn copy_to_fortran_slice(&self, py: Python<'_>, target: &mut [T]) -> PyResult<()> { - self.copy_to_slice_impl(py, target, b'F') + self._copy_to_slice(py, target, b'F') } - fn copy_to_slice_impl(&self, py: Python<'_>, target: &mut [T], fort: u8) -> PyResult<()> { + fn _copy_to_slice(&self, py: Python<'_>, target: &mut [T], fort: u8) -> PyResult<()> { if mem::size_of_val(target) != self.len_bytes() { return Err(PyBufferError::new_err(format!( "slice to copy to (of length {}) does not match buffer length of {}", @@ -516,7 +516,7 @@ impl PyBuffer { /// /// Fails if the buffer format is not compatible with type `T`. pub fn to_vec(&self, py: Python<'_>) -> PyResult> { - self.to_vec_impl(py, b'C') + self._to_vec(py, b'C') } /// Copies the buffer elements to a newly allocated vector. @@ -524,10 +524,10 @@ impl PyBuffer { /// /// Fails if the buffer format is not compatible with type `T`. pub fn to_fortran_vec(&self, py: Python<'_>) -> PyResult> { - self.to_vec_impl(py, b'F') + self._to_vec(py, b'F') } - fn to_vec_impl(&self, py: Python<'_>, fort: u8) -> PyResult> { + fn _to_vec(&self, py: Python<'_>, fort: u8) -> PyResult> { let item_count = self.item_count(); let mut vec: Vec = Vec::with_capacity(item_count); unsafe { @@ -564,7 +564,7 @@ impl PyBuffer { /// use `::is_compatible_format(buf.format())`. /// Alternatively, `match buffer::ElementType::from_format(buf.format())`. pub fn copy_from_slice(&self, py: Python<'_>, source: &[T]) -> PyResult<()> { - self.copy_from_slice_impl(py, source, b'C') + self._copy_from_slice(py, source, b'C') } /// Copies the specified slice into the buffer. @@ -578,10 +578,10 @@ impl PyBuffer { /// use `::is_compatible_format(buf.format())`. /// Alternatively, `match buffer::ElementType::from_format(buf.format())`. pub fn copy_from_fortran_slice(&self, py: Python<'_>, source: &[T]) -> PyResult<()> { - self.copy_from_slice_impl(py, source, b'F') + self._copy_from_slice(py, source, b'F') } - fn copy_from_slice_impl(&self, py: Python<'_>, source: &[T], fort: u8) -> PyResult<()> { + fn _copy_from_slice(&self, py: Python<'_>, source: &[T], fort: u8) -> PyResult<()> { if self.readonly() { return Err(PyBufferError::new_err("cannot write to read-only buffer")); } else if mem::size_of_val(source) != self.len_bytes() { diff --git a/src/impl_/pyfunction.rs b/src/impl_/pyfunction.rs index 95d8350d270..14cdbd48f85 100644 --- a/src/impl_/pyfunction.rs +++ b/src/impl_/pyfunction.rs @@ -2,7 +2,7 @@ use crate::{derive_utils::PyFunctionArguments, types::PyCFunction, PyResult}; pub use crate::impl_::pymethods::PyMethodDef; -pub fn wrap_pyfunction_impl<'a>( +pub fn _wrap_pyfunction<'a>( method_def: &PyMethodDef, py_or_module: impl Into>, ) -> PyResult<&'a PyCFunction> { diff --git a/src/impl_/pymethods.rs b/src/impl_/pymethods.rs index 60db3fbb1bf..98089d209d9 100644 --- a/src/impl_/pymethods.rs +++ b/src/impl_/pymethods.rs @@ -247,7 +247,7 @@ impl PySetterDef { /// Calls an implementation of __traverse__ for tp_traverse #[doc(hidden)] -pub unsafe fn call_traverse_impl( +pub unsafe fn _call_traverse( slf: *mut ffi::PyObject, impl_: fn(&T, PyVisit<'_>) -> Result<(), PyTraverseError>, visit: ffi::visitproc, diff --git a/src/impl_/trampoline.rs b/src/impl_/trampoline.rs index efe50e20727..1fcdebc25e7 100644 --- a/src/impl_/trampoline.rs +++ b/src/impl_/trampoline.rs @@ -18,7 +18,7 @@ use crate::{ pub unsafe fn module_init( f: for<'py> unsafe fn(Python<'py>) -> PyResult>, ) -> *mut ffi::PyObject { - trampoline_inner(|py| f(py).map(|module| module.into_ptr())) + trampoline(|py| f(py).map(|module| module.into_ptr())) } #[inline] @@ -28,7 +28,7 @@ pub unsafe fn noargs( f: for<'py> unsafe fn(Python<'py>, *mut ffi::PyObject) -> PyResult<*mut ffi::PyObject>, ) -> *mut ffi::PyObject { debug_assert!(args.is_null()); - trampoline_inner(|py| f(py, slf)) + trampoline(|py| f(py, slf)) } macro_rules! trampoline { @@ -38,7 +38,7 @@ macro_rules! trampoline { $($arg_names: $arg_types,)* f: for<'py> unsafe fn (Python<'py>, $($arg_types),*) -> PyResult<$ret>, ) -> $ret { - trampoline_inner(|py| f(py, $($arg_names,)*)) + trampoline(|py| f(py, $($arg_names,)*)) } } } @@ -131,7 +131,7 @@ pub unsafe fn releasebufferproc( buf: *mut ffi::Py_buffer, f: for<'py> unsafe fn(Python<'py>, *mut ffi::PyObject, *mut ffi::Py_buffer) -> PyResult<()>, ) { - trampoline_inner_unraisable(|py| f(py, slf, buf), slf) + trampoline_unraisable(|py| f(py, slf, buf), slf) } #[inline] @@ -143,7 +143,7 @@ pub(crate) unsafe fn dealloc( // so pass null_mut() to the context. // // (Note that we don't allow the implementation `f` to fail.) - trampoline_inner_unraisable( + trampoline_unraisable( |py| { f(py, slf); Ok(()) @@ -168,7 +168,7 @@ trampoline!( /// Panics during execution are trapped so that they don't propagate through any /// outer FFI boundary. #[inline] -pub(crate) fn trampoline_inner(body: F) -> R +pub(crate) fn trampoline(body: F) -> R where F: for<'py> FnOnce(Python<'py>) -> PyResult + UnwindSafe, R: PyCallbackOutput, @@ -214,7 +214,7 @@ where /// /// ctx must be either a valid ffi::PyObject or NULL #[inline] -unsafe fn trampoline_inner_unraisable(body: F, ctx: *mut ffi::PyObject) +unsafe fn trampoline_unraisable(body: F, ctx: *mut ffi::PyObject) where F: for<'py> FnOnce(Python<'py>) -> PyResult<()> + UnwindSafe, { diff --git a/src/macros.rs b/src/macros.rs index d9237e2c959..560d43da1ca 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -125,12 +125,12 @@ macro_rules! wrap_pyfunction { ($function:path) => { &|py_or_module| { use $function as wrapped_pyfunction; - $crate::impl_::pyfunction::wrap_pyfunction_impl(&wrapped_pyfunction::DEF, py_or_module) + $crate::impl_::pyfunction::_wrap_pyfunction(&wrapped_pyfunction::DEF, py_or_module) } }; ($function:path, $py_or_module:expr) => {{ use $function as wrapped_pyfunction; - $crate::impl_::pyfunction::wrap_pyfunction_impl(&wrapped_pyfunction::DEF, $py_or_module) + $crate::impl_::pyfunction::_wrap_pyfunction(&wrapped_pyfunction::DEF, $py_or_module) }}; } diff --git a/src/pyclass/create_type_object.rs b/src/pyclass/create_type_object.rs index bc2058f50f0..09aec7a4418 100644 --- a/src/pyclass/create_type_object.rs +++ b/src/pyclass/create_type_object.rs @@ -7,7 +7,7 @@ use crate::{ }, impl_::{ pymethods::{get_doc, get_name, Getter, Setter}, - trampoline::trampoline_inner, + trampoline::trampoline, }, types::PyType, Py, PyClass, PyGetterDef, PyMethodDefType, PyResult, PySetterDef, PyTypeInfo, Python, @@ -413,7 +413,7 @@ unsafe extern "C" fn no_constructor_defined( _args: *mut ffi::PyObject, _kwds: *mut ffi::PyObject, ) -> *mut ffi::PyObject { - trampoline_inner(|_| { + trampoline(|_| { Err(crate::exceptions::PyTypeError::new_err( "No constructor defined", )) @@ -513,7 +513,7 @@ impl GetSetDefType { ) -> *mut ffi::PyObject { // Safety: PyO3 sets the closure when constructing the ffi getter so this cast should always be valid let getter: Getter = std::mem::transmute(closure); - trampoline_inner(|py| getter(py, slf)) + trampoline(|py| getter(py, slf)) } (Some(getter), None, closure as Getter as _) } @@ -525,7 +525,7 @@ impl GetSetDefType { ) -> c_int { // Safety: PyO3 sets the closure when constructing the ffi setter so this cast should always be valid let setter: Setter = std::mem::transmute(closure); - trampoline_inner(|py| setter(py, slf, value)) + trampoline(|py| setter(py, slf, value)) } (None, Some(setter), closure as Setter as _) } @@ -535,7 +535,7 @@ impl GetSetDefType { closure: *mut c_void, ) -> *mut ffi::PyObject { let getset: &GetterAndSetter = &*(closure as *const GetterAndSetter); - trampoline_inner(|py| (getset.getter)(py, slf)) + trampoline(|py| (getset.getter)(py, slf)) } unsafe extern "C" fn getset_setter( @@ -544,7 +544,7 @@ impl GetSetDefType { closure: *mut c_void, ) -> c_int { let getset: &GetterAndSetter = &*(closure as *const GetterAndSetter); - trampoline_inner(|py| (getset.setter)(py, slf, value)) + trampoline(|py| (getset.setter)(py, slf, value)) } ( Some(getset_getter), diff --git a/src/types/dict.rs b/src/types/dict.rs index b4ed14c3f89..8f9d393e6e2 100644 --- a/src/types/dict.rs +++ b/src/types/dict.rs @@ -142,17 +142,20 @@ impl PyDict { where K: ToPyObject, { - self.get_item_impl(key.to_object(self.py())) - } - - fn get_item_impl(&self, key: PyObject) -> Option<&PyAny> { - let py = self.py(); - unsafe { - let ptr = ffi::PyDict_GetItem(self.as_ptr(), key.as_ptr()); + fn inner(dict: &PyDict, key: PyObject) -> Option<&PyAny> { + let py = dict.py(); // PyDict_GetItem returns a borrowed ptr, must make it owned for safety (see #890). // PyObject::from_borrowed_ptr_or_opt will take ownership in this way. - PyObject::from_borrowed_ptr_or_opt(py, ptr).map(|pyobject| pyobject.into_ref(py)) + unsafe { + PyObject::from_borrowed_ptr_or_opt( + py, + ffi::PyDict_GetItem(dict.as_ptr(), key.as_ptr()), + ) + } + .map(|pyobject| pyobject.into_ref(py)) } + + inner(self, key.to_object(self.py())) } /// Gets an item from the dictionary, @@ -164,20 +167,22 @@ impl PyDict { where K: ToPyObject, { - self.get_item_with_error_impl(key.to_object(self.py())) - } - - fn get_item_with_error_impl(&self, key: PyObject) -> PyResult> { - let py = self.py(); - unsafe { - let ptr = ffi::PyDict_GetItemWithError(self.as_ptr(), key.as_ptr()); + fn inner(dict: &PyDict, key: PyObject) -> PyResult> { + let py = dict.py(); // PyDict_GetItemWithError returns a borrowed ptr, must make it owned for safety (see #890). // PyObject::from_borrowed_ptr_or_opt will take ownership in this way. - PyObject::from_borrowed_ptr_or_opt(py, ptr) - .map(|pyobject| Ok(pyobject.into_ref(py))) - .or_else(|| PyErr::take(py).map(Err)) - .transpose() + unsafe { + PyObject::from_borrowed_ptr_or_opt( + py, + ffi::PyDict_GetItemWithError(dict.as_ptr(), key.as_ptr()), + ) + } + .map(|pyobject| Ok(pyobject.into_ref(py))) + .or_else(|| PyErr::take(py).map(Err)) + .transpose() } + + inner(self, key.to_object(self.py())) } /// Sets an item value. diff --git a/src/types/frozenset.rs b/src/types/frozenset.rs index ff91be9251e..067318e397a 100644 --- a/src/types/frozenset.rs +++ b/src/types/frozenset.rs @@ -197,7 +197,7 @@ pub(crate) fn new_from_iter( py: Python<'_>, elements: impl IntoIterator, ) -> PyResult> { - fn new_from_iter_inner( + fn inner( py: Python<'_>, elements: &mut dyn Iterator, ) -> PyResult> { @@ -217,7 +217,7 @@ pub(crate) fn new_from_iter( } let mut iter = elements.into_iter().map(|e| e.to_object(py)); - new_from_iter_inner(py, &mut iter) + inner(py, &mut iter) } #[cfg(test)] diff --git a/src/types/set.rs b/src/types/set.rs index af174e1b7c5..7ded352bfc3 100644 --- a/src/types/set.rs +++ b/src/types/set.rs @@ -238,10 +238,7 @@ pub(crate) fn new_from_iter( py: Python<'_>, elements: impl IntoIterator, ) -> PyResult> { - fn new_from_iter_inner( - py: Python<'_>, - elements: &mut dyn Iterator, - ) -> PyResult> { + fn inner(py: Python<'_>, elements: &mut dyn Iterator) -> PyResult> { let set: Py = unsafe { // We create the `Py` pointer because its Drop cleans up the set if user code panics. Py::from_owned_ptr_or_err(py, ffi::PySet_New(std::ptr::null_mut()))? @@ -258,7 +255,7 @@ pub(crate) fn new_from_iter( } let mut iter = elements.into_iter().map(|e| e.to_object(py)); - new_from_iter_inner(py, &mut iter) + inner(py, &mut iter) } #[cfg(test)] diff --git a/tests/ui/traverse.stderr b/tests/ui/traverse.stderr index 873ef864a23..e2718c76e56 100644 --- a/tests/ui/traverse.stderr +++ b/tests/ui/traverse.stderr @@ -19,5 +19,5 @@ error[E0308]: mismatched types note: function defined here --> src/impl_/pymethods.rs | - | pub unsafe fn call_traverse_impl( - | ^^^^^^^^^^^^^^^^^^ + | pub unsafe fn _call_traverse( + | ^^^^^^^^^^^^^^ From b97e8d8532fc6bbf93dbe8ad82d99bbf4821774d Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Tue, 4 Jul 2023 09:00:26 +0100 Subject: [PATCH 07/57] apply conventions for ffi calls --- src/conversions/std/num.rs | 15 ++++----- src/err/mod.rs | 18 +++++----- src/ffi/tests.rs | 15 ++++----- src/impl_/extract_argument.rs | 19 ++++++----- src/types/any.rs | 56 ++++++++++++++++--------------- src/types/dict.rs | 37 ++++++++++---------- src/types/frozenset.rs | 19 +++++++---- src/types/list.rs | 28 +++++++--------- src/types/sequence.rs | 63 +++++++++++++++++++---------------- src/types/set.rs | 19 ++++++----- 10 files changed, 150 insertions(+), 139 deletions(-) diff --git a/src/conversions/std/num.rs b/src/conversions/std/num.rs index 89e86b8a080..11c184f7373 100644 --- a/src/conversions/std/num.rs +++ b/src/conversions/std/num.rs @@ -248,19 +248,17 @@ mod slow_128bit_int_conversion { impl IntoPy for $rust_type { fn into_py(self, py: Python<'_>) -> PyObject { - let lower = self as u64; - let upper = (self >> SHIFT) as $half_type; + let lower = (self as u64).into_py(py); + let upper = ((self >> SHIFT) as $half_type).into_py(py); + let shift = SHIFT.into_py(py); unsafe { let shifted = PyObject::from_owned_ptr( py, - ffi::PyNumber_Lshift( - upper.into_py(py).as_ptr(), - SHIFT.into_py(py).as_ptr(), - ), + ffi::PyNumber_Lshift(upper.as_ptr(), shift.as_ptr()), ); PyObject::from_owned_ptr( py, - ffi::PyNumber_Or(shifted.as_ptr(), lower.into_py(py).as_ptr()), + ffi::PyNumber_Or(shifted.as_ptr(), lower.as_ptr()), ) } } @@ -280,9 +278,10 @@ mod slow_128bit_int_conversion { -1 as _, ffi::PyLong_AsUnsignedLongLongMask(ob.as_ptr()), )? as $rust_type; + let shift = SHIFT.into_py(py); let shifted = PyObject::from_owned_ptr_or_err( py, - ffi::PyNumber_Rshift(ob.as_ptr(), SHIFT.into_py(py).as_ptr()), + ffi::PyNumber_Rshift(ob.as_ptr(), shift.as_ptr()), )?; let upper: $half_type = shifted.extract(py)?; Ok((<$rust_type>::from(upper) << SHIFT) | lower) diff --git a/src/err/mod.rs b/src/err/mod.rs index 9c71b439352..05072601f78 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -314,7 +314,7 @@ impl PyErr { (ptype, pvalue, ptraceback) }; - if ptype.as_ptr() == PanicException::type_object(py).as_ptr() { + if ptype.as_ptr() == PanicException::type_object_raw(py).cast() { let msg: String = pvalue .as_ref() .and_then(|obj| obj.extract(py).ok()) @@ -439,9 +439,8 @@ impl PyErr { where T: ToPyObject, { - unsafe { - ffi::PyErr_GivenExceptionMatches(self.type_ptr(py), exc.to_object(py).as_ptr()) != 0 - } + let exc = exc.to_object(py); + unsafe { ffi::PyErr_GivenExceptionMatches(self.type_ptr(py), exc.as_ptr()) != 0 } } /// Returns true if the current exception is instance of `T`. @@ -610,18 +609,21 @@ impl PyErr { /// Return the cause (either an exception instance, or None, set by `raise ... from ...`) /// associated with the exception, as accessible from Python through `__cause__`. pub fn cause(&self, py: Python<'_>) -> Option { - let ptr = unsafe { ffi::PyException_GetCause(self.value(py).as_ptr()) }; - let obj = unsafe { py.from_owned_ptr_or_opt::(ptr) }; + let value = self.value(py); + let obj = + unsafe { py.from_owned_ptr_or_opt::(ffi::PyException_GetCause(value.as_ptr())) }; obj.map(Self::from_value) } /// Set the cause associated with the exception, pass `None` to clear it. pub fn set_cause(&self, py: Python<'_>, cause: Option) { + let value = self.value(py); + let cause = cause.map(|err| err.into_value(py)); unsafe { // PyException_SetCause _steals_ a reference to cause, so must use .into_ptr() ffi::PyException_SetCause( - self.value(py).as_ptr(), - cause.map_or(std::ptr::null_mut(), |err| err.into_value(py).into_ptr()), + value.as_ptr(), + cause.map_or(std::ptr::null_mut(), IntoPyPointer::into_ptr), ); } } diff --git a/src/ffi/tests.rs b/src/ffi/tests.rs index 68ddab76305..f9edd8ee3ac 100644 --- a/src/ffi/tests.rs +++ b/src/ffi/tests.rs @@ -71,12 +71,8 @@ fn test_timezone_from_offset() { use crate::types::PyDelta; Python::with_gil(|py| { - let tz: &PyAny = unsafe { - PyDateTime_IMPORT(); - py.from_borrowed_ptr(PyTimeZone_FromOffset( - PyDelta::new(py, 0, 100, 0, false).unwrap().as_ptr(), - )) - }; + let delta = PyDelta::new(py, 0, 100, 0, false).unwrap(); + let tz: &PyAny = unsafe { py.from_borrowed_ptr(PyTimeZone_FromOffset(delta.as_ptr())) }; crate::py_run!( py, tz, @@ -92,11 +88,12 @@ fn test_timezone_from_offset_and_name() { use crate::types::PyDelta; Python::with_gil(|py| { + let delta = PyDelta::new(py, 0, 100, 0, false).unwrap(); + let tzname = PyString::new(py, "testtz"); let tz: &PyAny = unsafe { - PyDateTime_IMPORT(); py.from_borrowed_ptr(PyTimeZone_FromOffsetAndName( - PyDelta::new(py, 0, 100, 0, false).unwrap().as_ptr(), - PyString::new(py, "testtz").as_ptr(), + delta.as_ptr(), + tzname.as_ptr(), )) }; crate::py_run!( diff --git a/src/impl_/extract_argument.rs b/src/impl_/extract_argument.rs index dd361e4b538..56af8921f11 100644 --- a/src/impl_/extract_argument.rs +++ b/src/impl_/extract_argument.rs @@ -728,14 +728,14 @@ mod tests { }; Python::with_gil(|py| { + let args = PyTuple::new(py, Vec::<&PyAny>::new()); + let kwargs = [("foo".to_object(py).into_ref(py), 0u8)].into_py_dict(py); let err = unsafe { function_description .extract_arguments_tuple_dict::( py, - PyTuple::new(py, Vec::<&PyAny>::new()).as_ptr(), - [("foo".to_object(py).into_ref(py), 0u8)] - .into_py_dict(py) - .as_ptr(), + args.as_ptr(), + kwargs.as_ptr(), &mut [], ) .unwrap_err() @@ -759,14 +759,14 @@ mod tests { }; Python::with_gil(|py| { + let args = PyTuple::new(py, Vec::<&PyAny>::new()); + let kwargs = [(1u8.to_object(py).into_ref(py), 1u8)].into_py_dict(py); let err = unsafe { function_description .extract_arguments_tuple_dict::( py, - PyTuple::new(py, Vec::<&PyAny>::new()).as_ptr(), - [(1u8.to_object(py).into_ref(py), 1u8)] - .into_py_dict(py) - .as_ptr(), + args.as_ptr(), + kwargs.as_ptr(), &mut [], ) .unwrap_err() @@ -790,11 +790,12 @@ mod tests { }; Python::with_gil(|py| { + let args = PyTuple::new(py, Vec::<&PyAny>::new()); let mut output = [None, None]; let err = unsafe { function_description.extract_arguments_tuple_dict::( py, - PyTuple::new(py, Vec::<&PyAny>::new()).as_ptr(), + args.as_ptr(), std::ptr::null_mut(), &mut output, ) diff --git a/src/types/any.rs b/src/types/any.rs index f448e43bf05..28de2602c91 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -369,13 +369,17 @@ impl PyAny { where O: ToPyObject, { - unsafe { - self.py().from_owned_ptr_or_err(ffi::PyObject_RichCompare( - self.as_ptr(), - other.to_object(self.py()).as_ptr(), - compare_op as c_int, - )) + fn inner(slf: &PyAny, other: PyObject, compare_op: CompareOp) -> PyResult<&PyAny> { + unsafe { + slf.py().from_owned_ptr_or_err(ffi::PyObject_RichCompare( + slf.as_ptr(), + other.as_ptr(), + compare_op as c_int, + )) + } } + + inner(self, other.to_object(self.py()), compare_op) } /// Tests whether this object is less than another. @@ -767,12 +771,14 @@ impl PyAny { where K: ToPyObject, { - unsafe { - self.py().from_owned_ptr_or_err(ffi::PyObject_GetItem( - self.as_ptr(), - key.to_object(self.py()).as_ptr(), - )) + fn inner(slf: &PyAny, key: PyObject) -> PyResult<&PyAny> { + unsafe { + slf.py() + .from_owned_ptr_or_err(ffi::PyObject_GetItem(slf.as_ptr(), key.as_ptr())) + } } + + inner(self, key.to_object(self.py())) } /// Sets a collection item value. @@ -783,17 +789,14 @@ impl PyAny { K: ToPyObject, V: ToPyObject, { - let py = self.py(); - unsafe { - err::error_on_minusone( - py, - ffi::PyObject_SetItem( - self.as_ptr(), - key.to_object(py).as_ptr(), - value.to_object(py).as_ptr(), - ), - ) + fn inner(slf: &PyAny, key: PyObject, value: PyObject) -> PyResult<()> { + err::error_on_minusone(slf.py(), unsafe { + ffi::PyObject_SetItem(slf.as_ptr(), key.as_ptr(), value.as_ptr()) + }) } + + let py = self.py(); + inner(self, key.to_object(py), value.to_object(py)) } /// Deletes an item from the collection. @@ -803,12 +806,13 @@ impl PyAny { where K: ToPyObject, { - unsafe { - err::error_on_minusone( - self.py(), - ffi::PyObject_DelItem(self.as_ptr(), key.to_object(self.py()).as_ptr()), - ) + fn inner(slf: &PyAny, key: PyObject) -> PyResult<()> { + err::error_on_minusone(slf.py(), unsafe { + ffi::PyObject_DelItem(slf.as_ptr(), key.as_ptr()) + }) } + + inner(self, key.to_object(self.py())) } /// Takes an object and returns an iterator for it. diff --git a/src/types/dict.rs b/src/types/dict.rs index 8f9d393e6e2..d272758043e 100644 --- a/src/types/dict.rs +++ b/src/types/dict.rs @@ -124,13 +124,15 @@ impl PyDict { where K: ToPyObject, { - unsafe { - match ffi::PyDict_Contains(self.as_ptr(), key.to_object(self.py()).as_ptr()) { + fn inner(dict: &PyDict, key: PyObject) -> PyResult { + match unsafe { ffi::PyDict_Contains(dict.as_ptr(), key.as_ptr()) } { 1 => Ok(true), 0 => Ok(false), - _ => Err(PyErr::fetch(self.py())), + _ => Err(PyErr::fetch(dict.py())), } } + + inner(self, key.to_object(self.py())) } /// Gets an item from the dictionary. @@ -193,17 +195,14 @@ impl PyDict { K: ToPyObject, V: ToPyObject, { - let py = self.py(); - unsafe { - err::error_on_minusone( - py, - ffi::PyDict_SetItem( - self.as_ptr(), - key.to_object(py).as_ptr(), - value.to_object(py).as_ptr(), - ), - ) + fn inner(dict: &PyDict, key: PyObject, value: PyObject) -> PyResult<()> { + err::error_on_minusone(dict.py(), unsafe { + ffi::PyDict_SetItem(dict.as_ptr(), key.as_ptr(), value.as_ptr()) + }) } + + let py = self.py(); + inner(self, key.to_object(py), value.to_object(py)) } /// Deletes an item. @@ -213,13 +212,13 @@ impl PyDict { where K: ToPyObject, { - let py = self.py(); - unsafe { - err::error_on_minusone( - py, - ffi::PyDict_DelItem(self.as_ptr(), key.to_object(py).as_ptr()), - ) + fn inner(dict: &PyDict, key: PyObject) -> PyResult<()> { + err::error_on_minusone(dict.py(), unsafe { + ffi::PyDict_DelItem(dict.as_ptr(), key.as_ptr()) + }) } + + inner(self, key.to_object(self.py())) } /// Returns a list of dict keys. diff --git a/src/types/frozenset.rs b/src/types/frozenset.rs index 067318e397a..14216c9db62 100644 --- a/src/types/frozenset.rs +++ b/src/types/frozenset.rs @@ -28,10 +28,13 @@ impl<'py> PyFrozenSetBuilder<'py> { where K: ToPyObject, { - let py = self.py_frozen_set.py(); - err::error_on_minusone(py, unsafe { - ffi::PySet_Add(self.py_frozen_set.as_ptr(), key.to_object(py).as_ptr()) - }) + fn inner(frozenset: &PyFrozenSet, key: PyObject) -> PyResult<()> { + err::error_on_minusone(frozenset.py(), unsafe { + ffi::PySet_Add(frozenset.as_ptr(), key.as_ptr()) + }) + } + + inner(self.py_frozen_set, key.to_object(self.py_frozen_set.py())) } /// Finish building the set and take ownership of its current value @@ -94,13 +97,15 @@ impl PyFrozenSet { where K: ToPyObject, { - unsafe { - match ffi::PySet_Contains(self.as_ptr(), key.to_object(self.py()).as_ptr()) { + fn inner(frozenset: &PyFrozenSet, key: PyObject) -> PyResult { + match unsafe { ffi::PySet_Contains(frozenset.as_ptr(), key.as_ptr()) } { 1 => Ok(true), 0 => Ok(false), - _ => Err(PyErr::fetch(self.py())), + _ => Err(PyErr::fetch(frozenset.py())), } } + + inner(self, key.to_object(self.py())) } /// Returns an iterator of values in this frozen set. diff --git a/src/types/list.rs b/src/types/list.rs index 5e951c29c8e..106b5aa20e2 100644 --- a/src/types/list.rs +++ b/src/types/list.rs @@ -218,13 +218,13 @@ impl PyList { where I: ToPyObject, { - let py = self.py(); - unsafe { - err::error_on_minusone( - py, - ffi::PyList_Append(self.as_ptr(), item.to_object(py).as_ptr()), - ) + fn inner(list: &PyList, item: PyObject) -> PyResult<()> { + err::error_on_minusone(list.py(), unsafe { + ffi::PyList_Append(list.as_ptr(), item.as_ptr()) + }) } + + inner(self, item.to_object(self.py())) } /// Inserts an item at the specified index. @@ -234,17 +234,13 @@ impl PyList { where I: ToPyObject, { - let py = self.py(); - unsafe { - err::error_on_minusone( - py, - ffi::PyList_Insert( - self.as_ptr(), - get_ssize_index(index), - item.to_object(py).as_ptr(), - ), - ) + fn inner(list: &PyList, index: usize, item: PyObject) -> PyResult<()> { + err::error_on_minusone(list.py(), unsafe { + ffi::PyList_Insert(list.as_ptr(), get_ssize_index(index), item.as_ptr()) + }) } + + inner(self, index, item.to_object(self.py())) } /// Determines if self contains `value`. diff --git a/src/types/sequence.rs b/src/types/sequence.rs index 73e110cd40e..659490c619e 100644 --- a/src/types/sequence.rs +++ b/src/types/sequence.rs @@ -6,7 +6,7 @@ use crate::internal_tricks::get_ssize_index; use crate::sync::GILOnceCell; use crate::type_object::PyTypeInfo; use crate::types::{PyAny, PyList, PyString, PyTuple, PyType}; -use crate::{ffi, PyNativeType, ToPyObject}; +use crate::{ffi, PyNativeType, PyObject, ToPyObject}; use crate::{AsPyPointer, IntoPyPointer, Py, Python}; use crate::{FromPyObject, PyTryFrom}; @@ -128,17 +128,13 @@ impl PySequence { where I: ToPyObject, { - let py = self.py(); - unsafe { - err::error_on_minusone( - py, - ffi::PySequence_SetItem( - self.as_ptr(), - get_ssize_index(i), - item.to_object(py).as_ptr(), - ), - ) + fn inner(seq: &PySequence, i: usize, item: PyObject) -> PyResult<()> { + err::error_on_minusone(seq.py(), unsafe { + ffi::PySequence_SetItem(seq.as_ptr(), get_ssize_index(i), item.as_ptr()) + }) } + + inner(self, i, item.to_object(self.py())) } /// Deletes the `i`th element of self. @@ -193,13 +189,16 @@ impl PySequence { where V: ToPyObject, { - let r = - unsafe { ffi::PySequence_Count(self.as_ptr(), value.to_object(self.py()).as_ptr()) }; - if r == -1 { - Err(PyErr::fetch(self.py())) - } else { - Ok(r as usize) + fn inner(seq: &PySequence, value: PyObject) -> PyResult { + let r = unsafe { ffi::PySequence_Count(seq.as_ptr(), value.as_ptr()) }; + if r == -1 { + Err(PyErr::fetch(seq.py())) + } else { + Ok(r as usize) + } } + + inner(self, value.to_object(self.py())) } /// Determines if self contains `value`. @@ -210,13 +209,16 @@ impl PySequence { where V: ToPyObject, { - let r = - unsafe { ffi::PySequence_Contains(self.as_ptr(), value.to_object(self.py()).as_ptr()) }; - match r { - 0 => Ok(false), - 1 => Ok(true), - _ => Err(PyErr::fetch(self.py())), + fn inner(seq: &PySequence, value: PyObject) -> PyResult { + let r = unsafe { ffi::PySequence_Contains(seq.as_ptr(), value.as_ptr()) }; + match r { + 0 => Ok(false), + 1 => Ok(true), + _ => Err(PyErr::fetch(seq.py())), + } } + + inner(self, value.to_object(self.py())) } /// Returns the first index `i` for which `self[i] == value`. @@ -227,13 +229,16 @@ impl PySequence { where V: ToPyObject, { - let r = - unsafe { ffi::PySequence_Index(self.as_ptr(), value.to_object(self.py()).as_ptr()) }; - if r == -1 { - Err(PyErr::fetch(self.py())) - } else { - Ok(r as usize) + fn inner(seq: &PySequence, value: PyObject) -> PyResult { + let r = unsafe { ffi::PySequence_Index(seq.as_ptr(), value.as_ptr()) }; + if r == -1 { + Err(PyErr::fetch(seq.py())) + } else { + Ok(r as usize) + } } + + inner(self, value.to_object(self.py())) } /// Returns a fresh list based on the Sequence. diff --git a/src/types/set.rs b/src/types/set.rs index 7ded352bfc3..d487b125374 100644 --- a/src/types/set.rs +++ b/src/types/set.rs @@ -71,13 +71,15 @@ impl PySet { where K: ToPyObject, { - unsafe { - match ffi::PySet_Contains(self.as_ptr(), key.to_object(self.py()).as_ptr()) { + fn inner(set: &PySet, key: PyObject) -> PyResult { + match unsafe { ffi::PySet_Contains(set.as_ptr(), key.as_ptr()) } { 1 => Ok(true), 0 => Ok(false), - _ => Err(PyErr::fetch(self.py())), + _ => Err(PyErr::fetch(set.py())), } } + + inner(self, key.to_object(self.py())) } /// Removes the element from the set if it is present. @@ -95,12 +97,13 @@ impl PySet { where K: ToPyObject, { - unsafe { - err::error_on_minusone( - self.py(), - ffi::PySet_Add(self.as_ptr(), key.to_object(self.py()).as_ptr()), - ) + fn inner(set: &PySet, key: PyObject) -> PyResult<()> { + err::error_on_minusone(set.py(), unsafe { + ffi::PySet_Add(set.as_ptr(), key.as_ptr()) + }) } + + inner(self, key.to_object(self.py())) } /// Removes and returns an arbitrary element from the set. From 9594018f7ba510df8bd20c9458ae2ee3a407d153 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Tue, 4 Jul 2023 20:34:23 +0100 Subject: [PATCH 08/57] use error_on_minusone in more cases --- src/conversions/std/osstr.rs | 6 +----- src/err/mod.rs | 24 +++++++++++++++++++++--- src/types/any.rs | 14 ++++---------- src/types/mapping.rs | 7 ++----- src/types/sequence.rs | 21 ++++++--------------- 5 files changed, 34 insertions(+), 38 deletions(-) diff --git a/src/conversions/std/osstr.rs b/src/conversions/std/osstr.rs index 12d00ffc431..f91822a874f 100644 --- a/src/conversions/std/osstr.rs +++ b/src/conversions/std/osstr.rs @@ -1,6 +1,4 @@ use crate::types::PyString; -#[cfg(windows)] -use crate::PyErr; use crate::{ ffi, AsPyPointer, FromPyObject, IntoPy, PyAny, PyObject, PyResult, Python, ToPyObject, }; @@ -92,9 +90,7 @@ impl FromPyObject<'_> for OsString { // ourselves let size = unsafe { ffi::PyUnicode_AsWideChar(pystring.as_ptr(), std::ptr::null_mut(), 0) }; - if size == -1 { - return Err(PyErr::fetch(ob.py())); - } + crate::err::error_on_minusone(ob.py(), size)?; let mut buffer = vec![0; size as usize]; let bytes_read = diff --git a/src/err/mod.rs b/src/err/mod.rs index 05072601f78..9eee0a67faa 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -9,7 +9,6 @@ use crate::{AsPyPointer, IntoPy, IntoPyPointer, Py, PyAny, PyObject, Python, ToP use std::borrow::Cow; use std::cell::UnsafeCell; use std::ffi::CString; -use std::os::raw::c_int; mod err_state; mod impls; @@ -792,14 +791,33 @@ pub fn panic_after_error(_py: Python<'_>) -> ! { /// Returns Ok if the error code is not -1. #[inline] -pub fn error_on_minusone(py: Python<'_>, result: c_int) -> PyResult<()> { - if result != -1 { +pub(crate) fn error_on_minusone(py: Python<'_>, result: T) -> PyResult<()> { + if result != T::MINUS_ONE { Ok(()) } else { Err(PyErr::fetch(py)) } } +pub(crate) trait SignedInteger: Eq { + const MINUS_ONE: Self; +} + +macro_rules! impl_signed_integer { + ($t:ty) => { + impl SignedInteger for $t { + const MINUS_ONE: Self = -1; + } + }; +} + +impl_signed_integer!(i8); +impl_signed_integer!(i16); +impl_signed_integer!(i32); +impl_signed_integer!(i64); +impl_signed_integer!(i128); +impl_signed_integer!(isize); + #[inline] fn exceptions_must_derive_from_base_exception(py: Python<'_>) -> PyErr { PyErr::from_state(PyErrState::exceptions_must_derive_from_base_exception(py)) diff --git a/src/types/any.rs b/src/types/any.rs index 28de2602c91..b3c08373623 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -956,11 +956,8 @@ impl PyAny { /// This is equivalent to the Python expression `hash(self)`. pub fn hash(&self) -> PyResult { let v = unsafe { ffi::PyObject_Hash(self.as_ptr()) }; - if v == -1 { - Err(PyErr::fetch(self.py())) - } else { - Ok(v) - } + crate::err::error_on_minusone(self.py(), v)?; + Ok(v) } /// Returns the length of the sequence or mapping. @@ -968,11 +965,8 @@ impl PyAny { /// This is equivalent to the Python expression `len(self)`. pub fn len(&self) -> PyResult { let v = unsafe { ffi::PyObject_Size(self.as_ptr()) }; - if v == -1 { - Err(PyErr::fetch(self.py())) - } else { - Ok(v as usize) - } + crate::err::error_on_minusone(self.py(), v)?; + Ok(v as usize) } /// Returns the list of attributes of this object. diff --git a/src/types/mapping.rs b/src/types/mapping.rs index fa73176d142..f580e7ffe55 100644 --- a/src/types/mapping.rs +++ b/src/types/mapping.rs @@ -17,11 +17,8 @@ impl PyMapping { #[inline] pub fn len(&self) -> PyResult { let v = unsafe { ffi::PyMapping_Size(self.as_ptr()) }; - if v == -1 { - Err(PyErr::fetch(self.py())) - } else { - Ok(v as usize) - } + crate::err::error_on_minusone(self.py(), v)?; + Ok(v as usize) } /// Returns whether the mapping is empty. diff --git a/src/types/sequence.rs b/src/types/sequence.rs index 659490c619e..f2c4f5e0c5f 100644 --- a/src/types/sequence.rs +++ b/src/types/sequence.rs @@ -23,11 +23,8 @@ impl PySequence { #[inline] pub fn len(&self) -> PyResult { let v = unsafe { ffi::PySequence_Size(self.as_ptr()) }; - if v == -1 { - Err(PyErr::fetch(self.py())) - } else { - Ok(v as usize) - } + crate::err::error_on_minusone(self.py(), v)?; + Ok(v as usize) } /// Returns whether the sequence is empty. @@ -191,11 +188,8 @@ impl PySequence { { fn inner(seq: &PySequence, value: PyObject) -> PyResult { let r = unsafe { ffi::PySequence_Count(seq.as_ptr(), value.as_ptr()) }; - if r == -1 { - Err(PyErr::fetch(seq.py())) - } else { - Ok(r as usize) - } + crate::err::error_on_minusone(seq.py(), r)?; + Ok(r as usize) } inner(self, value.to_object(self.py())) @@ -231,11 +225,8 @@ impl PySequence { { fn inner(seq: &PySequence, value: PyObject) -> PyResult { let r = unsafe { ffi::PySequence_Index(seq.as_ptr(), value.as_ptr()) }; - if r == -1 { - Err(PyErr::fetch(seq.py())) - } else { - Ok(r as usize) - } + crate::err::error_on_minusone(seq.py(), r)?; + Ok(r as usize) } inner(self, value.to_object(self.py())) From 35aa4f3d744247340efbf3f70ea4a8ae36740304 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Tue, 4 Jul 2023 20:50:54 +0100 Subject: [PATCH 09/57] move `unsafe` block inside `error_on_minusone` calls --- src/buffer.rs | 119 ++++++++++++-------------- src/conversions/std/num.rs | 43 +++++----- src/err/mod.rs | 36 ++++---- src/impl_/pyclass/lazy_type_object.rs | 5 +- src/instance.rs | 9 +- src/marker.rs | 3 +- src/types/any.rs | 26 +++--- src/types/dict.rs | 21 ++--- src/types/frozenset.rs | 4 +- src/types/list.rs | 36 ++++---- src/types/mapping.rs | 2 +- src/types/sequence.rs | 35 +++----- src/types/set.rs | 4 +- 13 files changed, 154 insertions(+), 189 deletions(-) diff --git a/src/buffer.rs b/src/buffer.rs index 79e416ba939..eb48a7564c6 100644 --- a/src/buffer.rs +++ b/src/buffer.rs @@ -193,14 +193,13 @@ impl PyBuffer { pub fn get(obj: &PyAny) -> PyResult> { // TODO: use nightly API Box::new_uninit() once stable let mut buf = Box::new(mem::MaybeUninit::uninit()); - let buf: Box = unsafe { - err::error_on_minusone( - obj.py(), - ffi::PyObject_GetBuffer(obj.as_ptr(), buf.as_mut_ptr(), ffi::PyBUF_FULL_RO), - )?; + let buf: Box = { + err::error_on_minusone(obj.py(), unsafe { + ffi::PyObject_GetBuffer(obj.as_ptr(), buf.as_mut_ptr(), ffi::PyBUF_FULL_RO) + })?; // Safety: buf is initialized by PyObject_GetBuffer. // TODO: use nightly API Box::assume_init() once stable - mem::transmute(buf) + unsafe { mem::transmute(buf) } }; // Create PyBuffer immediately so that if validation checks fail, the PyBuffer::drop code // will call PyBuffer_Release (thus avoiding any leaks). @@ -493,22 +492,20 @@ impl PyBuffer { self.item_count() ))); } - unsafe { - err::error_on_minusone( - py, - ffi::PyBuffer_ToContiguous( - target.as_ptr() as *mut raw::c_void, - #[cfg(Py_3_11)] - &*self.0, - #[cfg(not(Py_3_11))] - { - &*self.0 as *const ffi::Py_buffer as *mut ffi::Py_buffer - }, - self.0.len, - fort as std::os::raw::c_char, - ), + + err::error_on_minusone(py, unsafe { + ffi::PyBuffer_ToContiguous( + target.as_ptr() as *mut raw::c_void, + #[cfg(Py_3_11)] + &*self.0, + #[cfg(not(Py_3_11))] + { + &*self.0 as *const ffi::Py_buffer as *mut ffi::Py_buffer + }, + self.0.len, + fort as std::os::raw::c_char, ) - } + }) } /// Copies the buffer elements to a newly allocated vector. @@ -530,26 +527,24 @@ impl PyBuffer { fn _to_vec(&self, py: Python<'_>, fort: u8) -> PyResult> { let item_count = self.item_count(); let mut vec: Vec = Vec::with_capacity(item_count); - unsafe { - // Copy the buffer into the uninitialized space in the vector. - // Due to T:Copy, we don't need to be concerned with Drop impls. - err::error_on_minusone( - py, - ffi::PyBuffer_ToContiguous( - vec.as_ptr() as *mut raw::c_void, - #[cfg(Py_3_11)] - &*self.0, - #[cfg(not(Py_3_11))] - { - &*self.0 as *const ffi::Py_buffer as *mut ffi::Py_buffer - }, - self.0.len, - fort as std::os::raw::c_char, - ), - )?; - // set vector length to mark the now-initialized space as usable - vec.set_len(item_count); - } + + // Copy the buffer into the uninitialized space in the vector. + // Due to T:Copy, we don't need to be concerned with Drop impls. + err::error_on_minusone(py, unsafe { + ffi::PyBuffer_ToContiguous( + vec.as_ptr() as *mut raw::c_void, + #[cfg(Py_3_11)] + &*self.0, + #[cfg(not(Py_3_11))] + { + &*self.0 as *const ffi::Py_buffer as *mut ffi::Py_buffer + }, + self.0.len, + fort as std::os::raw::c_char, + ) + })?; + // set vector length to mark the now-initialized space as usable + unsafe { vec.set_len(item_count) }; Ok(vec) } @@ -591,29 +586,27 @@ impl PyBuffer { self.item_count() ))); } - unsafe { - err::error_on_minusone( - py, - ffi::PyBuffer_FromContiguous( - #[cfg(Py_3_11)] - &*self.0, - #[cfg(not(Py_3_11))] - { - &*self.0 as *const ffi::Py_buffer as *mut ffi::Py_buffer - }, - #[cfg(Py_3_11)] - { - source.as_ptr() as *const raw::c_void - }, - #[cfg(not(Py_3_11))] - { - source.as_ptr() as *mut raw::c_void - }, - self.0.len, - fort as std::os::raw::c_char, - ), + + err::error_on_minusone(py, unsafe { + ffi::PyBuffer_FromContiguous( + #[cfg(Py_3_11)] + &*self.0, + #[cfg(not(Py_3_11))] + { + &*self.0 as *const ffi::Py_buffer as *mut ffi::Py_buffer + }, + #[cfg(Py_3_11)] + { + source.as_ptr() as *const raw::c_void + }, + #[cfg(not(Py_3_11))] + { + source.as_ptr() as *mut raw::c_void + }, + self.0.len, + fort as std::os::raw::c_char, ) - } + }) } /// Releases the buffer object, freeing the reference to the Python object diff --git a/src/conversions/std/num.rs b/src/conversions/std/num.rs index 11c184f7373..3427942ee11 100644 --- a/src/conversions/std/num.rs +++ b/src/conversions/std/num.rs @@ -178,16 +178,18 @@ mod fast_128bit_int_conversion { } impl IntoPy for $rust_type { fn into_py(self, py: Python<'_>) -> PyObject { + // Always use little endian + let bytes = self.to_le_bytes(); unsafe { - // Always use little endian - let bytes = self.to_le_bytes(); - let obj = ffi::_PyLong_FromByteArray( - bytes.as_ptr() as *const std::os::raw::c_uchar, - bytes.len(), - 1, - $is_signed, - ); - PyObject::from_owned_ptr(py, obj) + PyObject::from_owned_ptr( + py, + ffi::_PyLong_FromByteArray( + bytes.as_ptr() as *const std::os::raw::c_uchar, + bytes.len(), + 1, + $is_signed, + ), + ) } } @@ -199,23 +201,20 @@ mod fast_128bit_int_conversion { impl<'source> FromPyObject<'source> for $rust_type { fn extract(ob: &'source PyAny) -> PyResult<$rust_type> { - unsafe { - let num = ffi::PyNumber_Index(ob.as_ptr()); - if num.is_null() { - return Err(PyErr::fetch(ob.py())); - } - let mut buffer = [0; std::mem::size_of::<$rust_type>()]; - let ok = ffi::_PyLong_AsByteArray( - num as *mut ffi::PyLongObject, + let num = unsafe { + PyObject::from_owned_ptr_or_err(ob.py(), ffi::PyNumber_Index(ob.as_ptr()))? + }; + let mut buffer = [0; std::mem::size_of::<$rust_type>()]; + crate::err::error_on_minusone(ob.py(), unsafe { + ffi::_PyLong_AsByteArray( + num.as_ptr() as *mut ffi::PyLongObject, buffer.as_mut_ptr(), buffer.len(), 1, $is_signed, - ); - ffi::Py_DECREF(num); - crate::err::error_on_minusone(ob.py(), ok)?; - Ok(<$rust_type>::from_le_bytes(buffer)) - } + ) + })?; + Ok(<$rust_type>::from_le_bytes(buffer)) } #[cfg(feature = "experimental-inspect")] diff --git a/src/err/mod.rs b/src/err/mod.rs index 9eee0a67faa..28f2b3295cb 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -529,16 +529,13 @@ impl PyErr { /// ``` pub fn warn(py: Python<'_>, category: &PyAny, message: &str, stacklevel: i32) -> PyResult<()> { let message = CString::new(message)?; - unsafe { - error_on_minusone( - py, - ffi::PyErr_WarnEx( - category.as_ptr(), - message.as_ptr(), - stacklevel as ffi::Py_ssize_t, - ), + error_on_minusone(py, unsafe { + ffi::PyErr_WarnEx( + category.as_ptr(), + message.as_ptr(), + stacklevel as ffi::Py_ssize_t, ) - } + }) } /// Issues a warning message, with more control over the warning attributes. @@ -569,19 +566,16 @@ impl PyErr { None => std::ptr::null_mut(), Some(obj) => obj.as_ptr(), }; - unsafe { - error_on_minusone( - py, - ffi::PyErr_WarnExplicit( - category.as_ptr(), - message.as_ptr(), - filename.as_ptr(), - lineno, - module_ptr, - registry, - ), + error_on_minusone(py, unsafe { + ffi::PyErr_WarnExplicit( + category.as_ptr(), + message.as_ptr(), + filename.as_ptr(), + lineno, + module_ptr, + registry, ) - } + }) } /// Clone the PyErr. This requires the GIL, which is why PyErr does not implement Clone. diff --git a/src/impl_/pyclass/lazy_type_object.rs b/src/impl_/pyclass/lazy_type_object.rs index 551b052ae71..0e606b588d0 100644 --- a/src/impl_/pyclass/lazy_type_object.rs +++ b/src/impl_/pyclass/lazy_type_object.rs @@ -202,8 +202,9 @@ fn initialize_tp_dict( // We hold the GIL: the dictionary update can be considered atomic from // the POV of other threads. for (key, val) in items { - let ret = unsafe { ffi::PyObject_SetAttrString(type_object, key.as_ptr(), val.into_ptr()) }; - crate::err::error_on_minusone(py, ret)?; + crate::err::error_on_minusone(py, unsafe { + ffi::PyObject_SetAttrString(type_object, key.as_ptr(), val.into_ptr()) + })?; } Ok(()) } diff --git a/src/instance.rs b/src/instance.rs index 0cbf8dbfc68..92f0126d629 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -650,12 +650,9 @@ impl Py { let attr_name = attr_name.into_py(py); let value = value.into_py(py); - unsafe { - err::error_on_minusone( - py, - ffi::PyObject_SetAttr(self.as_ptr(), attr_name.as_ptr(), value.as_ptr()), - ) - } + err::error_on_minusone(py, unsafe { + ffi::PyObject_SetAttr(self.as_ptr(), attr_name.as_ptr(), value.as_ptr()) + }) } /// Calls the object. diff --git a/src/marker.rs b/src/marker.rs index e5b3ed81ca5..d0f486001ed 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -876,8 +876,7 @@ impl<'py> Python<'py> { /// [1]: https://docs.python.org/3/c-api/exceptions.html?highlight=pyerr_checksignals#c.PyErr_CheckSignals /// [2]: https://docs.python.org/3/library/signal.html pub fn check_signals(self) -> PyResult<()> { - let v = unsafe { ffi::PyErr_CheckSignals() }; - err::error_on_minusone(self, v) + err::error_on_minusone(self, unsafe { ffi::PyErr_CheckSignals() }) } /// Create a new pool for managing PyO3's owned references. diff --git a/src/types/any.rs b/src/types/any.rs index b3c08373623..281e318701c 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -227,14 +227,14 @@ impl PyAny { N: IntoPy>, V: ToPyObject, { - let py = self.py(); - let attr_name = attr_name.into_py(py); - let value = value.to_object(py); - - unsafe { - let ret = ffi::PyObject_SetAttr(self.as_ptr(), attr_name.as_ptr(), value.as_ptr()); - err::error_on_minusone(py, ret) + fn inner(any: &PyAny, attr_name: Py, value: PyObject) -> PyResult<()> { + err::error_on_minusone(any.py(), unsafe { + ffi::PyObject_SetAttr(any.as_ptr(), attr_name.as_ptr(), value.as_ptr()) + }) } + + let py = self.py(); + inner(self, attr_name.into_py(py), value.to_object(py)) } /// Deletes an attribute. @@ -247,13 +247,13 @@ impl PyAny { where N: IntoPy>, { - let py = self.py(); - let attr_name = attr_name.into_py(py); - - unsafe { - let ret = ffi::PyObject_DelAttr(self.as_ptr(), attr_name.as_ptr()); - err::error_on_minusone(py, ret) + fn inner(any: &PyAny, attr_name: Py) -> PyResult<()> { + err::error_on_minusone(any.py(), unsafe { + ffi::PyObject_DelAttr(any.as_ptr(), attr_name.as_ptr()) + }) } + + inner(self, attr_name.into_py(self.py())) } /// Returns an [`Ordering`] between `self` and `other`. diff --git a/src/types/dict.rs b/src/types/dict.rs index d272758043e..cabad4716b3 100644 --- a/src/types/dict.rs +++ b/src/types/dict.rs @@ -68,14 +68,11 @@ impl PyDict { /// this keeps the last entry seen. #[cfg(not(PyPy))] pub fn from_sequence(py: Python<'_>, seq: PyObject) -> PyResult<&PyDict> { - unsafe { - let dict = py.from_owned_ptr::(ffi::PyDict_New()); - err::error_on_minusone( - py, - ffi::PyDict_MergeFromSeq2(dict.into_ptr(), seq.into_ptr(), 1), - )?; - Ok(dict) - } + let dict = Self::new(py); + err::error_on_minusone(py, unsafe { + ffi::PyDict_MergeFromSeq2(dict.into_ptr(), seq.into_ptr(), 1) + })?; + Ok(dict) } /// Returns a new dictionary that contains the same key-value pairs as self. @@ -273,7 +270,9 @@ impl PyDict { /// to use `self.update(other.as_mapping())`, note: `PyDict::as_mapping` is a zero-cost conversion. pub fn update(&self, other: &PyMapping) -> PyResult<()> { let py = self.py(); - unsafe { err::error_on_minusone(py, ffi::PyDict_Update(self.as_ptr(), other.as_ptr())) } + err::error_on_minusone(py, unsafe { + ffi::PyDict_Update(self.as_ptr(), other.as_ptr()) + }) } /// Add key/value pairs from another dictionary to this one only when they do not exist in this. @@ -286,7 +285,9 @@ impl PyDict { /// so should have the same performance as `update`. pub fn update_if_missing(&self, other: &PyMapping) -> PyResult<()> { let py = self.py(); - unsafe { err::error_on_minusone(py, ffi::PyDict_Merge(self.as_ptr(), other.as_ptr(), 0)) } + err::error_on_minusone(py, unsafe { + ffi::PyDict_Merge(self.as_ptr(), other.as_ptr(), 0) + }) } } diff --git a/src/types/frozenset.rs b/src/types/frozenset.rs index 14216c9db62..7bd8fe7c3c4 100644 --- a/src/types/frozenset.rs +++ b/src/types/frozenset.rs @@ -213,9 +213,7 @@ pub(crate) fn new_from_iter( let ptr = set.as_ptr(); for obj in elements { - unsafe { - err::error_on_minusone(py, ffi::PySet_Add(ptr, obj.as_ptr()))?; - } + err::error_on_minusone(py, unsafe { ffi::PySet_Add(ptr, obj.as_ptr()) })?; } Ok(set) diff --git a/src/types/list.rs b/src/types/list.rs index 106b5aa20e2..9294ee196c0 100644 --- a/src/types/list.rs +++ b/src/types/list.rs @@ -167,16 +167,13 @@ impl PyList { where I: ToPyObject, { - unsafe { - err::error_on_minusone( - self.py(), - ffi::PyList_SetItem( - self.as_ptr(), - get_ssize_index(index), - item.to_object(self.py()).into_ptr(), - ), - ) + fn inner(list: &PyList, index: usize, item: PyObject) -> PyResult<()> { + err::error_on_minusone(list.py(), unsafe { + ffi::PyList_SetItem(list.as_ptr(), get_ssize_index(index), item.into_ptr()) + }) } + + inner(self, index, item.to_object(self.py())) } /// Deletes the `index`th element of self. @@ -192,17 +189,14 @@ impl PyList { /// This is equivalent to the Python statement `self[low:high] = v`. #[inline] pub fn set_slice(&self, low: usize, high: usize, seq: &PyAny) -> PyResult<()> { - unsafe { - err::error_on_minusone( - self.py(), - ffi::PyList_SetSlice( - self.as_ptr(), - get_ssize_index(low), - get_ssize_index(high), - seq.as_ptr(), - ), + err::error_on_minusone(self.py(), unsafe { + ffi::PyList_SetSlice( + self.as_ptr(), + get_ssize_index(low), + get_ssize_index(high), + seq.as_ptr(), ) - } + }) } /// Deletes the slice from `low` to `high` from `self`. @@ -275,12 +269,12 @@ impl PyList { /// Sorts the list in-place. Equivalent to the Python expression `l.sort()`. pub fn sort(&self) -> PyResult<()> { - unsafe { err::error_on_minusone(self.py(), ffi::PyList_Sort(self.as_ptr())) } + err::error_on_minusone(self.py(), unsafe { ffi::PyList_Sort(self.as_ptr()) }) } /// Reverses the list in-place. Equivalent to the Python expression `l.reverse()`. pub fn reverse(&self) -> PyResult<()> { - unsafe { err::error_on_minusone(self.py(), ffi::PyList_Reverse(self.as_ptr())) } + err::error_on_minusone(self.py(), unsafe { ffi::PyList_Reverse(self.as_ptr()) }) } /// Return a new tuple containing the contents of the list; equivalent to the Python expression `tuple(list)`. diff --git a/src/types/mapping.rs b/src/types/mapping.rs index f580e7ffe55..b947f619a22 100644 --- a/src/types/mapping.rs +++ b/src/types/mapping.rs @@ -1,4 +1,4 @@ -use crate::err::{PyDowncastError, PyErr, PyResult}; +use crate::err::{PyDowncastError, PyResult}; use crate::sync::GILOnceCell; use crate::type_object::PyTypeInfo; use crate::types::{PyAny, PyDict, PySequence, PyType}; diff --git a/src/types/sequence.rs b/src/types/sequence.rs index f2c4f5e0c5f..1db0cd53482 100644 --- a/src/types/sequence.rs +++ b/src/types/sequence.rs @@ -139,12 +139,9 @@ impl PySequence { /// This is equivalent to the Python statement `del self[i]`. #[inline] pub fn del_item(&self, i: usize) -> PyResult<()> { - unsafe { - err::error_on_minusone( - self.py(), - ffi::PySequence_DelItem(self.as_ptr(), get_ssize_index(i)), - ) - } + err::error_on_minusone(self.py(), unsafe { + ffi::PySequence_DelItem(self.as_ptr(), get_ssize_index(i)) + }) } /// Assigns the sequence `v` to the slice of `self` from `i1` to `i2`. @@ -152,17 +149,14 @@ impl PySequence { /// This is equivalent to the Python statement `self[i1:i2] = v`. #[inline] pub fn set_slice(&self, i1: usize, i2: usize, v: &PyAny) -> PyResult<()> { - unsafe { - err::error_on_minusone( - self.py(), - ffi::PySequence_SetSlice( - self.as_ptr(), - get_ssize_index(i1), - get_ssize_index(i2), - v.as_ptr(), - ), + err::error_on_minusone(self.py(), unsafe { + ffi::PySequence_SetSlice( + self.as_ptr(), + get_ssize_index(i1), + get_ssize_index(i2), + v.as_ptr(), ) - } + }) } /// Deletes the slice from `i1` to `i2` from `self`. @@ -170,12 +164,9 @@ impl PySequence { /// This is equivalent to the Python statement `del self[i1:i2]`. #[inline] pub fn del_slice(&self, i1: usize, i2: usize) -> PyResult<()> { - unsafe { - err::error_on_minusone( - self.py(), - ffi::PySequence_DelSlice(self.as_ptr(), get_ssize_index(i1), get_ssize_index(i2)), - ) - } + err::error_on_minusone(self.py(), unsafe { + ffi::PySequence_DelSlice(self.as_ptr(), get_ssize_index(i1), get_ssize_index(i2)) + }) } /// Returns the number of occurrences of `value` in self, that is, return the diff --git a/src/types/set.rs b/src/types/set.rs index d487b125374..2c1c2925b3e 100644 --- a/src/types/set.rs +++ b/src/types/set.rs @@ -249,9 +249,7 @@ pub(crate) fn new_from_iter( let ptr = set.as_ptr(); for obj in elements { - unsafe { - err::error_on_minusone(py, ffi::PySet_Add(ptr, obj.as_ptr()))?; - } + err::error_on_minusone(py, unsafe { ffi::PySet_Add(ptr, obj.as_ptr()) })?; } Ok(set) From 1889c6936960ff044a0ca06d2559f2ca28e4ba02 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Tue, 4 Jul 2023 20:54:42 +0100 Subject: [PATCH 10/57] use concrete inner for `PyErr:matches` --- src/err/mod.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/err/mod.rs b/src/err/mod.rs index 28f2b3295cb..3af7b92bbc8 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -438,14 +438,16 @@ impl PyErr { where T: ToPyObject, { - let exc = exc.to_object(py); - unsafe { ffi::PyErr_GivenExceptionMatches(self.type_ptr(py), exc.as_ptr()) != 0 } + fn inner(err: &PyErr, py: Python<'_>, exc: PyObject) -> bool { + (unsafe { ffi::PyErr_GivenExceptionMatches(err.type_ptr(py), exc.as_ptr()) }) != 0 + } + inner(self, py, exc.to_object(py)) } /// Returns true if the current exception is instance of `T`. #[inline] pub fn is_instance(&self, py: Python<'_>, ty: &PyAny) -> bool { - unsafe { ffi::PyErr_GivenExceptionMatches(self.type_ptr(py), ty.as_ptr()) != 0 } + (unsafe { ffi::PyErr_GivenExceptionMatches(self.type_ptr(py), ty.as_ptr()) }) != 0 } /// Returns true if the current exception is instance of `T`. From 6d2cfccf04e4fecdb742601dfa5cac2b28a6769c Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Wed, 5 Jul 2023 11:10:02 +0100 Subject: [PATCH 11/57] fix `SystemError` raised from `PyUnicodeDecodeError_Create` on PyPy 3.10 --- newsfragments/3297.added.md | 1 + newsfragments/3297.fixed.md | 1 + pyo3-ffi/src/abstract_.rs | 17 +++++++++++++++-- pyo3-ffi/src/pyerrors.rs | 2 +- 4 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 newsfragments/3297.added.md create mode 100644 newsfragments/3297.fixed.md diff --git a/newsfragments/3297.added.md b/newsfragments/3297.added.md new file mode 100644 index 00000000000..65d54832e2d --- /dev/null +++ b/newsfragments/3297.added.md @@ -0,0 +1 @@ +Add FFI definitions `_PyObject_CallFunction_SizeT` and `_PyObject_CallMethod_SizeT`. diff --git a/newsfragments/3297.fixed.md b/newsfragments/3297.fixed.md new file mode 100644 index 00000000000..65e211b42be --- /dev/null +++ b/newsfragments/3297.fixed.md @@ -0,0 +1 @@ +Fix `SystemError` raised in `PyUnicodeDecodeError_Create` on PyPy 3.10. diff --git a/pyo3-ffi/src/abstract_.rs b/pyo3-ffi/src/abstract_.rs index 3d167360b99..0b3b7dbb3c2 100644 --- a/pyo3-ffi/src/abstract_.rs +++ b/pyo3-ffi/src/abstract_.rs @@ -52,8 +52,21 @@ extern "C" { ... ) -> *mut PyObject; - // skipped _PyObject_CallFunction_SizeT - // skipped _PyObject_CallMethod_SizeT + #[cfg(not(Py_3_13))] + #[cfg_attr(PyPy, link_name = "_PyPyObject_CallFunction_SizeT")] + pub fn _PyObject_CallFunction_SizeT( + callable_object: *mut PyObject, + format: *const c_char, + ... + ) -> *mut PyObject; + #[cfg(not(Py_3_13))] + #[cfg_attr(PyPy, link_name = "_PyPyObject_CallMethod_SizeT")] + pub fn _PyObject_CallMethod_SizeT( + o: *mut PyObject, + method: *const c_char, + format: *const c_char, + ... + ) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyObject_CallFunctionObjArgs")] pub fn PyObject_CallFunctionObjArgs(callable: *mut PyObject, ...) -> *mut PyObject; diff --git a/pyo3-ffi/src/pyerrors.rs b/pyo3-ffi/src/pyerrors.rs index b80f009b982..9da00ea390e 100644 --- a/pyo3-ffi/src/pyerrors.rs +++ b/pyo3-ffi/src/pyerrors.rs @@ -99,7 +99,7 @@ pub unsafe fn PyUnicodeDecodeError_Create( end: Py_ssize_t, reason: *const c_char, ) -> *mut PyObject { - crate::PyObject_CallFunction( + crate::_PyObject_CallFunction_SizeT( PyExc_UnicodeDecodeError, b"sy#nns\0".as_ptr().cast::(), encoding, From 26c29e580461dd077348969c90f0bd2dcde5c459 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Wed, 5 Jul 2023 22:52:41 +0100 Subject: [PATCH 12/57] fix FFI definition `Py_EnterRecursiveCall` --- newsfragments/3300.fixed.md | 1 + pyo3-ffi/src/ceval.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 newsfragments/3300.fixed.md diff --git a/newsfragments/3300.fixed.md b/newsfragments/3300.fixed.md new file mode 100644 index 00000000000..e7c09ff5d8f --- /dev/null +++ b/newsfragments/3300.fixed.md @@ -0,0 +1 @@ +Correct FFI definition `Py_EnterRecursiveCall` to return `c_int` (was incorrectly returning `()`). diff --git a/pyo3-ffi/src/ceval.rs b/pyo3-ffi/src/ceval.rs index 1eb59b03423..7aae25f8c3e 100644 --- a/pyo3-ffi/src/ceval.rs +++ b/pyo3-ffi/src/ceval.rs @@ -76,7 +76,7 @@ extern "C" { extern "C" { #[cfg(Py_3_9)] #[cfg_attr(PyPy, link_name = "PyPy_EnterRecursiveCall")] - pub fn Py_EnterRecursiveCall(arg1: *const c_char); + pub fn Py_EnterRecursiveCall(arg1: *const c_char) -> c_int; #[cfg(Py_3_9)] #[cfg_attr(PyPy, link_name = "PyPy_LeaveRecursiveCall")] pub fn Py_LeaveRecursiveCall(); From 5ff7a72248e2bcf43e200d9e031e561e66b5d5ce Mon Sep 17 00:00:00 2001 From: Adam Reichold Date: Sat, 8 Jul 2023 21:49:18 +0200 Subject: [PATCH 13/57] Add implementation of Iterator::size_hint for PyIterator When the Python iterator backing `PyIterator` has a `__length_hint__` special method, we can use this as a lower bound for Rust's `Iterator::size_hint` to e.g. support pre-allocation of collections. This is implemented using `PyObject_LengthHint` which is not available in the stable ABI and hence so is `Iterator::size_hint`. This should be fine since this is an optimization in any case and the stable ABI is expected to have slightly worse performance overall. --- benches/bench_any.rs | 17 ++++++++++++++++- pyo3-ffi/src/boolobject.rs | 4 ++-- pyo3-macros-backend/src/attributes.rs | 2 +- src/impl_/pyclass.rs | 2 +- src/types/iterator.rs | 17 +++++++++++++++++ 5 files changed, 37 insertions(+), 5 deletions(-) diff --git a/benches/bench_any.rs b/benches/bench_any.rs index ec23cedc1c0..765497fa079 100644 --- a/benches/bench_any.rs +++ b/benches/bench_any.rs @@ -5,7 +5,7 @@ use pyo3::{ PyBool, PyByteArray, PyBytes, PyDict, PyFloat, PyFrozenSet, PyInt, PyList, PyMapping, PySequence, PySet, PyString, PyTuple, }, - PyAny, Python, + PyAny, PyResult, Python, }; #[derive(PartialEq, Eq, Debug)] @@ -71,8 +71,23 @@ fn bench_identify_object_type(b: &mut Bencher<'_>) { }); } +fn bench_collect_generic_iterator(b: &mut Bencher<'_>) { + Python::with_gil(|py| { + let collection = py.eval("list(range(1 << 20))", None, None).unwrap(); + + b.iter(|| { + collection + .iter() + .unwrap() + .collect::>>() + .unwrap() + }); + }); +} + fn criterion_benchmark(c: &mut Criterion) { c.bench_function("identify_object_type", bench_identify_object_type); + c.bench_function("collect_generic_iterator", bench_collect_generic_iterator); } criterion_group!(benches, criterion_benchmark); diff --git a/pyo3-ffi/src/boolobject.rs b/pyo3-ffi/src/boolobject.rs index 17af974b04a..0e4958c8fba 100644 --- a/pyo3-ffi/src/boolobject.rs +++ b/pyo3-ffi/src/boolobject.rs @@ -23,12 +23,12 @@ extern "C" { #[inline] pub unsafe fn Py_False() -> *mut PyObject { - addr_of_mut_shim!(_Py_FalseStruct) as *mut PyLongObject as *mut PyObject + addr_of_mut_shim!(_Py_FalseStruct) as *mut PyObject } #[inline] pub unsafe fn Py_True() -> *mut PyObject { - addr_of_mut_shim!(_Py_TrueStruct) as *mut PyLongObject as *mut PyObject + addr_of_mut_shim!(_Py_TrueStruct) as *mut PyObject } #[inline] diff --git a/pyo3-macros-backend/src/attributes.rs b/pyo3-macros-backend/src/attributes.rs index d26f13b211f..1182a78ad03 100644 --- a/pyo3-macros-backend/src/attributes.rs +++ b/pyo3-macros-backend/src/attributes.rs @@ -187,7 +187,7 @@ pub fn take_pyo3_options(attrs: &mut Vec) -> Result Default for PyClassImplCollector { impl Clone for PyClassImplCollector { fn clone(&self) -> Self { - Self::new() + *self } } diff --git a/src/types/iterator.rs b/src/types/iterator.rs index 3b45c11378a..71ca5d305b4 100644 --- a/src/types/iterator.rs +++ b/src/types/iterator.rs @@ -55,6 +55,12 @@ impl<'p> Iterator for &'p PyIterator { None => PyErr::take(py).map(Err), } } + + #[cfg(not(Py_LIMITED_API))] + fn size_hint(&self) -> (usize, Option) { + let hint = unsafe { ffi::PyObject_LengthHint(self.0.as_ptr(), 0) }; + (hint.max(0) as usize, None) + } } // PyIter_Check does not exist in the limited API until 3.8 @@ -313,4 +319,15 @@ def fibonacci(target): ); }); } + + #[test] + #[cfg(not(Py_LIMITED_API))] + fn length_hint_becomes_size_hint_lower_bound() { + Python::with_gil(|py| { + let list = py.eval("[1, 2, 3]", None, None).unwrap(); + let iter = list.iter().unwrap(); + let hint = iter.size_hint(); + assert_eq!(hint, (3, None)); + }); + } } From 999325455012a33f694fd80a520e3ce0d635b7a7 Mon Sep 17 00:00:00 2001 From: mejrs <59372212+mejrs@users.noreply.github.com> Date: Tue, 11 Jul 2023 13:56:34 +0200 Subject: [PATCH 14/57] Two is not three --- guide/src/faq.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/guide/src/faq.md b/guide/src/faq.md index a475b2fe6e7..8308acc64de 100644 --- a/guide/src/faq.md +++ b/guide/src/faq.md @@ -187,7 +187,7 @@ This happens on Windows when linking to the python DLL fails or the wrong one is - `python3X.dll` for Python 3.X, e.g. `python310.dll` for Python 3.10 - `python3.dll` when using PyO3's `abi3` feature -The DLL needs to be locatable using the [Windows DLL search order](https://learn.microsoft.com/en-us/windows/win32/dlls/dynamic-link-library-search-order#standard-search-order-for-unpackaged-apps). The two easiest ways to achieve this are: +The DLL needs to be locatable using the [Windows DLL search order](https://learn.microsoft.com/en-us/windows/win32/dlls/dynamic-link-library-search-order#standard-search-order-for-unpackaged-apps). Some ways to achieve this are: - Put the Python DLL in the same folder as your build artifacts - Add the directory containing the Python DLL to your `PATH` environment variable, for example `C:\Users\\AppData\Local\Programs\Python\Python310` - If this happens when you are *distributing* your program, consider using [PyOxidizer](https://github.com/indygreg/PyOxidizer) to package it with your binary. From 946bc82c0ef0093f18dcec8b536019e2fb04bcc8 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Tue, 11 Jul 2023 05:55:57 -0400 Subject: [PATCH 15/57] Resolve nightly clippy warning in test_field_cfg --- tests/test_field_cfg.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_field_cfg.rs b/tests/test_field_cfg.rs index dc84701c2b3..bd671641e5b 100644 --- a/tests/test_field_cfg.rs +++ b/tests/test_field_cfg.rs @@ -8,7 +8,12 @@ struct CfgClass { #[cfg(any())] pub a: u32, #[pyo3(get, set)] - #[cfg(all())] + // This is always true + #[cfg(any( + target_family = "unix", + target_family = "windows", + target_family = "wasm" + ))] pub b: u32, } From 2f1f9bf5da24d5b476d2da9ac2d2e866a7b53a5f Mon Sep 17 00:00:00 2001 From: Adam Reichold Date: Sun, 9 Jul 2023 11:52:04 +0200 Subject: [PATCH 16/57] Start adding a performance section to the guide. --- guide/src/SUMMARY.md | 1 + guide/src/performance.md | 54 +++++++++++++++++++++++++++++++++++++ newsfragments/3304.added.md | 1 + src/lib.rs | 1 + 4 files changed, 57 insertions(+) create mode 100644 guide/src/performance.md create mode 100644 newsfragments/3304.added.md diff --git a/guide/src/SUMMARY.md b/guide/src/SUMMARY.md index 2198c3792e9..e75095f4bef 100644 --- a/guide/src/SUMMARY.md +++ b/guide/src/SUMMARY.md @@ -24,6 +24,7 @@ - [Debugging](debugging.md) - [Features reference](features.md) - [Memory management](memory.md) +- [Performance](performance.md) - [Advanced topics](advanced.md) - [Building and distribution](building_and_distribution.md) - [Supporting multiple Python versions](building_and_distribution/multiple_python_versions.md) diff --git a/guide/src/performance.md b/guide/src/performance.md new file mode 100644 index 00000000000..b6918c7c828 --- /dev/null +++ b/guide/src/performance.md @@ -0,0 +1,54 @@ +# Performance + +To achieve the best possible performance, it is useful to be aware of several tricks and sharp edges concerning PyO3's API. + +## `extract` versus `downcast` + +Pythonic API implemented using PyO3 are often polymorphic, i.e. they will accept `&PyAny` and try to turn this into multiple more concrete types to which the requested operation is applied. This often leads to chains of calls to `extract`, e.g. + +```rust +# #![allow(dead_code)] +# use pyo3::prelude::*; +# use pyo3::{exceptions::PyTypeError, types::PyList}; + +fn frobnicate_list(list: &PyList) -> PyResult<&PyAny> { + todo!() +} + +fn frobnicate_vec(vec: Vec<&PyAny>) -> PyResult<&PyAny> { + todo!() +} + +#[pyfunction] +fn frobnicate(value: &PyAny) -> PyResult<&PyAny> { + if let Ok(list) = value.extract::<&PyList>() { + frobnicate_list(list) + } else if let Ok(vec) = value.extract::>() { + frobnicate_vec(vec) + } else { + Err(PyTypeError::new_err("Cannot frobnicate that type.")) + } +} +``` + +This suboptimal as the `FromPyObject` trait requires `extract` to have a `Result` return type. For native types like `PyList`, it faster to use `downcast` (which `extract` calls internally) when the error value is ignored. This avoids the costly conversion of a `PyDowncastError` to a `PyErr` required to fulfil the `FromPyObject` contract, i.e. + +```rust +# #![allow(dead_code)] +# use pyo3::prelude::*; +# use pyo3::{exceptions::PyTypeError, types::PyList}; +# fn frobnicate_list(list: &PyList) -> PyResult<&PyAny> { todo!() } +# fn frobnicate_vec(vec: Vec<&PyAny>) -> PyResult<&PyAny> { todo!() } +# +#[pyfunction] +fn frobnicate(value: &PyAny) -> PyResult<&PyAny> { + // Use `downcast` instead of `extract` as turning `PyDowncastError` into `PyErr` is quite costly. + if let Ok(list) = value.downcast::() { + frobnicate_list(list) + } else if let Ok(vec) = value.extract::>() { + frobnicate_vec(vec) + } else { + Err(PyTypeError::new_err("Cannot frobnicate that type.")) + } +} +``` diff --git a/newsfragments/3304.added.md b/newsfragments/3304.added.md new file mode 100644 index 00000000000..f3b62eba58e --- /dev/null +++ b/newsfragments/3304.added.md @@ -0,0 +1 @@ +Added a "performance" section to the guide collecting performance-related tricks and problems. diff --git a/src/lib.rs b/src/lib.rs index ded43370148..8114e791339 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -504,6 +504,7 @@ pub mod doc_test { "guide/src/migration.md" => guide_migration_md, "guide/src/module.md" => guide_module_md, "guide/src/parallelism.md" => guide_parallelism_md, + "guide/src/performance.md" => guide_performance_md, "guide/src/python_from_rust.md" => guide_python_from_rust_md, "guide/src/python_typing_hints.md" => guide_python_typing_hints_md, "guide/src/rust_cpython.md" => guide_rust_cpython_md, From 26a40a5697be87db99cec629aa82f8b7b279c05b Mon Sep 17 00:00:00 2001 From: Adam Reichold Date: Sun, 9 Jul 2023 15:02:12 +0200 Subject: [PATCH 17/57] Add another performance subsection on implicit access to GIL token. --- guide/src/performance.md | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/guide/src/performance.md b/guide/src/performance.md index b6918c7c828..23fb59c4e90 100644 --- a/guide/src/performance.md +++ b/guide/src/performance.md @@ -52,3 +52,43 @@ fn frobnicate(value: &PyAny) -> PyResult<&PyAny> { } } ``` + +## Access to GIL-bound reference implies access to GIL token + +Calling `Python::with_gil` is effectively a no-op when the GIL is already held, but checking that this is the case still has a cost. If an existing GIL token can not be accessed, for example when implementing a pre-existing trait, but a GIL-bound reference is available, this cost can be avoided by exploiting that access to GIL-bound reference gives zero-cost access to a GIL token via `PyAny::py`. + +For example, instead of writing + +```rust +# #![allow(dead_code)] +# use pyo3::prelude::*; +# use pyo3::types::PyList; + +struct Foo(Py); + +struct FooRef<'a>(&'a PyList); + +impl PartialEq for FooRef<'_> { + fn eq(&self, other: &Foo) -> bool { + Python::with_gil(|py| self.0.len() == other.0.as_ref(py).len()) + } +} +``` + +use more efficient + +```rust +# #![allow(dead_code)] +# use pyo3::prelude::*; +# use pyo3::types::PyList; +# struct Foo(Py); +# struct FooRef<'a>(&'a PyList); +# +impl PartialEq for FooRef<'_> { + fn eq(&self, other: &Foo) -> bool { + // Access to `&'a PyAny` implies access to `Python<'a>`. + let py = self.0.py(); + self.0.len() == other.0.as_ref(py).len() + } +} +``` From 52ba4fe354e6c71740777894aafbce28d0039329 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Tue, 11 Jul 2023 19:59:12 +0100 Subject: [PATCH 18/57] remove some dead fields from FnArg --- pyo3-macros-backend/src/method.rs | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/pyo3-macros-backend/src/method.rs b/pyo3-macros-backend/src/method.rs index a867a301378..62ae9ef78cd 100644 --- a/pyo3-macros-backend/src/method.rs +++ b/pyo3-macros-backend/src/method.rs @@ -14,8 +14,6 @@ use syn::Result; #[derive(Clone, Debug)] pub struct FnArg<'a> { pub name: &'a syn::Ident, - pub by_ref: &'a Option, - pub mutability: &'a Option, pub ty: &'a syn::Type, pub optional: Option<&'a syn::Type>, pub default: Option, @@ -38,20 +36,13 @@ impl<'a> FnArg<'a> { } let arg_attrs = PyFunctionArgPyO3Attributes::from_attrs(&mut cap.attrs)?; - let (ident, by_ref, mutability) = match &*cap.pat { - syn::Pat::Ident(syn::PatIdent { - ident, - by_ref, - mutability, - .. - }) => (ident, by_ref, mutability), + let ident = match &*cap.pat { + syn::Pat::Ident(syn::PatIdent { ident, .. }) => ident, other => return Err(handle_argument_error(other)), }; Ok(FnArg { name: ident, - by_ref, - mutability, ty: &cap.ty, optional: utils::option_type_argument(&cap.ty), default: None, From 86012d01a34e873217708a465d00c80d4168ef02 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Tue, 11 Jul 2023 20:13:35 +0100 Subject: [PATCH 19/57] improve error span for mutable access to `#[pyclass(frozen)]` --- pyo3-macros-backend/src/method.rs | 76 ++++++++++++------- pyo3-macros-backend/src/pyclass.rs | 7 +- pyo3-macros-backend/src/pymethod.rs | 37 +++------ tests/ui/invalid_frozen_pyclass_borrow.rs | 11 +++ tests/ui/invalid_frozen_pyclass_borrow.stderr | 34 +++++++-- 5 files changed, 100 insertions(+), 65 deletions(-) diff --git a/pyo3-macros-backend/src/method.rs b/pyo3-macros-backend/src/method.rs index 62ae9ef78cd..7ea83b19f6b 100644 --- a/pyo3-macros-backend/src/method.rs +++ b/pyo3-macros-backend/src/method.rs @@ -9,7 +9,7 @@ use quote::ToTokens; use quote::{quote, quote_spanned}; use syn::ext::IdentExt; use syn::spanned::Spanned; -use syn::Result; +use syn::{Result, Token}; #[derive(Clone, Debug)] pub struct FnArg<'a> { @@ -137,7 +137,7 @@ impl FnType { #[derive(Clone, Debug)] pub enum SelfType { - Receiver { mutable: bool }, + Receiver { mutable: bool, span: Span }, TryFromPyCell(Span), } @@ -147,38 +147,55 @@ pub enum ExtractErrorMode { Raise, } +impl ExtractErrorMode { + pub fn handle_error(self, py: &syn::Ident, extract: TokenStream) -> TokenStream { + match self { + ExtractErrorMode::Raise => quote! { #extract? }, + ExtractErrorMode::NotImplemented => quote! { + match #extract { + ::std::result::Result::Ok(value) => value, + ::std::result::Result::Err(_) => { return _pyo3::callback::convert(#py, #py.NotImplemented()); }, + } + }, + } + } +} + impl SelfType { pub fn receiver(&self, cls: &syn::Type, error_mode: ExtractErrorMode) -> TokenStream { - let cell = match error_mode { - ExtractErrorMode::Raise => { - quote! { _py.from_borrowed_ptr::<_pyo3::PyAny>(_slf).downcast::<_pyo3::PyCell<#cls>>()? } - } - ExtractErrorMode::NotImplemented => { - quote! { - match _py.from_borrowed_ptr::<_pyo3::PyAny>(_slf).downcast::<_pyo3::PyCell<#cls>>() { - ::std::result::Result::Ok(cell) => cell, - ::std::result::Result::Err(_) => return _pyo3::callback::convert(_py, _py.NotImplemented()), - } - } - } - }; + let py = syn::Ident::new("_py", Span::call_site()); + let _slf = syn::Ident::new("_slf", Span::call_site()); match self { - SelfType::Receiver { mutable: false } => { - quote! { - let _cell = #cell; - let _ref = _cell.try_borrow()?; - let _slf: &#cls = &*_ref; - } - } - SelfType::Receiver { mutable: true } => { - quote! { - let _cell = #cell; - let mut _ref = _cell.try_borrow_mut()?; - let _slf: &mut #cls = &mut *_ref; + SelfType::Receiver { span, mutable } => { + let (method, mutability) = if *mutable { + ( + quote_spanned! { *span => extract_pyclass_ref_mut }, + Some(Token![mut](*span)), + ) + } else { + (quote_spanned! { *span => extract_pyclass_ref }, None) + }; + let extract = error_mode.handle_error( + &py, + quote_spanned! { *span => + _pyo3::impl_::extract_argument::#method( + #py.from_borrowed_ptr::<_pyo3::PyAny>(#_slf), + &mut holder, + ) + }, + ); + quote_spanned! { *span => + let mut holder = _pyo3::impl_::extract_argument::FunctionArgumentHolder::INIT; + let #_slf: &#mutability #cls = #extract; } } SelfType::TryFromPyCell(span) => { - let _slf = quote! { _slf }; + let cell = error_mode.handle_error( + &py, + quote!{ + _py.from_borrowed_ptr::<_pyo3::PyAny>(_slf).downcast::<_pyo3::PyCell<#cls>>() + } + ); quote_spanned! { *span => let _cell = #cell; #[allow(clippy::useless_conversion)] // In case _slf is PyCell @@ -249,8 +266,9 @@ pub fn parse_method_receiver(arg: &syn::FnArg) -> Result { ) => { bail_spanned!(recv.span() => RECEIVER_BY_VALUE_ERR); } - syn::FnArg::Receiver(syn::Receiver { mutability, .. }) => Ok(SelfType::Receiver { + syn::FnArg::Receiver(recv @ syn::Receiver { mutability, .. }) => Ok(SelfType::Receiver { mutable: mutability.is_some(), + span: recv.span(), }), syn::FnArg::Typed(syn::PatType { ty, .. }) => { if let syn::Type::ImplTrait(_) = &**ty { diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index f723da0951f..ca5d233eed0 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -1,5 +1,6 @@ use std::borrow::Cow; +use crate::attributes::kw::frozen; use crate::attributes::{ self, kw, take_pyo3_options, CrateAttribute, ExtendsAttribute, FreelistAttribute, ModuleAttribute, NameAttribute, NameLitStr, TextSignatureAttribute, @@ -355,7 +356,7 @@ fn impl_class( cls, args, methods_type, - descriptors_to_items(cls, field_options)?, + descriptors_to_items(cls, args.options.frozen, field_options)?, vec![], ) .doc(doc) @@ -674,6 +675,7 @@ fn extract_variant_data(variant: &mut syn::Variant) -> syn::Result, field_options: Vec<(&syn::Field, FieldPyO3Options)>, ) -> syn::Result> { let ty = syn::parse_quote!(#cls); @@ -700,7 +702,8 @@ fn descriptors_to_items( items.push(getter); } - if options.set.is_some() { + if let Some(set) = options.set { + ensure_spanned!(frozen.is_none(), set.span() => "cannot use `#[pyo3(set)]` on a `frozen` class"); let setter = impl_py_setter_def( &ty, PropertyType::Descriptor { diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index 75c52e3b053..4297eee9c41 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -520,9 +520,11 @@ pub fn impl_py_setter_def( }; let slf = match property_type { - PropertyType::Descriptor { .. } => { - SelfType::Receiver { mutable: true }.receiver(cls, ExtractErrorMode::Raise) + PropertyType::Descriptor { .. } => SelfType::Receiver { + mutable: true, + span: Span::call_site(), } + .receiver(cls, ExtractErrorMode::Raise), PropertyType::Function { self_type, .. } => { self_type.receiver(cls, ExtractErrorMode::Raise) } @@ -633,9 +635,11 @@ pub fn impl_py_getter_def( }; let slf = match property_type { - PropertyType::Descriptor { .. } => { - SelfType::Receiver { mutable: false }.receiver(cls, ExtractErrorMode::Raise) + PropertyType::Descriptor { .. } => SelfType::Receiver { + mutable: false, + span: Span::call_site(), } + .receiver(cls, ExtractErrorMode::Raise), PropertyType::Function { self_type, .. } => { self_type.receiver(cls, ExtractErrorMode::Raise) } @@ -948,8 +952,7 @@ impl Ty { #ident.to_borrowed_any(#py) }, ), - Ty::CompareOp => handle_error( - extract_error_mode, + Ty::CompareOp => extract_error_mode.handle_error( py, quote! { _pyo3::class::basic::CompareOp::from_raw(#ident) @@ -958,8 +961,7 @@ impl Ty { ), Ty::PySsizeT => { let ty = arg.ty; - handle_error( - extract_error_mode, + extract_error_mode.handle_error( py, quote! { ::std::convert::TryInto::<#ty>::try_into(#ident).map_err(|e| _pyo3::exceptions::PyValueError::new_err(e.to_string())) @@ -972,30 +974,13 @@ impl Ty { } } -fn handle_error( - extract_error_mode: ExtractErrorMode, - py: &syn::Ident, - extract: TokenStream, -) -> TokenStream { - match extract_error_mode { - ExtractErrorMode::Raise => quote! { #extract? }, - ExtractErrorMode::NotImplemented => quote! { - match #extract { - ::std::result::Result::Ok(value) => value, - ::std::result::Result::Err(_) => { return _pyo3::callback::convert(#py, #py.NotImplemented()); }, - } - }, - } -} - fn extract_object( extract_error_mode: ExtractErrorMode, py: &syn::Ident, name: &str, source: TokenStream, ) -> TokenStream { - handle_error( - extract_error_mode, + extract_error_mode.handle_error( py, quote! { _pyo3::impl_::extract_argument::extract_argument( diff --git a/tests/ui/invalid_frozen_pyclass_borrow.rs b/tests/ui/invalid_frozen_pyclass_borrow.rs index c7b2f27b5b1..1f18eab6170 100644 --- a/tests/ui/invalid_frozen_pyclass_borrow.rs +++ b/tests/ui/invalid_frozen_pyclass_borrow.rs @@ -6,6 +6,11 @@ pub struct Foo { field: u32, } +#[pymethods] +impl Foo { + fn mut_method(&mut self) {} +} + fn borrow_mut_fails(foo: Py, py: Python) { let borrow = foo.as_ref(py).borrow_mut(); } @@ -28,4 +33,10 @@ fn pyclass_get_of_mutable_class_fails(class: &PyCell) { class.get(); } +#[pyclass(frozen)] +pub struct SetOnFrozenClass { + #[pyo3(set)] + field: u32, +} + fn main() {} diff --git a/tests/ui/invalid_frozen_pyclass_borrow.stderr b/tests/ui/invalid_frozen_pyclass_borrow.stderr index b91d5c0cefb..e68c2a588f3 100644 --- a/tests/ui/invalid_frozen_pyclass_borrow.stderr +++ b/tests/ui/invalid_frozen_pyclass_borrow.stderr @@ -1,7 +1,25 @@ +error: cannot use `#[pyo3(set)]` on a `frozen` class + --> tests/ui/invalid_frozen_pyclass_borrow.rs:38:12 + | +38 | #[pyo3(set)] + | ^^^ + +error[E0271]: type mismatch resolving `::Frozen == False` + --> tests/ui/invalid_frozen_pyclass_borrow.rs:11:19 + | +11 | fn mut_method(&mut self) {} + | ^ expected `False`, found `True` + | +note: required by a bound in `extract_pyclass_ref_mut` + --> src/impl_/extract_argument.rs + | + | pub fn extract_pyclass_ref_mut<'a, 'py: 'a, T: PyClass>( + | ^^^^^^^^^^^^^^ required by this bound in `extract_pyclass_ref_mut` + error[E0271]: type mismatch resolving `::Frozen == False` - --> tests/ui/invalid_frozen_pyclass_borrow.rs:10:33 + --> tests/ui/invalid_frozen_pyclass_borrow.rs:15:33 | -10 | let borrow = foo.as_ref(py).borrow_mut(); +15 | let borrow = foo.as_ref(py).borrow_mut(); | ^^^^^^^^^^ expected `False`, found `True` | note: required by a bound in `pyo3::PyCell::::borrow_mut` @@ -11,9 +29,9 @@ note: required by a bound in `pyo3::PyCell::::borrow_mut` | ^^^^^^^^^^^^^^ required by this bound in `PyCell::::borrow_mut` error[E0271]: type mismatch resolving `::Frozen == False` - --> tests/ui/invalid_frozen_pyclass_borrow.rs:20:35 + --> tests/ui/invalid_frozen_pyclass_borrow.rs:25:35 | -20 | let borrow = child.as_ref(py).borrow_mut(); +25 | let borrow = child.as_ref(py).borrow_mut(); | ^^^^^^^^^^ expected `False`, found `True` | note: required by a bound in `pyo3::PyCell::::borrow_mut` @@ -23,9 +41,9 @@ note: required by a bound in `pyo3::PyCell::::borrow_mut` | ^^^^^^^^^^^^^^ required by this bound in `PyCell::::borrow_mut` error[E0271]: type mismatch resolving `::Frozen == True` - --> tests/ui/invalid_frozen_pyclass_borrow.rs:24:11 + --> tests/ui/invalid_frozen_pyclass_borrow.rs:29:11 | -24 | class.get(); +29 | class.get(); | ^^^ expected `True`, found `False` | note: required by a bound in `pyo3::Py::::get` @@ -35,9 +53,9 @@ note: required by a bound in `pyo3::Py::::get` | ^^^^^^^^^^^^^ required by this bound in `Py::::get` error[E0271]: type mismatch resolving `::Frozen == True` - --> tests/ui/invalid_frozen_pyclass_borrow.rs:28:11 + --> tests/ui/invalid_frozen_pyclass_borrow.rs:33:11 | -28 | class.get(); +33 | class.get(); | ^^^ expected `True`, found `False` | note: required by a bound in `pyo3::PyCell::::get` From 1434837eddc8156ea13ebcfccc07ead80396d5f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pan=20Pi=C5=82karz?= Date: Wed, 12 Jul 2023 15:51:18 +0200 Subject: [PATCH 20/57] Tiny grammar fix in error_handling.md --- guide/src/function/error_handling.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/guide/src/function/error_handling.md b/guide/src/function/error_handling.md index 38cc602eb72..b0f63885cdf 100644 --- a/guide/src/function/error_handling.md +++ b/guide/src/function/error_handling.md @@ -13,7 +13,7 @@ Rust code uses the generic [`Result`] enum to propagate errors. The error PyO3 has the [`PyErr`] type which represents a Python exception. If a PyO3 API could result in a Python exception being raised, the return type of that `API` will be [`PyResult`], which is an alias for the type `Result`. In summary: -- When Python exceptions are raised and caught by PyO3, the exception will stored in the `Err` variant of the `PyResult`. +- When Python exceptions are raised and caught by PyO3, the exception will be stored in the `Err` variant of the `PyResult`. - Passing Python exceptions through Rust code then uses all the "normal" techniques such as the `?` operator, with `PyErr` as the error type. - Finally, when a `PyResult` crosses from Rust back to Python via PyO3, if the result is an `Err` variant the contained exception will be raised. From 282ac9aa4e2659b976ff75aefe1292d449d31dda Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Wed, 12 Jul 2023 23:28:27 +0100 Subject: [PATCH 21/57] codecov: ignore coverage for pytests directory --- codecov.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codecov.yml b/codecov.yml index cfef461049a..1d903951f57 100644 --- a/codecov.yml +++ b/codecov.yml @@ -10,5 +10,5 @@ coverage: ignore: - tests/*.rs + - pytests/*.rs - src/test_hygiene/*.rs - - src/impl_/ghost.rs From 4edf263a4ae5ae6b6fd095247eac719d8cfff47b Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Fri, 14 Jul 2023 10:28:17 +0100 Subject: [PATCH 22/57] ci: updates for rust 1.71 --- benches/bench_dict.rs | 2 +- benches/bench_list.rs | 2 +- benches/bench_set.rs | 2 +- benches/bench_tuple.rs | 2 +- pyo3-macros-backend/src/pyimpl.rs | 2 +- pyo3-macros-backend/src/utils.rs | 2 +- pytests/src/dict_iter.rs | 2 +- src/conversions/hashbrown.rs | 2 +- src/conversions/indexmap.rs | 2 +- src/conversions/std/map.rs | 4 +-- src/exceptions.rs | 30 +++++++++---------- src/types/dict.rs | 10 +++---- src/types/frozenset.rs | 2 +- src/types/list.rs | 2 +- src/types/sequence.rs | 2 +- src/types/set.rs | 2 +- src/types/string.rs | 4 +-- tests/test_dict_iter.rs | 2 +- tests/ui/invalid_closure.stderr | 2 ++ tests/ui/invalid_frozen_pyclass_borrow.stderr | 12 ++++++++ tests/ui/not_send.stderr | 27 ++++++++++++++--- tests/ui/not_send2.stderr | 15 ++++++++-- 22 files changed, 88 insertions(+), 44 deletions(-) diff --git a/benches/bench_dict.rs b/benches/bench_dict.rs index 2b92159d10a..64398a65e39 100644 --- a/benches/bench_dict.rs +++ b/benches/bench_dict.rs @@ -10,7 +10,7 @@ fn iter_dict(b: &mut Bencher<'_>) { let dict = (0..LEN as u64).map(|i| (i, i * 2)).into_py_dict(py); let mut sum = 0; b.iter(|| { - for (k, _v) in dict.iter() { + for (k, _v) in dict { let i: u64 = k.extract().unwrap(); sum += i; } diff --git a/benches/bench_list.rs b/benches/bench_list.rs index dd305db727b..dd2e3db12ab 100644 --- a/benches/bench_list.rs +++ b/benches/bench_list.rs @@ -9,7 +9,7 @@ fn iter_list(b: &mut Bencher<'_>) { let list = PyList::new(py, 0..LEN); let mut sum = 0; b.iter(|| { - for x in list.iter() { + for x in list { let i: u64 = x.extract().unwrap(); sum += i; } diff --git a/benches/bench_set.rs b/benches/bench_set.rs index 58abc956337..1bc815997b9 100644 --- a/benches/bench_set.rs +++ b/benches/bench_set.rs @@ -24,7 +24,7 @@ fn iter_set(b: &mut Bencher<'_>) { let set = PySet::new(py, &(0..LEN).collect::>()).unwrap(); let mut sum = 0; b.iter(|| { - for x in set.iter() { + for x in set { let i: u64 = x.extract().unwrap(); sum += i; } diff --git a/benches/bench_tuple.rs b/benches/bench_tuple.rs index 07d7fe2dbda..e26c1700338 100644 --- a/benches/bench_tuple.rs +++ b/benches/bench_tuple.rs @@ -9,7 +9,7 @@ fn iter_tuple(b: &mut Bencher<'_>) { let tuple = PyTuple::new(py, 0..LEN); let mut sum = 0; b.iter(|| { - for x in tuple.iter() { + for x in tuple { let i: u64 = x.extract().unwrap(); sum += i; } diff --git a/pyo3-macros-backend/src/pyimpl.rs b/pyo3-macros-backend/src/pyimpl.rs index 0ab0695eca5..fd961aea190 100644 --- a/pyo3-macros-backend/src/pyimpl.rs +++ b/pyo3-macros-backend/src/pyimpl.rs @@ -97,7 +97,7 @@ pub fn impl_methods( let mut implemented_proto_fragments = HashSet::new(); - for iimpl in impls.iter_mut() { + for iimpl in impls { match iimpl { syn::ImplItem::Method(meth) => { let mut fun_options = PyFunctionOptions::from_attrs(&mut meth.attrs)?; diff --git a/pyo3-macros-backend/src/utils.rs b/pyo3-macros-backend/src/utils.rs index 5cffe120925..c37dba579e7 100644 --- a/pyo3-macros-backend/src/utils.rs +++ b/pyo3-macros-backend/src/utils.rs @@ -79,7 +79,7 @@ pub fn get_doc(attrs: &[syn::Attribute], mut text_signature: Option) -> let mut first = true; let mut current_part = text_signature.unwrap_or_default(); - for attr in attrs.iter() { + for attr in attrs { if attr.path.is_ident("doc") { if let Ok(DocArgs { _eq_token, diff --git a/pytests/src/dict_iter.rs b/pytests/src/dict_iter.rs index 35f3ad8d6b0..5f5992b6efc 100644 --- a/pytests/src/dict_iter.rs +++ b/pytests/src/dict_iter.rs @@ -22,7 +22,7 @@ impl DictSize { fn iter_dict(&mut self, _py: Python<'_>, dict: &PyDict) -> PyResult { let mut seen = 0u32; - for (sym, values) in dict.iter() { + for (sym, values) in dict { seen += 1; println!( "{:4}/{:4} iterations:{}=>{}", diff --git a/src/conversions/hashbrown.rs b/src/conversions/hashbrown.rs index c7a99ce0dcc..1c59fd19e7e 100644 --- a/src/conversions/hashbrown.rs +++ b/src/conversions/hashbrown.rs @@ -62,7 +62,7 @@ where fn extract(ob: &'source PyAny) -> Result { let dict: &PyDict = ob.downcast()?; let mut ret = hashbrown::HashMap::with_capacity_and_hasher(dict.len(), S::default()); - for (k, v) in dict.iter() { + for (k, v) in dict { ret.insert(K::extract(k)?, V::extract(v)?); } Ok(ret) diff --git a/src/conversions/indexmap.rs b/src/conversions/indexmap.rs index b6ed7a2ea20..706f5c4835f 100644 --- a/src/conversions/indexmap.rs +++ b/src/conversions/indexmap.rs @@ -130,7 +130,7 @@ where fn extract(ob: &'source PyAny) -> Result { let dict: &PyDict = ob.downcast()?; let mut ret = indexmap::IndexMap::with_capacity_and_hasher(dict.len(), S::default()); - for (k, v) in dict.iter() { + for (k, v) in dict { ret.insert(K::extract(k)?, V::extract(v)?); } Ok(ret) diff --git a/src/conversions/std/map.rs b/src/conversions/std/map.rs index f7e9b58ce91..f79b415b9fa 100644 --- a/src/conversions/std/map.rs +++ b/src/conversions/std/map.rs @@ -74,7 +74,7 @@ where fn extract(ob: &'source PyAny) -> Result { let dict: &PyDict = ob.downcast()?; let mut ret = collections::HashMap::with_capacity_and_hasher(dict.len(), S::default()); - for (k, v) in dict.iter() { + for (k, v) in dict { ret.insert(K::extract(k)?, V::extract(v)?); } Ok(ret) @@ -94,7 +94,7 @@ where fn extract(ob: &'source PyAny) -> Result { let dict: &PyDict = ob.downcast()?; let mut ret = collections::BTreeMap::new(); - for (k, v) in dict.iter() { + for (k, v) in dict { ret.insert(K::extract(k)?, V::extract(v)?); } Ok(ret) diff --git a/src/exceptions.rs b/src/exceptions.rs index 9cbc8587fe2..ff57b078a7a 100644 --- a/src/exceptions.rs +++ b/src/exceptions.rs @@ -724,7 +724,7 @@ impl_native_exception!( #[cfg(test)] macro_rules! test_exception { - ($exc_ty:ident $(, $constructor:expr)?) => { + ($exc_ty:ident $(, |$py:tt| $constructor:expr )?) => { #[allow(non_snake_case)] #[test] fn $exc_ty () { @@ -735,7 +735,7 @@ macro_rules! test_exception { let err: $crate::PyErr = { None $( - .or(Some($constructor(py))) + .or(Some({ let $py = py; $constructor })) )? .unwrap_or($exc_ty::new_err("a test exception")) }; @@ -770,12 +770,12 @@ pub mod asyncio { test_exception!(CancelledError); test_exception!(InvalidStateError); test_exception!(TimeoutError); - test_exception!(IncompleteReadError, |_| { - IncompleteReadError::new_err(("partial", "expected")) - }); - test_exception!(LimitOverrunError, |_| { - LimitOverrunError::new_err(("message", "consumed")) - }); + test_exception!(IncompleteReadError, |_| IncompleteReadError::new_err(( + "partial", "expected" + ))); + test_exception!(LimitOverrunError, |_| LimitOverrunError::new_err(( + "message", "consumed" + ))); test_exception!(QueueEmpty); test_exception!(QueueFull); } @@ -1033,9 +1033,10 @@ mod tests { }); } #[cfg(Py_3_11)] - test_exception!(PyBaseExceptionGroup, |_| { - PyBaseExceptionGroup::new_err(("msg", vec![PyValueError::new_err("err")])) - }); + test_exception!(PyBaseExceptionGroup, |_| PyBaseExceptionGroup::new_err(( + "msg", + vec![PyValueError::new_err("err")] + ))); test_exception!(PyBaseException); test_exception!(PyException); test_exception!(PyStopAsyncIteration); @@ -1072,10 +1073,9 @@ mod tests { let err = std::str::from_utf8(invalid_utf8).expect_err("should be invalid utf8"); PyErr::from_value(PyUnicodeDecodeError::new_utf8(py, invalid_utf8, err).unwrap()) }); - test_exception!(PyUnicodeEncodeError, |py: Python<'_>| { - py.eval("chr(40960).encode('ascii')", None, None) - .unwrap_err() - }); + test_exception!(PyUnicodeEncodeError, |py| py + .eval("chr(40960).encode('ascii')", None, None) + .unwrap_err()); test_exception!(PyUnicodeTranslateError, |_| { PyUnicodeTranslateError::new_err(("\u{3042}", 0, 1, "ouch")) }); diff --git a/src/types/dict.rs b/src/types/dict.rs index cabad4716b3..74e2d79adb5 100644 --- a/src/types/dict.rs +++ b/src/types/dict.rs @@ -654,7 +654,7 @@ mod tests { // Can't just compare against a vector of tuples since we don't have a guaranteed ordering. let mut key_sum = 0; let mut value_sum = 0; - for el in dict.items().iter() { + for el in dict.items() { let tuple = el.downcast::().unwrap(); key_sum += tuple.get_item(0).unwrap().extract::().unwrap(); value_sum += tuple.get_item(1).unwrap().extract::().unwrap(); @@ -675,7 +675,7 @@ mod tests { let dict: &PyDict = ob.downcast(py).unwrap(); // Can't just compare against a vector of tuples since we don't have a guaranteed ordering. let mut key_sum = 0; - for el in dict.keys().iter() { + for el in dict.keys() { key_sum += el.extract::().unwrap(); } assert_eq!(7 + 8 + 9, key_sum); @@ -693,7 +693,7 @@ mod tests { let dict: &PyDict = ob.downcast(py).unwrap(); // Can't just compare against a vector of tuples since we don't have a guaranteed ordering. let mut values_sum = 0; - for el in dict.values().iter() { + for el in dict.values() { values_sum += el.extract::().unwrap(); } assert_eq!(32 + 42 + 123, values_sum); @@ -711,7 +711,7 @@ mod tests { let dict: &PyDict = ob.downcast(py).unwrap(); let mut key_sum = 0; let mut value_sum = 0; - for (key, value) in dict.iter() { + for (key, value) in dict { key_sum += key.extract::().unwrap(); value_sum += value.extract::().unwrap(); } @@ -731,7 +731,7 @@ mod tests { let ob = v.to_object(py); let dict: &PyDict = ob.downcast(py).unwrap(); - for (key, value) in dict.iter() { + for (key, value) in dict { dict.set_item(key, value.extract::().unwrap() + 7) .unwrap(); } diff --git a/src/types/frozenset.rs b/src/types/frozenset.rs index 7bd8fe7c3c4..ff590495ba4 100644 --- a/src/types/frozenset.rs +++ b/src/types/frozenset.rs @@ -260,7 +260,7 @@ mod tests { let set = PyFrozenSet::new(py, &[1]).unwrap(); // iter method - for el in set.iter() { + for el in set { assert_eq!(1i32, el.extract::().unwrap()); } diff --git a/src/types/list.rs b/src/types/list.rs index 9294ee196c0..e86086467f9 100644 --- a/src/types/list.rs +++ b/src/types/list.rs @@ -474,7 +474,7 @@ mod tests { let v = vec![2, 3, 5, 7]; let list = PyList::new(py, &v); let mut idx = 0; - for el in list.iter() { + for el in list { assert_eq!(v[idx], el.extract::().unwrap()); idx += 1; } diff --git a/src/types/sequence.rs b/src/types/sequence.rs index 1db0cd53482..04570459bbc 100644 --- a/src/types/sequence.rs +++ b/src/types/sequence.rs @@ -773,7 +773,7 @@ mod tests { let seq = ob.downcast::(py).unwrap(); let repeat_seq = seq.repeat(3).unwrap(); assert_eq!(6, repeat_seq.len().unwrap()); - let repeated = vec!["foo", "bar", "foo", "bar", "foo", "bar"]; + let repeated = ["foo", "bar", "foo", "bar", "foo", "bar"]; for (el, rpt) in repeat_seq.iter().unwrap().zip(repeated.iter()) { assert_eq!(*rpt, el.unwrap().extract::().unwrap()); } diff --git a/src/types/set.rs b/src/types/set.rs index 2c1c2925b3e..d4a64d7b581 100644 --- a/src/types/set.rs +++ b/src/types/set.rs @@ -356,7 +356,7 @@ mod tests { let set = PySet::new(py, &[1]).unwrap(); // iter method - for el in set.iter() { + for el in set { assert_eq!(1i32, el.extract::<'_, i32>().unwrap()); } diff --git a/src/types/string.rs b/src/types/string.rs index 05b7109f280..80b0665b402 100644 --- a/src/types/string.rs +++ b/src/types/string.rs @@ -296,7 +296,7 @@ mod tests { #[test] fn test_to_str_surrogate() { Python::with_gil(|py| { - let obj: PyObject = py.eval(r#"'\ud800'"#, None, None).unwrap().into(); + let obj: PyObject = py.eval(r"'\ud800'", None, None).unwrap().into(); let py_string: &PyString = obj.downcast(py).unwrap(); assert!(py_string.to_str().is_err()); }) @@ -316,7 +316,7 @@ mod tests { fn test_to_string_lossy() { Python::with_gil(|py| { let obj: PyObject = py - .eval(r#"'🐈 Hello \ud800World'"#, None, None) + .eval(r"'🐈 Hello \ud800World'", None, None) .unwrap() .into(); let py_string: &PyString = obj.downcast(py).unwrap(); diff --git a/tests/test_dict_iter.rs b/tests/test_dict_iter.rs index 5d30d4c766a..dc32eb61fd7 100644 --- a/tests/test_dict_iter.rs +++ b/tests/test_dict_iter.rs @@ -8,7 +8,7 @@ fn iter_dict_nosegv() { const LEN: usize = 10_000_000; let dict = (0..LEN as u64).map(|i| (i, i * 2)).into_py_dict(py); let mut sum = 0; - for (k, _v) in dict.iter() { + for (k, _v) in dict { let i: u64 = k.extract().unwrap(); sum += i; } diff --git a/tests/ui/invalid_closure.stderr b/tests/ui/invalid_closure.stderr index 4844f766887..9a2a957592c 100644 --- a/tests/ui/invalid_closure.stderr +++ b/tests/ui/invalid_closure.stderr @@ -1,6 +1,8 @@ error[E0597]: `local_data` does not live long enough --> tests/ui/invalid_closure.rs:7:27 | +6 | let local_data = vec![0, 1, 2, 3, 4]; + | ---------- binding `local_data` declared here 7 | let ref_: &[u8] = &local_data; | ^^^^^^^^^^^ borrowed value does not live long enough ... diff --git a/tests/ui/invalid_frozen_pyclass_borrow.stderr b/tests/ui/invalid_frozen_pyclass_borrow.stderr index e68c2a588f3..5e09d512ae7 100644 --- a/tests/ui/invalid_frozen_pyclass_borrow.stderr +++ b/tests/ui/invalid_frozen_pyclass_borrow.stderr @@ -25,6 +25,9 @@ error[E0271]: type mismatch resolving `::Frozen == False` note: required by a bound in `pyo3::PyCell::::borrow_mut` --> src/pycell.rs | + | pub fn borrow_mut(&self) -> PyRefMut<'_, T> + | ---------- required by a bound in this associated function + | where | T: PyClass, | ^^^^^^^^^^^^^^ required by this bound in `PyCell::::borrow_mut` @@ -37,6 +40,9 @@ error[E0271]: type mismatch resolving `::Frozen == Fa note: required by a bound in `pyo3::PyCell::::borrow_mut` --> src/pycell.rs | + | pub fn borrow_mut(&self) -> PyRefMut<'_, T> + | ---------- required by a bound in this associated function + | where | T: PyClass, | ^^^^^^^^^^^^^^ required by this bound in `PyCell::::borrow_mut` @@ -49,6 +55,9 @@ error[E0271]: type mismatch resolving `::Frozen == True` note: required by a bound in `pyo3::Py::::get` --> src/instance.rs | + | pub fn get(&self) -> &T + | --- required by a bound in this associated function + | where | T: PyClass + Sync, | ^^^^^^^^^^^^^ required by this bound in `Py::::get` @@ -61,5 +70,8 @@ error[E0271]: type mismatch resolving `::Frozen == True` note: required by a bound in `pyo3::PyCell::::get` --> src/pycell.rs | + | pub fn get(&self) -> &T + | --- required by a bound in this associated function + | where | T: PyClass + Sync, | ^^^^^^^^^^^^^ required by this bound in `PyCell::::get` diff --git a/tests/ui/not_send.stderr b/tests/ui/not_send.stderr index 18547a10f80..395723cba1f 100644 --- a/tests/ui/not_send.stderr +++ b/tests/ui/not_send.stderr @@ -7,11 +7,27 @@ error[E0277]: `*mut pyo3::Python<'static>` cannot be shared between threads safe | required by a bound introduced by this call | = help: within `pyo3::Python<'_>`, the trait `Sync` is not implemented for `*mut pyo3::Python<'static>` - = note: required because it appears within the type `PhantomData<*mut Python<'static>>` - = note: required because it appears within the type `NotSend` +note: required because it appears within the type `PhantomData<*mut Python<'static>>` + --> $RUST/core/src/marker.rs + | + | pub struct PhantomData; + | ^^^^^^^^^^^ +note: required because it appears within the type `NotSend` + --> src/impl_/not_send.rs + | + | pub(crate) struct NotSend(PhantomData<*mut Python<'static>>); + | ^^^^^^^ = note: required because it appears within the type `(&GILGuard, NotSend)` - = note: required because it appears within the type `PhantomData<(&GILGuard, NotSend)>` - = note: required because it appears within the type `Python<'_>` +note: required because it appears within the type `PhantomData<(&GILGuard, NotSend)>` + --> $RUST/core/src/marker.rs + | + | pub struct PhantomData; + | ^^^^^^^^^^^ +note: required because it appears within the type `Python<'_>` + --> src/marker.rs + | + | pub struct Python<'py>(PhantomData<(&'py GILGuard, NotSend)>); + | ^^^^^^ = note: required for `&pyo3::Python<'_>` to implement `Send` note: required because it's used within this closure --> tests/ui/not_send.rs:4:22 @@ -22,5 +38,8 @@ note: required because it's used within this closure note: required by a bound in `pyo3::Python::<'py>::allow_threads` --> src/marker.rs | + | pub fn allow_threads(self, f: F) -> T + | ------------- required by a bound in this associated function + | where | F: Ungil + FnOnce() -> T, | ^^^^^ required by this bound in `Python::<'py>::allow_threads` diff --git a/tests/ui/not_send2.stderr b/tests/ui/not_send2.stderr index 8bfa6016d86..2e6db009bad 100644 --- a/tests/ui/not_send2.stderr +++ b/tests/ui/not_send2.stderr @@ -10,8 +10,16 @@ error[E0277]: `UnsafeCell` cannot be shared between threads safely | |_________^ `UnsafeCell` cannot be shared between threads safely | = help: within `&PyString`, the trait `Sync` is not implemented for `UnsafeCell` - = note: required because it appears within the type `PyAny` - = note: required because it appears within the type `PyString` +note: required because it appears within the type `PyAny` + --> src/types/any.rs + | + | pub struct PyAny(UnsafeCell); + | ^^^^^ +note: required because it appears within the type `PyString` + --> src/types/string.rs + | + | pub struct PyString(PyAny); + | ^^^^^^^^ = note: required because it appears within the type `&PyString` = note: required for `&&PyString` to implement `Send` note: required because it's used within this closure @@ -23,5 +31,8 @@ note: required because it's used within this closure note: required by a bound in `pyo3::Python::<'py>::allow_threads` --> src/marker.rs | + | pub fn allow_threads(self, f: F) -> T + | ------------- required by a bound in this associated function + | where | F: Ungil + FnOnce() -> T, | ^^^^^ required by this bound in `Python::<'py>::allow_threads` From 4556886181fc55fe5c0c27700e390c9a761b75d6 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Tue, 11 Jul 2023 22:13:21 +0100 Subject: [PATCH 23/57] normalize exception in `PyErr::matches` and `PyErr::get_type` --- newsfragments/3313.fixed.md | 1 + src/err/err_state.rs | 16 +++++++++---- src/err/mod.rs | 47 ++++++++++++++++++------------------- 3 files changed, 35 insertions(+), 29 deletions(-) create mode 100644 newsfragments/3313.fixed.md diff --git a/newsfragments/3313.fixed.md b/newsfragments/3313.fixed.md new file mode 100644 index 00000000000..e5fc5c0e153 --- /dev/null +++ b/newsfragments/3313.fixed.md @@ -0,0 +1 @@ +Fix case where `PyErr::matches` and `PyErr::is_instance` returned results inconsistent with `PyErr::get_type`. diff --git a/src/err/err_state.rs b/src/err/err_state.rs index bf4fb3fdfb2..50a17fda474 100644 --- a/src/err/err_state.rs +++ b/src/err/err_state.rs @@ -68,11 +68,17 @@ impl PyErrState { ) } } - PyErrState::LazyValue { ptype, pvalue } => ( - ptype.into_ptr(), - pvalue(py).into_ptr(), - std::ptr::null_mut(), - ), + PyErrState::LazyValue { ptype, pvalue } => { + if unsafe { ffi::PyExceptionClass_Check(ptype.as_ptr()) } == 0 { + Self::exceptions_must_derive_from_base_exception(py).into_ffi_tuple(py) + } else { + ( + ptype.into_ptr(), + pvalue(py).into_ptr(), + std::ptr::null_mut(), + ) + } + } PyErrState::FfiTuple { ptype, pvalue, diff --git a/src/err/mod.rs b/src/err/mod.rs index 3af7b92bbc8..f90b9995c41 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -138,10 +138,6 @@ impl PyErr { where A: PyErrArguments + Send + Sync + 'static, { - if unsafe { ffi::PyExceptionClass_Check(ty.as_ptr()) } == 0 { - return exceptions_must_derive_from_base_exception(ty.py()); - } - PyErr::from_state(PyErrState::LazyValue { ptype: ty.into(), pvalue: boxed_args(args), @@ -438,16 +434,13 @@ impl PyErr { where T: ToPyObject, { - fn inner(err: &PyErr, py: Python<'_>, exc: PyObject) -> bool { - (unsafe { ffi::PyErr_GivenExceptionMatches(err.type_ptr(py), exc.as_ptr()) }) != 0 - } - inner(self, py, exc.to_object(py)) + self.is_instance(py, exc.to_object(py).as_ref(py)) } /// Returns true if the current exception is instance of `T`. #[inline] pub fn is_instance(&self, py: Python<'_>, ty: &PyAny) -> bool { - (unsafe { ffi::PyErr_GivenExceptionMatches(self.type_ptr(py), ty.as_ptr()) }) != 0 + (unsafe { ffi::PyErr_GivenExceptionMatches(self.get_type(py).as_ptr(), ty.as_ptr()) }) != 0 } /// Returns true if the current exception is instance of `T`. @@ -630,19 +623,6 @@ impl PyErr { } } - /// Returns borrowed reference to this Err's type - fn type_ptr(&self, py: Python<'_>) -> *mut ffi::PyObject { - match unsafe { &*self.state.get() } { - // In lazy type case, normalize before returning ptype in case the type is not a valid - // exception type. - Some(PyErrState::LazyTypeAndValue { .. }) => self.normalized(py).ptype.as_ptr(), - Some(PyErrState::LazyValue { ptype, .. }) => ptype.as_ptr(), - Some(PyErrState::FfiTuple { ptype, .. }) => ptype.as_ptr(), - Some(PyErrState::Normalized(n)) => n.ptype.as_ptr(), - None => panic!("Cannot access exception type while normalizing"), - } - } - #[inline] fn normalized(&self, py: Python<'_>) -> &PyErrStateNormalized { if let Some(PyErrState::Normalized(n)) = unsafe { @@ -822,8 +802,8 @@ fn exceptions_must_derive_from_base_exception(py: Python<'_>) -> PyErr { #[cfg(test)] mod tests { use super::PyErrState; - use crate::exceptions; - use crate::{PyErr, Python}; + use crate::exceptions::{self, PyTypeError, PyValueError}; + use crate::{PyErr, PyTypeInfo, Python}; #[test] fn no_error() { @@ -937,6 +917,25 @@ mod tests { is_sync::(); } + #[test] + fn test_pyerr_matches() { + Python::with_gil(|py| { + let err = PyErr::new::("foo"); + assert!(err.matches(py, PyValueError::type_object(py))); + + assert!(err.matches( + py, + (PyValueError::type_object(py), PyTypeError::type_object(py)) + )); + + assert!(!err.matches(py, PyTypeError::type_object(py))); + + // String is not a valid exception class, so we should get a TypeError + let err: PyErr = PyErr::from_type(crate::types::PyString::type_object(py), "foo"); + assert!(err.matches(py, PyTypeError::type_object(py))); + }) + } + #[test] fn test_pyerr_cause() { Python::with_gil(|py| { From 4a4327b9330e6223904932ac2490fefad3ccc65e Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Fri, 14 Jul 2023 13:11:24 +0100 Subject: [PATCH 24/57] remove concept of "normalize" from PyErr docs --- src/err/mod.rs | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/src/err/mod.rs b/src/err/mod.rs index f90b9995c41..4ebb157b5d3 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -18,12 +18,13 @@ use err_state::{boxed_args, PyErrState, PyErrStateNormalized}; /// Represents a Python exception. /// -/// Python exceptions can be raised in a "lazy" fashion, where the full Python object for the -/// exception is not created until needed. The process of creating the full object is known -/// as "normalization". An exception which has not yet been created is known as "unnormalized". +/// To avoid needing access to [`Python`] in `Into` conversions to create `PyErr` (thus improving +/// compatibility with `?` and other Rust errors) this type supports creating exceptions instances +/// in a lazy fashion, where the full Python object for the exception is created only when needed. /// -/// This struct builds upon that design, supporting all lazily-created Python exceptions and also -/// supporting exceptions lazily-created from Rust. +/// Accessing the contained exception in any way, such as with [`value`](PyErr::value), +/// [`get_type`](PyErr::get_type), or [`is_instance`](PyErr::is_instance) will create the full +/// exception object if it was not already created. pub struct PyErr { // Safety: can only hand out references when in the "normalized" state. Will never change // after normalization. @@ -69,12 +70,13 @@ impl PyErr { /// * any other value: the exception instance will be created using the equivalent to the Python /// expression `T(value)` /// - /// This error will be stored in an unnormalized state. This avoids the need for the Python GIL + /// This exception instance will be initialized lazily. This avoids the need for the Python GIL /// to be held, but requires `args` to be `Send` and `Sync`. If `args` is not `Send` or `Sync`, /// consider using [`PyErr::from_value`] instead. /// - /// If an error occurs during normalization (for example if `T` is not a Python type which - /// extends from `BaseException`), then a different error may be produced during normalization. + /// If `T` does not inherit from `BaseException`, then a `TypeError` will be returned. + /// + /// If calling T's constructor with `args` raises an exception, that exception will be returned. /// /// # Examples /// @@ -130,10 +132,9 @@ impl PyErr { /// /// `args` is either a tuple or a single value, with the same meaning as in [`PyErr::new`]. /// - /// If an error occurs during normalization (for example if `T` is not a Python type which - /// extends from `BaseException`), then a different error may be produced during normalization. + /// If `ty` does not inherit from `BaseException`, then a `TypeError` will be returned. /// - /// This error will be stored in an unnormalized state. + /// If calling `ty` with `args` raises an exception, that exception will be returned. pub fn from_type(ty: &PyType, args: A) -> PyErr where A: PyErrArguments + Send + Sync + 'static, @@ -146,8 +147,7 @@ impl PyErr { /// Creates a new PyErr. /// - /// If `obj` is a Python exception object, the PyErr will contain that object. The error will be - /// in a normalized state. + /// If `obj` is a Python exception object, the PyErr will contain that object. /// /// If `obj` is a Python exception type object, this is equivalent to `PyErr::from_type(obj, ())`. /// @@ -200,8 +200,6 @@ impl PyErr { /// Returns the type of this exception. /// - /// The object will be normalized first if needed. - /// /// # Examples /// ```rust /// use pyo3::{exceptions::PyTypeError, types::PyType, PyErr, Python}; @@ -217,8 +215,6 @@ impl PyErr { /// Returns the value of this exception. /// - /// The object will be normalized first if needed. - /// /// # Examples /// /// ```rust @@ -244,8 +240,6 @@ impl PyErr { /// Returns the traceback of this exception object. /// - /// The object will be normalized first if needed. - /// /// # Examples /// ```rust /// use pyo3::{exceptions::PyTypeError, Python}; From e6b8f36f606336fef564eb3990e72d34621c4ced Mon Sep 17 00:00:00 2001 From: CallMeMSL <36897759+CallMeMSL@users.noreply.github.com> Date: Fri, 14 Jul 2023 15:50:14 +0200 Subject: [PATCH 25/57] Update outdated link in python_typing_hints.md and fix typos --- guide/src/python_typing_hints.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/guide/src/python_typing_hints.md b/guide/src/python_typing_hints.md index e5ce1fe70d8..9d6c206f110 100644 --- a/guide/src/python_typing_hints.md +++ b/guide/src/python_typing_hints.md @@ -1,6 +1,6 @@ # Typing and IDE hints for you Python package -PyO3 provides an easy to use interface to code native Python libraries in Rust. The accompanying Maturin allows you to build and publish them as a package. Yet, for the better user experience, Python libraries should provide typing hints and documentation for all public entities, so that IDEs can show them during development and type analyzing tools such as `mypy` can use them to properly verify the code. +PyO3 provides an easy to use interface to code native Python libraries in Rust. The accompanying Maturin allows you to build and publish them as a package. Yet, for a better user experience, Python libraries should provide typing hints and documentation for all public entities, so that IDEs can show them during development and type analyzing tools such as `mypy` can use them to properly verify the code. Currently the best solution for the problem is to manually maintain `*.pyi` files and ship them along with the package. @@ -12,7 +12,7 @@ There is a sketch of a roadmap towards completing [the `experimental-inspect` fe > A stubs file only contains a description of the public interface of the module without any implementations. -There is also [extensive documentation on type stubs on the offical Python typing documentation](https://typing.readthedocs.io/en/latest/source/stubs.html). +There is also [extensive documentation on type stubs on the official Python typing documentation](https://typing.readthedocs.io/en/latest/source/stubs.html). Most Python developers probably already encountered them when trying to use their IDE's "Go to Definition" function on any builtin type. For example, the definitions of a few standard exceptions look like this: @@ -37,7 +37,7 @@ class StopIteration(Exception): value: Any ``` -As we can see, those are not full definitions containing implementation, but just a description of interface. It is usually all that the user of the library needs. +As we can see, those are not full definitions containing implementation, but just a description of the interface. It is usually all that the user of the library needs. ### What do the PEPs say? @@ -75,7 +75,7 @@ When source files are in the same package as stub files, they should be placed n #### If you do not have other Python files -If you do not need to add any other Python files apart from `pyi` to the package, Maturin provides a way to do most of the work for you. As documented in the [Maturin Guide](https://github.com/PyO3/maturin/blob/084cfaced651b28616aeea1f818bdc933a536bfe/guide/src/project_layout.md#adding-python-type-information), the only thing you need to do is to create a stub file for your module named `.pyi` in your project root and Maturin will do the rest. +If you do not need to add any other Python files apart from `pyi` to the package, Maturin provides a way to do most of the work for you. As documented in the [Maturin Guide](https://github.com/PyO3/maturin/#mixed-rustpython-projects), the only thing you need to do is to create a stub file for your module named `.pyi` in your project root and Maturin will do the rest. ```text my-rust-project/ @@ -108,7 +108,7 @@ my-project └── lib.rs ``` -Let's go a little bit more into details regarding the files inside the package folder. +Let's go a little bit more into detail regarding the files inside the package folder. ##### `__init__.py` content From b2a608bf53503581697876fb0a61a4a066ae612a Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Tue, 11 Jul 2023 21:48:46 +0100 Subject: [PATCH 26/57] optimize is_instance for PyBaseException --- src/err/mod.rs | 6 ++---- src/exceptions.rs | 8 +++++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/err/mod.rs b/src/err/mod.rs index 4ebb157b5d3..4d3331b07d8 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -177,12 +177,10 @@ impl PyErr { /// }); /// ``` pub fn from_value(obj: &PyAny) -> PyErr { - let ptr = obj.as_ptr(); - - let state = if unsafe { ffi::PyExceptionInstance_Check(ptr) } != 0 { + let state = if let Ok(obj) = obj.downcast::() { PyErrState::Normalized(PyErrStateNormalized { ptype: obj.get_type().into(), - pvalue: unsafe { Py::from_borrowed_ptr(obj.py(), obj.as_ptr()) }, + pvalue: obj.into(), ptraceback: None, }) } else if unsafe { ffi::PyExceptionClass_Check(obj.as_ptr()) } != 0 { diff --git a/src/exceptions.rs b/src/exceptions.rs index ff57b078a7a..b8b2145ad2c 100644 --- a/src/exceptions.rs +++ b/src/exceptions.rs @@ -260,13 +260,13 @@ macro_rules! create_exception_type_object { } macro_rules! impl_native_exception ( - ($name:ident, $exc_name:ident, $doc:expr, $layout:path) => ( + ($name:ident, $exc_name:ident, $doc:expr, $layout:path $(, #checkfunction=$checkfunction:path)?) => ( #[doc = $doc] #[allow(clippy::upper_case_acronyms)] pub struct $name($crate::PyAny); $crate::impl_exception_boilerplate!($name); - $crate::pyobject_native_type!($name, $layout, *($crate::ffi::$exc_name as *mut $crate::ffi::PyTypeObject)); + $crate::pyobject_native_type!($name, $layout, *($crate::ffi::$exc_name as *mut $crate::ffi::PyTypeObject) $(, #checkfunction=$checkfunction)?); ); ($name:ident, $exc_name:ident, $doc:expr) => ( impl_native_exception!($name, $exc_name, $doc, $crate::ffi::PyBaseExceptionObject); @@ -359,7 +359,9 @@ Python::with_gil(|py| { impl_native_exception!( PyBaseException, PyExc_BaseException, - native_doc!("BaseException") + native_doc!("BaseException"), + ffi::PyBaseExceptionObject, + #checkfunction=ffi::PyExceptionInstance_Check ); impl_native_exception!(PyException, PyExc_Exception, native_doc!("Exception")); impl_native_exception!( From 8870f5b0d084028e867e3454f5ac3b0e82916e43 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Fri, 14 Jul 2023 13:18:57 +0100 Subject: [PATCH 27/57] macros: change self_arg to be an expression --- pyo3-macros-backend/src/method.rs | 78 ++++++++----------- pyo3-macros-backend/src/pymethod.rs | 114 +++++++++++++--------------- 2 files changed, 84 insertions(+), 108 deletions(-) diff --git a/pyo3-macros-backend/src/method.rs b/pyo3-macros-backend/src/method.rs index 7ea83b19f6b..f8109ac8c8f 100644 --- a/pyo3-macros-backend/src/method.rs +++ b/pyo3-macros-backend/src/method.rs @@ -9,7 +9,7 @@ use quote::ToTokens; use quote::{quote, quote_spanned}; use syn::ext::IdentExt; use syn::spanned::Spanned; -use syn::{Result, Token}; +use syn::Result; #[derive(Clone, Debug)] pub struct FnArg<'a> { @@ -101,38 +101,31 @@ pub enum FnType { } impl FnType { - pub fn self_conversion( - &self, - cls: Option<&syn::Type>, - error_mode: ExtractErrorMode, - ) -> TokenStream { + pub fn self_arg(&self, cls: Option<&syn::Type>, error_mode: ExtractErrorMode) -> TokenStream { match self { - FnType::Getter(st) | FnType::Setter(st) | FnType::Fn(st) => st.receiver( - cls.expect("no class given for Fn with a \"self\" receiver"), - error_mode, - ), + FnType::Getter(st) | FnType::Setter(st) | FnType::Fn(st) => { + let mut receiver = st.receiver( + cls.expect("no class given for Fn with a \"self\" receiver"), + error_mode, + ); + syn::Token![,](Span::call_site()).to_tokens(&mut receiver); + receiver + } FnType::FnNew | FnType::FnStatic | FnType::ClassAttribute => { quote!() } FnType::FnClass | FnType::FnNewClass => { quote! { - let _slf = _pyo3::types::PyType::from_type_ptr(_py, _slf as *mut _pyo3::ffi::PyTypeObject); + _pyo3::types::PyType::from_type_ptr(_py, _slf as *mut _pyo3::ffi::PyTypeObject), } } FnType::FnModule => { quote! { - let _slf = _py.from_borrowed_ptr::<_pyo3::types::PyModule>(_slf); + _py.from_borrowed_ptr::<_pyo3::types::PyModule>(_slf), } } } } - - pub fn self_arg(&self) -> TokenStream { - match self { - FnType::FnNew | FnType::FnStatic | FnType::ClassAttribute => quote!(), - _ => quote!(_slf,), - } - } } #[derive(Clone, Debug)] @@ -167,40 +160,34 @@ impl SelfType { let _slf = syn::Ident::new("_slf", Span::call_site()); match self { SelfType::Receiver { span, mutable } => { - let (method, mutability) = if *mutable { - ( - quote_spanned! { *span => extract_pyclass_ref_mut }, - Some(Token![mut](*span)), - ) + let method = if *mutable { + syn::Ident::new("extract_pyclass_ref_mut", *span) } else { - (quote_spanned! { *span => extract_pyclass_ref }, None) + syn::Ident::new("extract_pyclass_ref", *span) }; - let extract = error_mode.handle_error( + error_mode.handle_error( &py, quote_spanned! { *span => - _pyo3::impl_::extract_argument::#method( + _pyo3::impl_::extract_argument::#method::<#cls>( #py.from_borrowed_ptr::<_pyo3::PyAny>(#_slf), - &mut holder, + &mut { _pyo3::impl_::extract_argument::FunctionArgumentHolder::INIT }, ) }, - ); - quote_spanned! { *span => - let mut holder = _pyo3::impl_::extract_argument::FunctionArgumentHolder::INIT; - let #_slf: &#mutability #cls = #extract; - } + ) } SelfType::TryFromPyCell(span) => { - let cell = error_mode.handle_error( + error_mode.handle_error( &py, - quote!{ - _py.from_borrowed_ptr::<_pyo3::PyAny>(_slf).downcast::<_pyo3::PyCell<#cls>>() + quote_spanned! { *span => + #py.from_borrowed_ptr::<_pyo3::PyAny>(#_slf).downcast::<_pyo3::PyCell<#cls>>() + .map_err(::std::convert::Into::<_pyo3::PyErr>::into) + .and_then( + #[allow(clippy::useless_conversion)] // In case _slf is PyCell + |cell| ::std::convert::TryFrom::try_from(cell).map_err(::std::convert::Into::into) + ) + } - ); - quote_spanned! { *span => - let _cell = #cell; - #[allow(clippy::useless_conversion)] // In case _slf is PyCell - let #_slf = ::std::convert::TryFrom::try_from(_cell)?; - } + ) } } } @@ -433,8 +420,7 @@ impl<'a> FnSpec<'a> { cls: Option<&syn::Type>, ) -> Result { let deprecations = &self.deprecations; - let self_conversion = self.tp.self_conversion(cls, ExtractErrorMode::Raise); - let self_arg = self.tp.self_arg(); + let self_arg = self.tp.self_arg(cls, ExtractErrorMode::Raise); let py = syn::Ident::new("_py", Span::call_site()); let func_name = &self.name; @@ -460,6 +446,7 @@ impl<'a> FnSpec<'a> { } else { rust_call(vec![]) }; + quote! { unsafe fn #ident<'py>( #py: _pyo3::Python<'py>, @@ -467,7 +454,6 @@ impl<'a> FnSpec<'a> { ) -> _pyo3::PyResult<*mut _pyo3::ffi::PyObject> { let function = #rust_name; // Shadow the function name to avoid #3017 #deprecations - #self_conversion #call } } @@ -485,7 +471,6 @@ impl<'a> FnSpec<'a> { ) -> _pyo3::PyResult<*mut _pyo3::ffi::PyObject> { let function = #rust_name; // Shadow the function name to avoid #3017 #deprecations - #self_conversion #arg_convert #call } @@ -503,7 +488,6 @@ impl<'a> FnSpec<'a> { ) -> _pyo3::PyResult<*mut _pyo3::ffi::PyObject> { let function = #rust_name; // Shadow the function name to avoid #3017 #deprecations - #self_conversion #arg_convert #call } diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index 4297eee9c41..9b7a8a5aa19 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -471,8 +471,13 @@ fn impl_py_class_attribute(cls: &syn::Type, spec: &FnSpec<'_>) -> syn::Result) -> syn::Result { +fn impl_call_setter( + cls: &syn::Type, + spec: &FnSpec<'_>, + self_type: &SelfType, +) -> syn::Result { let (py_arg, args) = split_off_python_arg(&spec.signature.arguments); + let slf = self_type.receiver(cls, ExtractErrorMode::Raise); if args.is_empty() { bail_spanned!(spec.name.span() => "setter function expected to have one argument"); @@ -485,9 +490,9 @@ fn impl_call_setter(cls: &syn::Type, spec: &FnSpec<'_>) -> syn::Result { - // named struct field - quote!({ _slf.#ident = _val; }) - } - PropertyType::Descriptor { field_index, .. } => { - // tuple struct field - let index = syn::Index::from(field_index); - quote!({ _slf.#index = _val; }) - } - PropertyType::Function { spec, .. } => impl_call_setter(cls, spec)?, - }; - - let slf = match property_type { - PropertyType::Descriptor { .. } => SelfType::Receiver { - mutable: true, - span: Span::call_site(), - } - .receiver(cls, ExtractErrorMode::Raise), - PropertyType::Function { self_type, .. } => { - self_type.receiver(cls, ExtractErrorMode::Raise) + let slf = SelfType::Receiver { + mutable: true, + span: Span::call_site(), + } + .receiver(cls, ExtractErrorMode::Raise); + if let Some(ident) = &field.ident { + // named struct field + quote!({ #slf.#ident = _val; }) + } else { + // tuple struct field + let index = syn::Index::from(field_index); + quote!({ #slf.#index = _val; }) + } } + PropertyType::Function { + spec, self_type, .. + } => impl_call_setter(cls, spec, self_type)?, }; let wrapper_ident = match property_type { @@ -561,7 +560,6 @@ pub fn impl_py_setter_def( _slf: *mut _pyo3::ffi::PyObject, _value: *mut _pyo3::ffi::PyObject, ) -> _pyo3::PyResult<::std::os::raw::c_int> { - #slf let _value = _py .from_borrowed_ptr_or_opt(_value) .ok_or_else(|| { @@ -591,8 +589,13 @@ pub fn impl_py_setter_def( }) } -fn impl_call_getter(cls: &syn::Type, spec: &FnSpec<'_>) -> syn::Result { +fn impl_call_getter( + cls: &syn::Type, + spec: &FnSpec<'_>, + self_type: &SelfType, +) -> syn::Result { let (py_arg, args) = split_off_python_arg(&spec.signature.arguments); + let slf = self_type.receiver(cls, ExtractErrorMode::Raise); ensure_spanned!( args.is_empty(), args[0].ty.span() => "getter function can only have one argument (of type pyo3::Python)" @@ -600,9 +603,9 @@ fn impl_call_getter(cls: &syn::Type, spec: &FnSpec<'_>) -> syn::Result { - // named struct field - quote!(::std::clone::Clone::clone(&(_slf.#ident))) - } - PropertyType::Descriptor { field_index, .. } => { - // tuple struct field - let index = syn::Index::from(field_index); - quote!(::std::clone::Clone::clone(&(_slf.#index))) - } - PropertyType::Function { spec, .. } => impl_call_getter(cls, spec)?, - }; - - let slf = match property_type { - PropertyType::Descriptor { .. } => SelfType::Receiver { - mutable: false, - span: Span::call_site(), - } - .receiver(cls, ExtractErrorMode::Raise), - PropertyType::Function { self_type, .. } => { - self_type.receiver(cls, ExtractErrorMode::Raise) + let slf = SelfType::Receiver { + mutable: false, + span: Span::call_site(), + } + .receiver(cls, ExtractErrorMode::Raise); + if let Some(ident) = &field.ident { + // named struct field + quote!(::std::clone::Clone::clone(&(#slf.#ident))) + } else { + // tuple struct field + let index = syn::Index::from(field_index); + quote!(::std::clone::Clone::clone(&(#slf.#index))) + } } + PropertyType::Function { + spec, self_type, .. + } => impl_call_getter(cls, spec, self_type)?, }; let conversion = match property_type { @@ -690,7 +688,6 @@ pub fn impl_py_getter_def( _py: _pyo3::Python<'_>, _slf: *mut _pyo3::ffi::PyObject ) -> _pyo3::PyResult<*mut _pyo3::ffi::PyObject> { - #slf let item = #getter_impl; #conversion } @@ -1150,19 +1147,14 @@ fn generate_method_body( extract_error_mode: ExtractErrorMode, return_mode: Option<&ReturnMode>, ) -> Result { - let self_conversion = spec.tp.self_conversion(Some(cls), extract_error_mode); - let self_arg = spec.tp.self_arg(); + let self_arg = spec.tp.self_arg(Some(cls), extract_error_mode); let rust_name = spec.name; let args = extract_proto_arguments(py, spec, arguments, extract_error_mode)?; let call = quote! { _pyo3::callback::convert(#py, #cls::#rust_name(#self_arg #(#args),*)) }; - let body = if let Some(return_mode) = return_mode { + Ok(if let Some(return_mode) = return_mode { return_mode.return_call_output(py, call) } else { call - }; - Ok(quote! { - #self_conversion - #body }) } From 9d2da740bd99bdf118024444de1f89e9757a3621 Mon Sep 17 00:00:00 2001 From: ringsaturn Date: Sun, 16 Jul 2023 22:26:33 +0800 Subject: [PATCH 28/57] Add tzfpy --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index cc45f93e29c..2138bdc3d54 100644 --- a/README.md +++ b/README.md @@ -212,6 +212,7 @@ about this topic. - [rust-python-coverage](https://github.com/cjermain/rust-python-coverage) _Example PyO3 project with automated test coverage for Rust and Python._ - [tiktoken](https://github.com/openai/tiktoken) _A fast BPE tokeniser for use with OpenAI's models._ - [tokenizers](https://github.com/huggingface/tokenizers/tree/main/bindings/python) _Python bindings to the Hugging Face tokenizers (NLP) written in Rust._ +- [tzfpy](http://github.com/ringsaturn/tzfpy) _A fast package to convert longitude/latitude to timezone name._ - [wasmer-python](https://github.com/wasmerio/wasmer-python) _Python library to run WebAssembly binaries._ ## Articles and other media From f96c677457c23244c5a4a50c357712ccd6d71f73 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Fri, 14 Jul 2023 13:42:35 +0100 Subject: [PATCH 29/57] clippy: deny / fix used-underscope-binding lint --- .cargo/config | 1 + pyo3-ffi/src/cpython/abstract_.rs | 4 ++-- pyo3-ffi/src/cpython/unicodeobject.rs | 30 +++++++++++++-------------- pyo3-macros-backend/src/method.rs | 8 +++---- src/pyclass/gc.rs | 8 +++++-- tests/test_class_new.rs | 18 ++++++++-------- tests/test_pep_587.rs | 5 ++++- tests/test_proto_methods.rs | 10 ++++----- 8 files changed, 46 insertions(+), 38 deletions(-) diff --git a/.cargo/config b/.cargo/config index e87eb0f23b1..e5cba4fd74b 100644 --- a/.cargo/config +++ b/.cargo/config @@ -17,6 +17,7 @@ rustflags = [ "-Dclippy::todo", "-Dclippy::unnecessary_wraps", "-Dclippy::useless_transmute", + "-Dclippy::used_underscore_binding", "-Delided_lifetimes_in_paths", "-Dunused_lifetimes", "-Drust_2021_prelude_collisions" diff --git a/pyo3-ffi/src/cpython/abstract_.rs b/pyo3-ffi/src/cpython/abstract_.rs index 7d97abc4f6f..dd029f3324a 100644 --- a/pyo3-ffi/src/cpython/abstract_.rs +++ b/pyo3-ffi/src/cpython/abstract_.rs @@ -177,8 +177,8 @@ extern "C" { #[inline(always)] pub unsafe fn PyObject_CallOneArg(func: *mut PyObject, arg: *mut PyObject) -> *mut PyObject { assert!(!arg.is_null()); - let _args = [std::ptr::null_mut(), arg]; - let args = _args.as_ptr().offset(1); // For PY_VECTORCALL_ARGUMENTS_OFFSET + let args_array = [std::ptr::null_mut(), arg]; + let args = args_array.as_ptr().offset(1); // For PY_VECTORCALL_ARGUMENTS_OFFSET let tstate = PyThreadState_GET(); let nargsf = 1 | PY_VECTORCALL_ARGUMENTS_OFFSET; _PyObject_VectorcallTstate(tstate, func, args, nargsf as size_t, std::ptr::null_mut()) diff --git a/pyo3-ffi/src/cpython/unicodeobject.rs b/pyo3-ffi/src/cpython/unicodeobject.rs index 1bd93655c6a..f3f7207ca5e 100644 --- a/pyo3-ffi/src/cpython/unicodeobject.rs +++ b/pyo3-ffi/src/cpython/unicodeobject.rs @@ -148,8 +148,8 @@ const STATE_READY_WIDTH: u8 = 1; #[repr(C)] #[repr(align(4))] struct PyASCIIObjectState { - _bitfield_align: [u8; 0], - _bitfield: BitfieldUnit<[u8; 4usize]>, + bitfield_align: [u8; 0], + bitfield: BitfieldUnit<[u8; 4usize]>, } // c_uint and u32 are not necessarily the same type on all targets / architectures @@ -158,7 +158,7 @@ impl PyASCIIObjectState { #[inline] unsafe fn interned(&self) -> c_uint { std::mem::transmute( - self._bitfield + self.bitfield .get(STATE_INTERNED_INDEX, STATE_INTERNED_WIDTH) as u32, ) } @@ -166,57 +166,57 @@ impl PyASCIIObjectState { #[inline] unsafe fn set_interned(&mut self, val: c_uint) { let val: u32 = std::mem::transmute(val); - self._bitfield + self.bitfield .set(STATE_INTERNED_INDEX, STATE_INTERNED_WIDTH, val as u64) } #[inline] unsafe fn kind(&self) -> c_uint { - std::mem::transmute(self._bitfield.get(STATE_KIND_INDEX, STATE_KIND_WIDTH) as u32) + std::mem::transmute(self.bitfield.get(STATE_KIND_INDEX, STATE_KIND_WIDTH) as u32) } #[inline] unsafe fn set_kind(&mut self, val: c_uint) { let val: u32 = std::mem::transmute(val); - self._bitfield + self.bitfield .set(STATE_KIND_INDEX, STATE_KIND_WIDTH, val as u64) } #[inline] unsafe fn compact(&self) -> c_uint { - std::mem::transmute(self._bitfield.get(STATE_COMPACT_INDEX, STATE_COMPACT_WIDTH) as u32) + std::mem::transmute(self.bitfield.get(STATE_COMPACT_INDEX, STATE_COMPACT_WIDTH) as u32) } #[inline] unsafe fn set_compact(&mut self, val: c_uint) { let val: u32 = std::mem::transmute(val); - self._bitfield + self.bitfield .set(STATE_COMPACT_INDEX, STATE_COMPACT_WIDTH, val as u64) } #[inline] unsafe fn ascii(&self) -> c_uint { - std::mem::transmute(self._bitfield.get(STATE_ASCII_INDEX, STATE_ASCII_WIDTH) as u32) + std::mem::transmute(self.bitfield.get(STATE_ASCII_INDEX, STATE_ASCII_WIDTH) as u32) } #[inline] unsafe fn set_ascii(&mut self, val: c_uint) { let val: u32 = std::mem::transmute(val); - self._bitfield + self.bitfield .set(STATE_ASCII_INDEX, STATE_ASCII_WIDTH, val as u64) } #[cfg(not(Py_3_12))] #[inline] unsafe fn ready(&self) -> c_uint { - std::mem::transmute(self._bitfield.get(STATE_READY_INDEX, STATE_READY_WIDTH) as u32) + std::mem::transmute(self.bitfield.get(STATE_READY_INDEX, STATE_READY_WIDTH) as u32) } #[cfg(not(Py_3_12))] #[inline] unsafe fn set_ready(&mut self, val: c_uint) { let val: u32 = std::mem::transmute(val); - self._bitfield + self.bitfield .set(STATE_READY_INDEX, STATE_READY_WIDTH, val as u64) } } @@ -225,8 +225,8 @@ impl From for PyASCIIObjectState { #[inline] fn from(value: u32) -> Self { PyASCIIObjectState { - _bitfield_align: [], - _bitfield: BitfieldUnit::new(value.to_ne_bytes()), + bitfield_align: [], + bitfield: BitfieldUnit::new(value.to_ne_bytes()), } } } @@ -234,7 +234,7 @@ impl From for PyASCIIObjectState { impl From for u32 { #[inline] fn from(value: PyASCIIObjectState) -> Self { - u32::from_ne_bytes(value._bitfield.storage) + u32::from_ne_bytes(value.bitfield.storage) } } diff --git a/pyo3-macros-backend/src/method.rs b/pyo3-macros-backend/src/method.rs index f8109ac8c8f..9bd33cb5a8e 100644 --- a/pyo3-macros-backend/src/method.rs +++ b/pyo3-macros-backend/src/method.rs @@ -157,7 +157,7 @@ impl ExtractErrorMode { impl SelfType { pub fn receiver(&self, cls: &syn::Type, error_mode: ExtractErrorMode) -> TokenStream { let py = syn::Ident::new("_py", Span::call_site()); - let _slf = syn::Ident::new("_slf", Span::call_site()); + let slf = syn::Ident::new("_slf", Span::call_site()); match self { SelfType::Receiver { span, mutable } => { let method = if *mutable { @@ -169,7 +169,7 @@ impl SelfType { &py, quote_spanned! { *span => _pyo3::impl_::extract_argument::#method::<#cls>( - #py.from_borrowed_ptr::<_pyo3::PyAny>(#_slf), + #py.from_borrowed_ptr::<_pyo3::PyAny>(#slf), &mut { _pyo3::impl_::extract_argument::FunctionArgumentHolder::INIT }, ) }, @@ -179,10 +179,10 @@ impl SelfType { error_mode.handle_error( &py, quote_spanned! { *span => - #py.from_borrowed_ptr::<_pyo3::PyAny>(#_slf).downcast::<_pyo3::PyCell<#cls>>() + #py.from_borrowed_ptr::<_pyo3::PyAny>(#slf).downcast::<_pyo3::PyCell<#cls>>() .map_err(::std::convert::Into::<_pyo3::PyErr>::into) .and_then( - #[allow(clippy::useless_conversion)] // In case _slf is PyCell + #[allow(clippy::useless_conversion)] // In case slf is PyCell |cell| ::std::convert::TryFrom::try_from(cell).map_err(::std::convert::Into::into) ) diff --git a/src/pyclass/gc.rs b/src/pyclass/gc.rs index 900027f7bfb..7878ccf5ca8 100644 --- a/src/pyclass/gc.rs +++ b/src/pyclass/gc.rs @@ -38,7 +38,11 @@ impl<'p> PyVisit<'p> { /// Creates the PyVisit from the arguments to tp_traverse #[doc(hidden)] - pub unsafe fn from_raw(visit: ffi::visitproc, arg: *mut c_void, _py: Python<'p>) -> Self { - Self { visit, arg, _py } + pub unsafe fn from_raw(visit: ffi::visitproc, arg: *mut c_void, py: Python<'p>) -> Self { + Self { + visit, + arg, + _py: py, + } } } diff --git a/tests/test_class_new.rs b/tests/test_class_new.rs index ff159c610f8..77c571eac3f 100644 --- a/tests/test_class_new.rs +++ b/tests/test_class_new.rs @@ -82,14 +82,14 @@ fn tuple_class_with_new() { #[pyclass] #[derive(Debug)] struct NewWithOneArg { - _data: i32, + data: i32, } #[pymethods] impl NewWithOneArg { #[new] fn new(arg: i32) -> NewWithOneArg { - NewWithOneArg { _data: arg } + NewWithOneArg { data: arg } } } @@ -100,14 +100,14 @@ fn new_with_one_arg() { let wrp = typeobj.call((42,), None).unwrap(); let obj = wrp.downcast::>().unwrap(); let obj_ref = obj.borrow(); - assert_eq!(obj_ref._data, 42); + assert_eq!(obj_ref.data, 42); }); } #[pyclass] struct NewWithTwoArgs { - _data1: i32, - _data2: i32, + data1: i32, + data2: i32, } #[pymethods] @@ -115,8 +115,8 @@ impl NewWithTwoArgs { #[new] fn new(arg1: i32, arg2: i32) -> Self { NewWithTwoArgs { - _data1: arg1, - _data2: arg2, + data1: arg1, + data2: arg2, } } } @@ -131,8 +131,8 @@ fn new_with_two_args() { .unwrap(); let obj = wrp.downcast::>().unwrap(); let obj_ref = obj.borrow(); - assert_eq!(obj_ref._data1, 10); - assert_eq!(obj_ref._data2, 20); + assert_eq!(obj_ref.data1, 10); + assert_eq!(obj_ref.data2, 20); }); } diff --git a/tests/test_pep_587.rs b/tests/test_pep_587.rs index ca5c3f65ea2..24e1f07d2d8 100644 --- a/tests/test_pep_587.rs +++ b/tests/test_pep_587.rs @@ -29,7 +29,10 @@ fn test_default_interpreter() { unsafe { ffi::PyConfig_InitPythonConfig(&mut config) }; // Require manually calling _Py_InitializeMain to exercise more ffi code - config._init_main = 0; + #[allow(clippy::used_underscore_binding)] + { + config._init_main = 0; + } #[cfg(Py_3_10)] unsafe { diff --git a/tests/test_proto_methods.rs b/tests/test_proto_methods.rs index e04ce45a6ad..eba5a0cec7b 100644 --- a/tests/test_proto_methods.rs +++ b/tests/test_proto_methods.rs @@ -14,14 +14,14 @@ struct EmptyClass; struct ExampleClass { #[pyo3(get, set)] value: i32, - _custom_attr: Option, + custom_attr: Option, } #[pymethods] impl ExampleClass { fn __getattr__(&self, py: Python<'_>, attr: &str) -> PyResult { if attr == "special_custom_attr" { - Ok(self._custom_attr.into_py(py)) + Ok(self.custom_attr.into_py(py)) } else { Err(PyAttributeError::new_err(attr.to_string())) } @@ -29,7 +29,7 @@ impl ExampleClass { fn __setattr__(&mut self, attr: &str, value: &PyAny) -> PyResult<()> { if attr == "special_custom_attr" { - self._custom_attr = Some(value.extract()?); + self.custom_attr = Some(value.extract()?); Ok(()) } else { Err(PyAttributeError::new_err(attr.to_string())) @@ -38,7 +38,7 @@ impl ExampleClass { fn __delattr__(&mut self, attr: &str) -> PyResult<()> { if attr == "special_custom_attr" { - self._custom_attr = None; + self.custom_attr = None; Ok(()) } else { Err(PyAttributeError::new_err(attr.to_string())) @@ -68,7 +68,7 @@ fn make_example(py: Python<'_>) -> &PyCell { py, ExampleClass { value: 5, - _custom_attr: Some(20), + custom_attr: Some(20), }, ) .unwrap() From a6575b3e563cca7eb73ec25e181d579af7c8ddae Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Sun, 16 Jul 2023 20:32:46 +0100 Subject: [PATCH 30/57] merge PyErr internal states for simplicity --- src/err/err_state.rs | 63 ++++++++++++++++++-------------------------- src/err/mod.rs | 38 ++++++++++---------------- 2 files changed, 40 insertions(+), 61 deletions(-) diff --git a/src/err/err_state.rs b/src/err/err_state.rs index 50a17fda474..abf1817cd57 100644 --- a/src/err/err_state.rs +++ b/src/err/err_state.rs @@ -2,7 +2,7 @@ use crate::{ exceptions::{PyBaseException, PyTypeError}, ffi, types::{PyTraceback, PyType}, - AsPyPointer, IntoPy, IntoPyPointer, Py, PyObject, Python, + AsPyPointer, IntoPy, IntoPyPointer, Py, PyAny, PyObject, PyTypeInfo, Python, }; #[derive(Clone)] @@ -12,15 +12,16 @@ pub(crate) struct PyErrStateNormalized { pub ptraceback: Option>, } +pub(crate) struct PyErrStateLazyFnOutput { + pub(crate) ptype: PyObject, + pub(crate) pvalue: PyObject, +} + +pub(crate) type PyErrStateLazyFn = + dyn for<'py> FnOnce(Python<'py>) -> PyErrStateLazyFnOutput + Send + Sync; + pub(crate) enum PyErrState { - LazyTypeAndValue { - ptype: for<'py> fn(Python<'py>) -> &PyType, - pvalue: Box FnOnce(Python<'py>) -> PyObject + Send + Sync>, - }, - LazyValue { - ptype: Py, - pvalue: Box FnOnce(Python<'py>) -> PyObject + Send + Sync>, - }, + Lazy(Box), FfiTuple { ptype: PyObject, pvalue: Option, @@ -44,10 +45,14 @@ where } } -pub(crate) fn boxed_args( - args: impl PyErrArguments + 'static, -) -> Box FnOnce(Python<'py>) -> PyObject + Send + Sync> { - Box::new(|py| args.arguments(py)) +impl PyErrState { + pub(crate) fn lazy(ptype: &PyAny, args: impl PyErrArguments + 'static) -> Self { + let ptype = ptype.into(); + PyErrState::Lazy(Box::new(move |py| PyErrStateLazyFnOutput { + ptype, + pvalue: args.arguments(py), + })) + } } impl PyErrState { @@ -56,27 +61,12 @@ impl PyErrState { py: Python<'_>, ) -> (*mut ffi::PyObject, *mut ffi::PyObject, *mut ffi::PyObject) { match self { - PyErrState::LazyTypeAndValue { ptype, pvalue } => { - let ty = ptype(py); - if unsafe { ffi::PyExceptionClass_Check(ty.as_ptr()) } == 0 { - Self::exceptions_must_derive_from_base_exception(py).into_ffi_tuple(py) - } else { - ( - ptype(py).into_ptr(), - pvalue(py).into_ptr(), - std::ptr::null_mut(), - ) - } - } - PyErrState::LazyValue { ptype, pvalue } => { + PyErrState::Lazy(lazy) => { + let PyErrStateLazyFnOutput { ptype, pvalue } = lazy(py); if unsafe { ffi::PyExceptionClass_Check(ptype.as_ptr()) } == 0 { Self::exceptions_must_derive_from_base_exception(py).into_ffi_tuple(py) } else { - ( - ptype.into_ptr(), - pvalue(py).into_ptr(), - std::ptr::null_mut(), - ) + (ptype.into_ptr(), pvalue.into_ptr(), std::ptr::null_mut()) } } PyErrState::FfiTuple { @@ -92,11 +82,10 @@ impl PyErrState { } } - #[inline] - pub(crate) fn exceptions_must_derive_from_base_exception(py: Python<'_>) -> Self { - PyErrState::LazyValue { - ptype: py.get_type::().into(), - pvalue: boxed_args("exceptions must derive from BaseException"), - } + fn exceptions_must_derive_from_base_exception(py: Python<'_>) -> Self { + PyErrState::lazy( + PyTypeError::type_object(py), + "exceptions must derive from BaseException", + ) } } diff --git a/src/err/mod.rs b/src/err/mod.rs index 4d3331b07d8..cdfb64b1249 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -14,7 +14,7 @@ mod err_state; mod impls; pub use err_state::PyErrArguments; -use err_state::{boxed_args, PyErrState, PyErrStateNormalized}; +use err_state::{PyErrState, PyErrStateLazyFnOutput, PyErrStateNormalized}; /// Represents a Python exception. /// @@ -119,10 +119,12 @@ impl PyErr { T: PyTypeInfo, A: PyErrArguments + Send + Sync + 'static, { - PyErr::from_state(PyErrState::LazyTypeAndValue { - ptype: T::type_object, - pvalue: boxed_args(args), - }) + PyErr::from_state(PyErrState::Lazy(Box::new(move |py| { + PyErrStateLazyFnOutput { + ptype: T::type_object(py).into(), + pvalue: args.arguments(py), + } + }))) } /// Constructs a new PyErr from the given Python type and arguments. @@ -139,10 +141,7 @@ impl PyErr { where A: PyErrArguments + Send + Sync + 'static, { - PyErr::from_state(PyErrState::LazyValue { - ptype: ty.into(), - pvalue: boxed_args(args), - }) + PyErr::from_state(PyErrState::lazy(ty, args)) } /// Creates a new PyErr. @@ -183,14 +182,10 @@ impl PyErr { pvalue: obj.into(), ptraceback: None, }) - } else if unsafe { ffi::PyExceptionClass_Check(obj.as_ptr()) } != 0 { - PyErrState::FfiTuple { - ptype: obj.into(), - pvalue: None, - ptraceback: None, - } } else { - return exceptions_must_derive_from_base_exception(obj.py()); + // Assume obj is Type[Exception]; let later normalization handle if this + // is not the case + PyErrState::lazy(obj, obj.py().None()) }; PyErr::from_state(state) @@ -277,9 +272,9 @@ impl PyErr { ffi::PyErr_Fetch(&mut ptype, &mut pvalue, &mut ptraceback); // Convert to Py immediately so that any references are freed by early return. - let ptype = Py::from_owned_ptr_or_opt(py, ptype); - let pvalue = Py::from_owned_ptr_or_opt(py, pvalue); - let ptraceback = Py::from_owned_ptr_or_opt(py, ptraceback); + let ptype = PyObject::from_owned_ptr_or_opt(py, ptype); + let pvalue = PyObject::from_owned_ptr_or_opt(py, pvalue); + let ptraceback = PyObject::from_owned_ptr_or_opt(py, ptraceback); // A valid exception state should always have a non-null ptype, but the other two may be // null. @@ -786,11 +781,6 @@ impl_signed_integer!(i64); impl_signed_integer!(i128); impl_signed_integer!(isize); -#[inline] -fn exceptions_must_derive_from_base_exception(py: Python<'_>) -> PyErr { - PyErr::from_state(PyErrState::exceptions_must_derive_from_base_exception(py)) -} - #[cfg(test)] mod tests { use super::PyErrState; From 269454d2e87ec039fee021f2150b89ce673cce6e Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Tue, 11 Jul 2023 08:36:44 +0100 Subject: [PATCH 31/57] Preserve panic message after exception is normalized --- newsfragments/3326.fixed.md | 1 + src/err/mod.rs | 24 ++++++++++++++++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 newsfragments/3326.fixed.md diff --git a/newsfragments/3326.fixed.md b/newsfragments/3326.fixed.md new file mode 100644 index 00000000000..8353c598f72 --- /dev/null +++ b/newsfragments/3326.fixed.md @@ -0,0 +1 @@ +Fix loss of panic message in `PanicException` when unwinding after the exception was "normalized". diff --git a/src/err/mod.rs b/src/err/mod.rs index cdfb64b1249..8516cc9b1db 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -297,9 +297,10 @@ impl PyErr { }; if ptype.as_ptr() == PanicException::type_object_raw(py).cast() { - let msg: String = pvalue + let msg = pvalue .as_ref() - .and_then(|obj| obj.extract(py).ok()) + .and_then(|obj| obj.as_ref(py).str().ok()) + .map(|py_str| py_str.to_string_lossy().into()) .unwrap_or_else(|| String::from("Unwrapped panic from Python code")); eprintln!( @@ -845,6 +846,25 @@ mod tests { }); } + #[test] + #[should_panic(expected = "new panic")] + #[cfg(not(Py_3_12))] + fn fetching_normalized_panic_exception_resumes_unwind() { + use crate::panic::PanicException; + + Python::with_gil(|py| { + let err: PyErr = PanicException::new_err("new panic"); + // Restoring an error doesn't normalize it before Python 3.12, + // so we have to explicitly test this case. + let _ = err.normalized(py); + err.restore(py); + assert!(PyErr::occurred(py)); + + // should resume unwind + let _ = PyErr::fetch(py); + }); + } + #[test] fn err_debug() { // Debug representation should be like the following (without the newlines): From d94c09b2f3b49948666f400b974bcc769e85d436 Mon Sep 17 00:00:00 2001 From: Zak Stucke Date: Wed, 19 Jul 2023 11:58:24 +0300 Subject: [PATCH 32/57] Prevent traceback loss on conversion to and from PyErr --- newsfragments/3328.fixed.md | 2 ++ src/err/mod.rs | 19 ++++++++++++--- src/types/traceback.rs | 47 ++++++++++++++++++++++++++++++++++++- 3 files changed, 64 insertions(+), 4 deletions(-) create mode 100644 newsfragments/3328.fixed.md diff --git a/newsfragments/3328.fixed.md b/newsfragments/3328.fixed.md new file mode 100644 index 00000000000..18c19212957 --- /dev/null +++ b/newsfragments/3328.fixed.md @@ -0,0 +1,2 @@ +Fixed `PyErr.from_value()` to maintain original traceback. +Fixed `PyErr.into_value()` to maintain original traceback. diff --git a/src/err/mod.rs b/src/err/mod.rs index 8516cc9b1db..1cfd403564e 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -177,10 +177,16 @@ impl PyErr { /// ``` pub fn from_value(obj: &PyAny) -> PyErr { let state = if let Ok(obj) = obj.downcast::() { + let pvalue: Py = obj.into(); + + let ptraceback = unsafe { + Py::from_owned_ptr_or_opt(obj.py(), ffi::PyException_GetTraceback(pvalue.as_ptr())) + }; + PyErrState::Normalized(PyErrStateNormalized { ptype: obj.get_type().into(), - pvalue: obj.into(), - ptraceback: None, + pvalue, + ptraceback, }) } else { // Assume obj is Type[Exception]; let later normalization handle if this @@ -228,7 +234,14 @@ impl PyErr { // NB technically this causes one reference count increase and decrease in quick succession // on pvalue, but it's probably not worth optimizing this right now for the additional code // complexity. - self.normalized(py).pvalue.clone_ref(py) + let normalized = self.normalized(py); + let exc = normalized.pvalue.clone_ref(py); + if let Some(tb) = normalized.ptraceback.as_ref() { + unsafe { + ffi::PyException_SetTraceback(exc.as_ptr(), tb.as_ptr()); + } + } + exc } /// Returns the traceback of this exception object. diff --git a/src/types/traceback.rs b/src/types/traceback.rs index 82916558a55..dfd318a7d30 100644 --- a/src/types/traceback.rs +++ b/src/types/traceback.rs @@ -65,7 +65,7 @@ impl PyTraceback { #[cfg(test)] mod tests { - use crate::Python; + use crate::{prelude::*, types::PyDict}; #[test] fn format_traceback() { @@ -80,4 +80,49 @@ mod tests { ); }) } + + #[test] + fn test_err_from_value() { + Python::with_gil(|py| { + let locals = PyDict::new(py); + // Produce an error from python so that it has a traceback + py.run( + r" +try: + raise ValueError('raised exception') +except Exception as e: + err = e +", + None, + Some(locals), + ) + .unwrap(); + let err = PyErr::from_value(locals.get_item("err").unwrap()); + let traceback = err.value(py).getattr("__traceback__").unwrap(); + assert!(err.traceback(py).unwrap().is(traceback)); + }) + } + + #[test] + fn test_err_into_py() { + Python::with_gil(|py| { + let locals = PyDict::new(py); + // Produce an error from python so that it has a traceback + py.run( + r" +def f(): + raise ValueError('raised exception') +", + None, + Some(locals), + ) + .unwrap(); + let f = locals.get_item("f").unwrap(); + let err = f.call0().unwrap_err(); + let traceback = err.traceback(py).unwrap(); + let err_object = err.clone_ref(py).into_py(py).into_ref(py); + + assert!(err_object.getattr("__traceback__").unwrap().is(traceback)); + }) + } } From a4949aea088c0a23f1ff42d4f6a6ac7ca4d3eba1 Mon Sep 17 00:00:00 2001 From: Zak Stucke Date: Wed, 19 Jul 2023 12:51:14 +0300 Subject: [PATCH 33/57] Combined changelog lines --- newsfragments/3328.fixed.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/newsfragments/3328.fixed.md b/newsfragments/3328.fixed.md index 18c19212957..7db0293b4d0 100644 --- a/newsfragments/3328.fixed.md +++ b/newsfragments/3328.fixed.md @@ -1,2 +1 @@ -Fixed `PyErr.from_value()` to maintain original traceback. -Fixed `PyErr.into_value()` to maintain original traceback. +Fixed `PyErr.from_value()` and `PyErr.into_value()` to both maintain original traceback instead of losing it on conversion. \ No newline at end of file From 756314b4fa68ca6bb9fa37da1b36fb54fb2c4f57 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Mon, 10 Jul 2023 23:41:20 +0100 Subject: [PATCH 34/57] fix exception handling on Python 3.12 --- newsfragments/3306.changed.md | 1 + src/err/err_state.rs | 108 +++++++++++++++++++++++++++++++--- src/err/mod.rs | 95 ++++++++++++++++-------------- 3 files changed, 151 insertions(+), 53 deletions(-) create mode 100644 newsfragments/3306.changed.md diff --git a/newsfragments/3306.changed.md b/newsfragments/3306.changed.md new file mode 100644 index 00000000000..a2bbc985512 --- /dev/null +++ b/newsfragments/3306.changed.md @@ -0,0 +1 @@ +Update `PyErr` for 3.12 betas to avoid deprecated ffi methods. diff --git a/src/err/err_state.rs b/src/err/err_state.rs index abf1817cd57..a7408efdba2 100644 --- a/src/err/err_state.rs +++ b/src/err/err_state.rs @@ -7,11 +7,37 @@ use crate::{ #[derive(Clone)] pub(crate) struct PyErrStateNormalized { + #[cfg(not(Py_3_12))] pub ptype: Py, pub pvalue: Py, + #[cfg(not(Py_3_12))] pub ptraceback: Option>, } +impl PyErrStateNormalized { + #[cfg(not(Py_3_12))] + pub(crate) fn ptype<'py>(&'py self, py: Python<'py>) -> &'py PyType { + self.ptype.as_ref(py) + } + + #[cfg(Py_3_12)] + pub(crate) fn ptype<'py>(&'py self, py: Python<'py>) -> &'py PyType { + self.pvalue.as_ref(py).get_type() + } + + #[cfg(not(Py_3_12))] + pub(crate) fn ptraceback<'py>(&'py self, py: Python<'py>) -> Option<&'py PyTraceback> { + self.ptraceback + .as_ref() + .map(|traceback| traceback.as_ref(py)) + } + + #[cfg(Py_3_12)] + pub(crate) fn ptraceback<'py>(&'py self, py: Python<'py>) -> Option<&'py PyTraceback> { + unsafe { py.from_owned_ptr_or_opt(ffi::PyException_GetTraceback(self.pvalue.as_ptr())) } + } +} + pub(crate) struct PyErrStateLazyFnOutput { pub(crate) ptype: PyObject, pub(crate) pvalue: PyObject, @@ -22,6 +48,7 @@ pub(crate) type PyErrStateLazyFn = pub(crate) enum PyErrState { Lazy(Box), + #[cfg(not(Py_3_12))] FfiTuple { ptype: PyObject, pvalue: Option, @@ -53,9 +80,23 @@ impl PyErrState { pvalue: args.arguments(py), })) } -} -impl PyErrState { + pub(crate) fn normalized(pvalue: &PyBaseException) -> Self { + Self::Normalized(PyErrStateNormalized { + #[cfg(not(Py_3_12))] + ptype: pvalue.get_type().into(), + pvalue: pvalue.into(), + #[cfg(not(Py_3_12))] + ptraceback: unsafe { + Py::from_owned_ptr_or_opt( + pvalue.py(), + ffi::PyException_GetTraceback(pvalue.as_ptr()), + ) + }, + }) + } + + #[cfg(not(Py_3_12))] pub(crate) fn into_ffi_tuple( self, py: Python<'_>, @@ -64,7 +105,11 @@ impl PyErrState { PyErrState::Lazy(lazy) => { let PyErrStateLazyFnOutput { ptype, pvalue } = lazy(py); if unsafe { ffi::PyExceptionClass_Check(ptype.as_ptr()) } == 0 { - Self::exceptions_must_derive_from_base_exception(py).into_ffi_tuple(py) + PyErrState::lazy( + PyTypeError::type_object(py), + "exceptions must derive from BaseException", + ) + .into_ffi_tuple(py) } else { (ptype.into_ptr(), pvalue.into_ptr(), std::ptr::null_mut()) } @@ -82,10 +127,57 @@ impl PyErrState { } } - fn exceptions_must_derive_from_base_exception(py: Python<'_>) -> Self { - PyErrState::lazy( - PyTypeError::type_object(py), - "exceptions must derive from BaseException", - ) + #[cfg(not(Py_3_12))] + pub(crate) fn normalize(self, py: Python<'_>) -> PyErrStateNormalized { + let (mut ptype, mut pvalue, mut ptraceback) = self.into_ffi_tuple(py); + + unsafe { + ffi::PyErr_NormalizeException(&mut ptype, &mut pvalue, &mut ptraceback); + PyErrStateNormalized { + ptype: Py::from_owned_ptr_or_opt(py, ptype).expect("Exception type missing"), + pvalue: Py::from_owned_ptr_or_opt(py, pvalue).expect("Exception value missing"), + ptraceback: Py::from_owned_ptr_or_opt(py, ptraceback), + } + } + } + + #[cfg(Py_3_12)] + pub(crate) fn normalize(self, py: Python<'_>) -> PyErrStateNormalized { + // To keep the implementation simple, just write the exception into the interpreter, + // which will cause it to be normalized + self.restore(py); + // Safety: self.restore(py) will set the raised exception + let pvalue = unsafe { Py::from_owned_ptr(py, ffi::PyErr_GetRaisedException()) }; + PyErrStateNormalized { pvalue } + } + + #[cfg(not(Py_3_12))] + pub(crate) fn restore(self, py: Python<'_>) { + let (ptype, pvalue, ptraceback) = self.into_ffi_tuple(py); + unsafe { ffi::PyErr_Restore(ptype, pvalue, ptraceback) } + } + + #[cfg(Py_3_12)] + pub(crate) fn restore(self, py: Python<'_>) { + match self { + PyErrState::Lazy(lazy) => { + let PyErrStateLazyFnOutput { ptype, pvalue } = lazy(py); + unsafe { + if ffi::PyExceptionClass_Check(ptype.as_ptr()) == 0 { + ffi::PyErr_SetString( + PyTypeError::type_object_raw(py).cast(), + "exceptions must derive from BaseException\0" + .as_ptr() + .cast(), + ) + } else { + ffi::PyErr_SetObject(ptype.as_ptr(), pvalue.as_ptr()) + } + } + } + PyErrState::Normalized(PyErrStateNormalized { pvalue }) => unsafe { + ffi::PyErr_SetRaisedException(pvalue.into_ptr()) + }, + } } } diff --git a/src/err/mod.rs b/src/err/mod.rs index 1cfd403564e..f50b9568763 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -177,17 +177,7 @@ impl PyErr { /// ``` pub fn from_value(obj: &PyAny) -> PyErr { let state = if let Ok(obj) = obj.downcast::() { - let pvalue: Py = obj.into(); - - let ptraceback = unsafe { - Py::from_owned_ptr_or_opt(obj.py(), ffi::PyException_GetTraceback(pvalue.as_ptr())) - }; - - PyErrState::Normalized(PyErrStateNormalized { - ptype: obj.get_type().into(), - pvalue, - ptraceback, - }) + PyErrState::normalized(obj) } else { // Assume obj is Type[Exception]; let later normalization handle if this // is not the case @@ -209,7 +199,7 @@ impl PyErr { /// }); /// ``` pub fn get_type<'py>(&'py self, py: Python<'py>) -> &'py PyType { - self.normalized(py).ptype.as_ref(py) + self.normalized(py).ptype(py) } /// Returns the value of this exception. @@ -236,7 +226,7 @@ impl PyErr { // complexity. let normalized = self.normalized(py); let exc = normalized.pvalue.clone_ref(py); - if let Some(tb) = normalized.ptraceback.as_ref() { + if let Some(tb) = normalized.ptraceback(py) { unsafe { ffi::PyException_SetTraceback(exc.as_ptr(), tb.as_ptr()); } @@ -256,10 +246,7 @@ impl PyErr { /// }); /// ``` pub fn traceback<'py>(&'py self, py: Python<'py>) -> Option<&'py PyTraceback> { - self.normalized(py) - .ptraceback - .as_ref() - .map(|obj| obj.as_ref(py)) + self.normalized(py).ptraceback(py) } /// Gets whether an error is present in the Python interpreter's global state. @@ -278,6 +265,11 @@ impl PyErr { /// expected to have been set, for example from [`PyErr::occurred`] or by an error return value /// from a C FFI function, use [`PyErr::fetch`]. pub fn take(py: Python<'_>) -> Option { + Self::_take(py) + } + + #[cfg(not(Py_3_12))] + fn _take(py: Python<'_>) -> Option { let (ptype, pvalue, ptraceback) = unsafe { let mut ptype: *mut ffi::PyObject = std::ptr::null_mut(); let mut pvalue: *mut ffi::PyObject = std::ptr::null_mut(); @@ -316,17 +308,12 @@ impl PyErr { .map(|py_str| py_str.to_string_lossy().into()) .unwrap_or_else(|| String::from("Unwrapped panic from Python code")); - eprintln!( - "--- PyO3 is resuming a panic after fetching a PanicException from Python. ---" - ); - eprintln!("Python stack trace below:"); - - unsafe { - ffi::PyErr_Restore(ptype.into_ptr(), pvalue.into_ptr(), ptraceback.into_ptr()); - ffi::PyErr_PrintEx(0); - } - - std::panic::resume_unwind(Box::new(msg)) + let state = PyErrState::FfiTuple { + ptype, + pvalue, + ptraceback, + }; + Self::print_panic_and_unwind(py, state, msg) } Some(PyErr::from_state(PyErrState::FfiTuple { @@ -336,6 +323,35 @@ impl PyErr { })) } + #[cfg(Py_3_12)] + fn _take(py: Python<'_>) -> Option { + let pvalue = unsafe { + py.from_owned_ptr_or_opt::(ffi::PyErr_GetRaisedException()) + }?; + if pvalue.get_type().as_ptr() == PanicException::type_object_raw(py).cast() { + let msg: String = pvalue + .str() + .map(|py_str| py_str.to_string_lossy().into()) + .unwrap_or_else(|_| String::from("Unwrapped panic from Python code")); + Self::print_panic_and_unwind(py, PyErrState::normalized(pvalue), msg) + } + + Some(PyErr::from_state(PyErrState::normalized(pvalue))) + } + + fn print_panic_and_unwind(py: Python<'_>, state: PyErrState, msg: String) -> ! { + eprintln!("--- PyO3 is resuming a panic after fetching a PanicException from Python. ---"); + eprintln!("Python stack trace below:"); + + state.restore(py); + + unsafe { + ffi::PyErr_PrintEx(0); + } + + std::panic::resume_unwind(Box::new(msg)) + } + /// Equivalent to [PyErr::take], but when no error is set: /// - Panics in debug mode. /// - Returns a `SystemError` in release mode. @@ -457,15 +473,10 @@ impl PyErr { /// This is the opposite of `PyErr::fetch()`. #[inline] pub fn restore(self, py: Python<'_>) { - let state = match self.state.into_inner() { - Some(state) => state, - // Safety: restore takes `self` by value so nothing else is accessing this err - // and the invariant is that state is always defined except during make_normalized - None => unsafe { std::hint::unreachable_unchecked() }, - }; - - let (ptype, pvalue, ptraceback) = state.into_ffi_tuple(py); - unsafe { ffi::PyErr_Restore(ptype, pvalue, ptraceback) } + self.state + .into_inner() + .expect("PyErr state should never be invalid outside of normalization") + .restore(py) } /// Reports the error as unraisable. @@ -649,17 +660,10 @@ impl PyErr { .take() .expect("Cannot normalize a PyErr while already normalizing it.") }; - let (mut ptype, mut pvalue, mut ptraceback) = state.into_ffi_tuple(py); unsafe { - ffi::PyErr_NormalizeException(&mut ptype, &mut pvalue, &mut ptraceback); let self_state = &mut *self.state.get(); - *self_state = Some(PyErrState::Normalized(PyErrStateNormalized { - ptype: Py::from_owned_ptr_or_opt(py, ptype).expect("Exception type missing"), - pvalue: Py::from_owned_ptr_or_opt(py, pvalue).expect("Exception value missing"), - ptraceback: Py::from_owned_ptr_or_opt(py, ptraceback), - })); - + *self_state = Some(PyErrState::Normalized(state.normalize(py))); match self_state { Some(PyErrState::Normalized(n)) => n, _ => unreachable!(), @@ -826,6 +830,7 @@ mod tests { assert!(err.is_instance_of::(py)); err.restore(py); let err = PyErr::fetch(py); + assert!(err.is_instance_of::(py)); assert_eq!( err.to_string(), From 9c5ec1da8dde93e6ef3533f2cb65b7191bdb6c42 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Thu, 20 Jul 2023 08:54:24 -0400 Subject: [PATCH 35/57] Install wheel for the setuptools-rust-starter example Seems to be needed on python 3.12 --- examples/setuptools-rust-starter/requirements-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/setuptools-rust-starter/requirements-dev.txt b/examples/setuptools-rust-starter/requirements-dev.txt index c52fb607d12..3235670811c 100644 --- a/examples/setuptools-rust-starter/requirements-dev.txt +++ b/examples/setuptools-rust-starter/requirements-dev.txt @@ -1,3 +1,4 @@ pytest>=3.5.0 setuptools_rust~=1.0.0 pip>=21.3 +wheel From c7e528e722947bd289e1f9235b17d384b9e9570a Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Fri, 21 Jul 2023 10:34:13 +0100 Subject: [PATCH 36/57] add 3.12 and PyPy 3.10 to clippy jobs --- noxfile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/noxfile.py b/noxfile.py index 23f04785fed..4d6a0176c87 100644 --- a/noxfile.py +++ b/noxfile.py @@ -15,8 +15,8 @@ PYO3_DIR = Path(__file__).parent -PY_VERSIONS = ("3.7", "3.8", "3.9", "3.10", "3.11") -PYPY_VERSIONS = ("3.7", "3.8", "3.9") +PY_VERSIONS = ("3.7", "3.8", "3.9", "3.10", "3.11", "3.12") +PYPY_VERSIONS = ("3.7", "3.8", "3.9", "3.10") @nox.session(venv_backend="none") From 4d54c505f63d8207a002b5ec913f86f340aa604d Mon Sep 17 00:00:00 2001 From: Tom Godkin Date: Sat, 22 Jul 2023 21:30:46 +0100 Subject: [PATCH 37/57] Add haem to example projects --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 2138bdc3d54..2dc7b821065 100644 --- a/README.md +++ b/README.md @@ -192,6 +192,7 @@ about this topic. - [fastuuid](https://github.com/thedrow/fastuuid/) _Python bindings to Rust's UUID library._ - [feos](https://github.com/feos-org/feos) _Lightning fast thermodynamic modeling in Rust with fully developed Python interface._ - [forust](https://github.com/jinlow/forust) _A lightweight gradient boosted decision tree library written in Rust._ +- [haem](https://github.com/BooleanCat/haem) _A Python library for working on Bioinformatics problems._ - [html-py-ever](https://github.com/PyO3/setuptools-rust/tree/main/examples/html-py-ever) _Using [html5ever](https://github.com/servo/html5ever) through [kuchiki](https://github.com/kuchiki-rs/kuchiki) to speed up html parsing and css-selecting._ - [hyperjson](https://github.com/mre/hyperjson) _A hyper-fast Python module for reading/writing JSON data using Rust's serde-json._ - [inline-python](https://github.com/fusion-engineering/inline-python) _Inline Python code directly in your Rust code._ From b85a8ee57f8b1e0e36127b08894abbf8e434cf6a Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Sat, 22 Jul 2023 21:38:57 +0100 Subject: [PATCH 38/57] ci: avoid failure to build numpy on 3.12 --- pytests/noxfile.py | 13 ++++++++----- pytests/tests/test_sequence.py | 15 ++++----------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/pytests/noxfile.py b/pytests/noxfile.py index 2eff279391d..bab55868011 100644 --- a/pytests/noxfile.py +++ b/pytests/noxfile.py @@ -1,21 +1,24 @@ import nox -import platform +from nox.command import CommandFailed nox.options.sessions = ["test"] @nox.session -def test(session): +def test(session: nox.Session): session.install("-rrequirements-dev.txt") - if platform.system() == "Linux" and platform.python_implementation() == "CPython": - session.install("numpy>=1.16") + try: + session.install("--only-binary=numpy", "numpy>=1.16") + except CommandFailed: + # No binary wheel for numpy available on this platform + pass session.install("maturin") session.run_always("maturin", "develop") session.run("pytest", *session.posargs) @nox.session -def bench(session): +def bench(session: nox.Session): session.install("-rrequirements-dev.txt") session.install(".") session.run("pytest", "--benchmark-enable", "--benchmark-only", *session.posargs) diff --git a/pytests/tests/test_sequence.py b/pytests/tests/test_sequence.py index 91aac50ac72..b943ba48fad 100644 --- a/pytests/tests/test_sequence.py +++ b/pytests/tests/test_sequence.py @@ -1,5 +1,4 @@ import pytest -import platform from pyo3_pytests import sequence @@ -21,21 +20,15 @@ def test_vec_from_str(): sequence.vec_to_vec_pystring("123") -@pytest.mark.skipif( - platform.system() != "Linux" or platform.python_implementation() != "CPython", - reason="Binary NumPy wheels are not available for all platforms and Python implementations", -) def test_vec_from_array(): - import numpy + # binary numpy wheel not available on all platforms + numpy = pytest.importorskip("numpy") assert sequence.vec_to_vec_i32(numpy.array([1, 2, 3])) == [1, 2, 3] -@pytest.mark.skipif( - platform.system() != "Linux" or platform.python_implementation() != "CPython", - reason="Binary NumPy wheels are not available for all platforms and Python implementations", -) def test_rust_array_from_array(): - import numpy + # binary numpy wheel not available on all platforms + numpy = pytest.importorskip("numpy") assert sequence.array_to_array_i32(numpy.array([1, 2, 3])) == [1, 2, 3] From e3f17dc004ea61c9c314f1dd50e4a6a4a4c6c4d1 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Sun, 23 Jul 2023 16:32:02 +0200 Subject: [PATCH 39/57] Add PyType_GetDict for Python 3.12 --- newsfragments/3339.added.md | 1 + pyo3-ffi/src/cpython/object.rs | 3 +++ 2 files changed, 4 insertions(+) create mode 100644 newsfragments/3339.added.md diff --git a/newsfragments/3339.added.md b/newsfragments/3339.added.md new file mode 100644 index 00000000000..88884bfd841 --- /dev/null +++ b/newsfragments/3339.added.md @@ -0,0 +1 @@ +Define `PyType_GetDict()` FFI for CPython 3.12 or later. diff --git a/pyo3-ffi/src/cpython/object.rs b/pyo3-ffi/src/cpython/object.rs index 76ab074f3d0..8209524fce7 100644 --- a/pyo3-ffi/src/cpython/object.rs +++ b/pyo3-ffi/src/cpython/object.rs @@ -349,6 +349,9 @@ pub unsafe fn PyHeapType_GET_MEMBERS( // skipped _PyType_GetModuleByDef extern "C" { + #[cfg(Py_3_12)] + pub fn PyType_GetDict(o: *mut PyTypeObject) -> *mut PyObject; + #[cfg_attr(PyPy, link_name = "PyPyObject_Print")] pub fn PyObject_Print(o: *mut PyObject, fp: *mut ::libc::FILE, flags: c_int) -> c_int; From 5f05dbb433989c16cd9b44af4e8bfc3c41bae81f Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Fri, 21 Jul 2023 14:30:02 +0100 Subject: [PATCH 40/57] add PyErr::display --- newsfragments/3334.added.md | 1 + pyo3-ffi/src/pythonrun.rs | 3 +++ src/err/mod.rs | 23 +++++++++++++++++++++-- src/exceptions.rs | 16 ++++++++-------- src/instance.rs | 2 +- src/marker.rs | 7 +++++-- src/types/any.rs | 5 ++++- tests/test_append_to_inittab.rs | 2 +- tests/test_class_new.rs | 4 ++-- tests/test_datetime.rs | 2 +- tests/test_inheritance.rs | 2 +- tests/test_proto_methods.rs | 6 +++--- 12 files changed, 51 insertions(+), 22 deletions(-) create mode 100644 newsfragments/3334.added.md diff --git a/newsfragments/3334.added.md b/newsfragments/3334.added.md new file mode 100644 index 00000000000..fd3824ad0dc --- /dev/null +++ b/newsfragments/3334.added.md @@ -0,0 +1 @@ +Add `PyErr::Display` for all Python versions, and FFI symbol `PyErr_DisplayException` for Python 3.12. diff --git a/pyo3-ffi/src/pythonrun.rs b/pyo3-ffi/src/pythonrun.rs index 196cf5354e5..e5f20de0058 100644 --- a/pyo3-ffi/src/pythonrun.rs +++ b/pyo3-ffi/src/pythonrun.rs @@ -15,6 +15,9 @@ extern "C" { pub fn PyErr_PrintEx(arg1: c_int); #[cfg_attr(PyPy, link_name = "PyPyErr_Display")] pub fn PyErr_Display(arg1: *mut PyObject, arg2: *mut PyObject, arg3: *mut PyObject); + + #[cfg(Py_3_12)] + pub fn PyErr_DisplayException(exc: *mut PyObject); } // skipped PyOS_InputHook diff --git a/src/err/mod.rs b/src/err/mod.rs index f50b9568763..a824a383c18 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -431,13 +431,32 @@ impl PyErr { } /// Prints a standard traceback to `sys.stderr`. + pub fn display(&self, py: Python<'_>) { + #[cfg(Py_3_12)] + unsafe { + ffi::PyErr_DisplayException(self.value(py).as_ptr()) + } + + #[cfg(not(Py_3_12))] + unsafe { + ffi::PyErr_Display( + self.get_type(py).as_ptr(), + self.value(py).as_ptr(), + self.traceback(py) + .map_or(std::ptr::null_mut(), PyTraceback::as_ptr), + ) + } + } + + /// Calls `sys.excepthook` and then prints a standard traceback to `sys.stderr`. pub fn print(&self, py: Python<'_>) { self.clone_ref(py).restore(py); unsafe { ffi::PyErr_PrintEx(0) } } - /// Prints a standard traceback to `sys.stderr`, and sets - /// `sys.last_{type,value,traceback}` attributes to this exception's data. + /// Calls `sys.excepthook` and then prints a standard traceback to `sys.stderr`. + /// + /// Additionally sets `sys.last_{type,value,traceback,exc}` attributes to this exception. pub fn print_and_set_sys_last_vars(&self, py: Python<'_>) { self.clone_ref(py).restore(py); unsafe { ffi::PyErr_PrintEx(1) } diff --git a/src/exceptions.rs b/src/exceptions.rs index b8b2145ad2c..48a09865666 100644 --- a/src/exceptions.rs +++ b/src/exceptions.rs @@ -813,20 +813,20 @@ mod tests { let err: PyErr = gaierror::new_err(()); let socket = py .import("socket") - .map_err(|e| e.print(py)) + .map_err(|e| e.display(py)) .expect("could not import socket"); let d = PyDict::new(py); d.set_item("socket", socket) - .map_err(|e| e.print(py)) + .map_err(|e| e.display(py)) .expect("could not setitem"); d.set_item("exc", err) - .map_err(|e| e.print(py)) + .map_err(|e| e.display(py)) .expect("could not setitem"); py.run("assert isinstance(exc, socket.gaierror)", None, Some(d)) - .map_err(|e| e.print(py)) + .map_err(|e| e.display(py)) .expect("assertion failed"); }); } @@ -837,15 +837,15 @@ mod tests { let err: PyErr = MessageError::new_err(()); let email = py .import("email") - .map_err(|e| e.print(py)) + .map_err(|e| e.display(py)) .expect("could not import email"); let d = PyDict::new(py); d.set_item("email", email) - .map_err(|e| e.print(py)) + .map_err(|e| e.display(py)) .expect("could not setitem"); d.set_item("exc", err) - .map_err(|e| e.print(py)) + .map_err(|e| e.display(py)) .expect("could not setitem"); py.run( @@ -853,7 +853,7 @@ mod tests { None, Some(d), ) - .map_err(|e| e.print(py)) + .map_err(|e| e.display(py)) .expect("assertion failed"); }); } diff --git a/src/instance.rs b/src/instance.rs index 92f0126d629..aa9cbf34f01 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -1237,7 +1237,7 @@ a = A() Python::with_gil(|py| { let v = py .eval("...", None, None) - .map_err(|e| e.print(py)) + .map_err(|e| e.display(py)) .unwrap() .to_object(py); diff --git a/src/marker.rs b/src/marker.rs index d0f486001ed..8912f664b69 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -1041,7 +1041,7 @@ mod tests { // Make sure builtin names are accessible let v: i32 = py .eval("min(1, 2)", None, None) - .map_err(|e| e.print(py)) + .map_err(|e| e.display(py)) .unwrap() .extract() .unwrap(); @@ -1158,7 +1158,10 @@ mod tests { Python::with_gil(|py| { assert_eq!(py.Ellipsis().to_string(), "Ellipsis"); - let v = py.eval("...", None, None).map_err(|e| e.print(py)).unwrap(); + let v = py + .eval("...", None, None) + .map_err(|e| e.display(py)) + .unwrap(); assert!(v.eq(py.Ellipsis()).unwrap()); }); diff --git a/src/types/any.rs b/src/types/any.rs index 281e318701c..5c6b84d6b66 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -1411,7 +1411,10 @@ class SimpleClass: #[test] fn test_is_ellipsis() { Python::with_gil(|py| { - let v = py.eval("...", None, None).map_err(|e| e.print(py)).unwrap(); + let v = py + .eval("...", None, None) + .map_err(|e| e.display(py)) + .unwrap(); assert!(v.is_ellipsis()); diff --git a/tests/test_append_to_inittab.rs b/tests/test_append_to_inittab.rs index 220a73912f1..e0a57da1b5c 100644 --- a/tests/test_append_to_inittab.rs +++ b/tests/test_append_to_inittab.rs @@ -26,7 +26,7 @@ assert module_with_functions.foo() == 123 None, None, ) - .map_err(|e| e.print(py)) + .map_err(|e| e.display(py)) .unwrap(); }) } diff --git a/tests/test_class_new.rs b/tests/test_class_new.rs index 77c571eac3f..8cb426861db 100644 --- a/tests/test_class_new.rs +++ b/tests/test_class_new.rs @@ -127,7 +127,7 @@ fn new_with_two_args() { let typeobj = py.get_type::(); let wrp = typeobj .call((10, 20), None) - .map_err(|e| e.print(py)) + .map_err(|e| e.display(py)) .unwrap(); let obj = wrp.downcast::>().unwrap(); let obj_ref = obj.borrow(); @@ -172,7 +172,7 @@ assert c.from_rust is False let globals = PyModule::import(py, "__main__").unwrap().dict(); globals.set_item("SuperClass", super_cls).unwrap(); py.run(source, Some(globals), None) - .map_err(|e| e.print(py)) + .map_err(|e| e.display(py)) .unwrap(); }); } diff --git a/tests/test_datetime.rs b/tests/test_datetime.rs index f863afdce74..37b2d060303 100644 --- a/tests/test_datetime.rs +++ b/tests/test_datetime.rs @@ -86,7 +86,7 @@ fn test_time_check() { fn test_datetime_check() { Python::with_gil(|py| { let (obj, sub_obj, sub_sub_obj) = _get_subclasses(py, "datetime", "2018, 1, 1, 13, 30, 15") - .map_err(|e| e.print(py)) + .map_err(|e| e.display(py)) .unwrap(); unsafe { PyDateTime_IMPORT() } diff --git a/tests/test_inheritance.rs b/tests/test_inheritance.rs index 8c46a4de4ae..aa4166a754d 100644 --- a/tests/test_inheritance.rs +++ b/tests/test_inheritance.rs @@ -26,7 +26,7 @@ fn subclass() { None, Some(d), ) - .map_err(|e| e.print(py)) + .map_err(|e| e.display(py)) .unwrap(); }); } diff --git a/tests/test_proto_methods.rs b/tests/test_proto_methods.rs index eba5a0cec7b..50584fbf9b6 100644 --- a/tests/test_proto_methods.rs +++ b/tests/test_proto_methods.rs @@ -689,7 +689,7 @@ asyncio.run(main()) let globals = PyModule::import(py, "__main__").unwrap().dict(); globals.set_item("Once", once).unwrap(); py.run(source, Some(globals), None) - .map_err(|e| e.print(py)) + .map_err(|e| e.display(py)) .unwrap(); }); } @@ -746,7 +746,7 @@ asyncio.run(main()) .set_item("AsyncIterator", py.get_type::()) .unwrap(); py.run(source, Some(globals), None) - .map_err(|e| e.print(py)) + .map_err(|e| e.display(py)) .unwrap(); }); } @@ -815,7 +815,7 @@ assert c.counter.count == 1 let globals = PyModule::import(py, "__main__").unwrap().dict(); globals.set_item("Counter", counter).unwrap(); py.run(source, Some(globals), None) - .map_err(|e| e.print(py)) + .map_err(|e| e.display(py)) .unwrap(); }); } From b349313ff6979c05d150d2ddedd80d65d4b32ae1 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Tue, 25 Jul 2023 07:24:12 +0100 Subject: [PATCH 41/57] always run ffi-check on dev pythons --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f1e92149e7c..438f545b273 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,7 +25,7 @@ on: jobs: build: - continue-on-error: ${{ contains(fromJSON('["3.7", "3.12-dev", "pypy3.7", "pypy3.10"]'), inputs.python-version) }} + continue-on-error: ${{ endsWith(inputs.python-version, '-dev') || contains(fromJSON('["3.7", "pypy3.7", "pypy3.10"]'), inputs.python-version) }} runs-on: ${{ inputs.os }} steps: - uses: actions/checkout@v3 @@ -138,7 +138,7 @@ jobs: - name: Run pyo3-ffi-check # pypy 3.7 and 3.8 are not PEP 3123 compliant so fail checks here, nor # is pypy 3.9 on windows - if: ${{ steps.ffi-changes.outputs.changed == 'true' && inputs.rust == 'stable' && inputs.python-version != 'pypy3.7' && inputs.python-version != 'pypy3.8' && !(inputs.python-version == 'pypy3.9' && contains(inputs.os, 'windows')) }} + if: ${{ endsWith(inputs.python-version, '-dev') || (steps.ffi-changes.outputs.changed == 'true' && inputs.rust == 'stable' && inputs.python-version != 'pypy3.7' && inputs.python-version != 'pypy3.8' && !(inputs.python-version == 'pypy3.9' && contains(inputs.os, 'windows'))) }} run: nox -s ffi-check From 4d2e498b9c80693b2ee89f281824d63879a45ff8 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Mon, 24 Jul 2023 22:19:33 +0100 Subject: [PATCH 42/57] remove PyUnicode_WCHAR_KIND from docs on Py_3_12 --- pyo3-ffi/src/cpython/unicodeobject.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pyo3-ffi/src/cpython/unicodeobject.rs b/pyo3-ffi/src/cpython/unicodeobject.rs index f3f7207ca5e..c1bebc4f1bc 100644 --- a/pyo3-ffi/src/cpython/unicodeobject.rs +++ b/pyo3-ffi/src/cpython/unicodeobject.rs @@ -284,8 +284,9 @@ impl PyASCIIObject { /// Get the `kind` field of the [`PyASCIIObject`] state bitfield. /// - /// Returns one of: [`PyUnicode_WCHAR_KIND`], [`PyUnicode_1BYTE_KIND`], [`PyUnicode_2BYTE_KIND`], - /// [`PyUnicode_4BYTE_KIND`] + /// Returns one of: + #[cfg_attr(not(Py_3_12), doc = "[`PyUnicode_WCHAR_KIND`], ")] + /// [`PyUnicode_1BYTE_KIND`], [`PyUnicode_2BYTE_KIND`], or [`PyUnicode_4BYTE_KIND`]. #[inline] pub unsafe fn kind(&self) -> c_uint { PyASCIIObjectState::from(self.state).kind() @@ -293,8 +294,9 @@ impl PyASCIIObject { /// Set the `kind` field of the [`PyASCIIObject`] state bitfield. /// - /// Calling this function with an argument that is not [`PyUnicode_WCHAR_KIND`], [`PyUnicode_1BYTE_KIND`], - /// [`PyUnicode_2BYTE_KIND`], or [`PyUnicode_4BYTE_KIND`] is invalid. + /// Calling this function with an argument that is not + #[cfg_attr(not(Py_3_12), doc = "[`PyUnicode_WCHAR_KIND`], ")] + /// [`PyUnicode_1BYTE_KIND`], [`PyUnicode_2BYTE_KIND`], or [`PyUnicode_4BYTE_KIND`] is invalid. #[inline] pub unsafe fn set_kind(&mut self, val: c_uint) { let mut state = PyASCIIObjectState::from(self.state); From b57c33befcd61bf1252ac21ff615807fe48729d6 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Tue, 25 Jul 2023 05:26:38 +0100 Subject: [PATCH 43/57] fix ffi check failures for 3.12.0b4 --- newsfragments/3342.changed.md | 1 + pyo3-ffi-check/macro/Cargo.toml | 1 + pyo3-ffi-check/macro/src/lib.rs | 23 ++++++++++-- pyo3-ffi-check/src/main.rs | 10 +++-- pyo3-ffi/src/cpython/code.rs | 59 ++++++++++++++++++++++++++---- pyo3-ffi/src/cpython/funcobject.rs | 2 + pyo3-ffi/src/cpython/genobject.rs | 1 + pyo3-ffi/src/cpython/object.rs | 2 + 8 files changed, 85 insertions(+), 14 deletions(-) create mode 100644 newsfragments/3342.changed.md diff --git a/newsfragments/3342.changed.md b/newsfragments/3342.changed.md new file mode 100644 index 00000000000..71435df06ab --- /dev/null +++ b/newsfragments/3342.changed.md @@ -0,0 +1 @@ +Update `pyo3::ffi` struct definitions to be compatible with 3.12.0b4. diff --git a/pyo3-ffi-check/macro/Cargo.toml b/pyo3-ffi-check/macro/Cargo.toml index 999342fa9c9..206e2f44620 100644 --- a/pyo3-ffi-check/macro/Cargo.toml +++ b/pyo3-ffi-check/macro/Cargo.toml @@ -12,3 +12,4 @@ glob = "0.3" quote = "1" proc-macro2 = "1" scraper = "0.17" +pyo3-build-config = { path = "../../pyo3-build-config" } diff --git a/pyo3-ffi-check/macro/src/lib.rs b/pyo3-ffi-check/macro/src/lib.rs index 5d03606276e..e572a9ad76c 100644 --- a/pyo3-ffi-check/macro/src/lib.rs +++ b/pyo3-ffi-check/macro/src/lib.rs @@ -1,8 +1,14 @@ use std::{env, fs, path::PathBuf}; use proc_macro2::{Ident, Span, TokenStream, TokenTree}; +use pyo3_build_config::PythonVersion; use quote::quote; +const PY_3_12: PythonVersion = PythonVersion { + major: 3, + minor: 12, +}; + /// Macro which expands to multiple macro calls, one per pyo3-ffi struct. #[proc_macro] pub fn for_all_structs(input: proc_macro::TokenStream) -> proc_macro::TokenStream { @@ -130,16 +136,27 @@ pub fn for_all_fields(input: proc_macro::TokenStream) -> proc_macro::TokenStream let mut output = TokenStream::new(); for el in html.select(&selector) { - let id = el + let field_name = el .value() .id() .unwrap() .strip_prefix("structfield.") .unwrap(); - let field_ident = Ident::new(id, Span::call_site()); + let field_ident = Ident::new(field_name, Span::call_site()); + + let bindgen_field_ident = if (pyo3_build_config::get().version >= PY_3_12) + && struct_name == "PyObject" + && field_name == "ob_refcnt" + { + // PyObject since 3.12 implements ob_refcnt as a union; bindgen creates + // an anonymous name for the field + Ident::new("__bindgen_anon_1", Span::call_site()) + } else { + field_ident.clone() + }; - output.extend(quote!(#macro_name!(#struct_name, #field_ident);)); + output.extend(quote!(#macro_name!(#struct_name, #field_ident, #bindgen_field_ident);)); } output.into() diff --git a/pyo3-ffi-check/src/main.rs b/pyo3-ffi-check/src/main.rs index 91a7dca6ee5..99713524702 100644 --- a/pyo3-ffi-check/src/main.rs +++ b/pyo3-ffi-check/src/main.rs @@ -47,9 +47,11 @@ fn main() { } macro_rules! check_field { - ($struct_name:ident, $field:ident) => {{ + ($struct_name:ident, $field:ident, $bindgen_field:ident) => {{ + #[allow(clippy::used_underscore_binding)] let pyo3_ffi_offset = memoffset::offset_of!(pyo3_ffi::$struct_name, $field); - let bindgen_offset = memoffset::offset_of!(bindings::$struct_name, $field); + #[allow(clippy::used_underscore_binding)] + let bindgen_offset = memoffset::offset_of!(bindings::$struct_name, $bindgen_field); if pyo3_ffi_offset != bindgen_offset { failed = true; @@ -79,7 +81,9 @@ fn main() { non_upper_case_globals, dead_code, improper_ctypes, - clippy::all + clippy::all, + // clippy fails with lots of errors if this is not set specifically + clippy::used_underscore_binding )] mod bindings { include!(concat!(env!("OUT_DIR"), "/bindings.rs")); diff --git a/pyo3-ffi/src/cpython/code.rs b/pyo3-ffi/src/cpython/code.rs index fa5b1c20df9..67a952b9b9b 100644 --- a/pyo3-ffi/src/cpython/code.rs +++ b/pyo3-ffi/src/cpython/code.rs @@ -4,12 +4,30 @@ use crate::pyport::Py_ssize_t; #[allow(unused_imports)] use std::os::raw::{c_char, c_int, c_short, c_uchar, c_void}; +#[cfg(all(Py_3_8, not(PyPy), not(Py_3_11)))] +opaque_struct!(_PyOpcache); + +pub const _PY_MONITORING_UNGROUPED_EVENTS: usize = 14; +pub const _PY_MONITORING_EVENTS: usize = 16; + +#[cfg(Py_3_12)] +#[repr(C)] +#[derive(Copy, Clone)] +pub struct _Py_Monitors { + pub tools: [u8; _PY_MONITORING_UNGROUPED_EVENTS], +} + // skipped _Py_CODEUNIT + // skipped _Py_OPCODE // skipped _Py_OPARG -#[cfg(all(Py_3_8, not(PyPy), not(Py_3_11)))] -opaque_struct!(_PyOpcache); +// skipped _py_make_codeunit + +// skipped _py_set_opcode + +// skipped _Py_MAKE_CODEUNIT +// skipped _Py_SET_OPCODE #[cfg(Py_3_12)] #[repr(C)] @@ -21,6 +39,27 @@ pub struct _PyCoCached { pub _co_freevars: *mut PyObject, } +#[cfg(Py_3_12)] +#[repr(C)] +#[derive(Copy, Clone)] +pub struct _PyCoLineInstrumentationData { + pub original_opcode: u8, + pub line_delta: i8, +} + +#[cfg(Py_3_12)] +#[repr(C)] +#[derive(Copy, Clone)] +pub struct _PyCoMonitoringData { + pub local_monitors: _Py_Monitors, + pub active_monitors: _Py_Monitors, + pub tools: *mut u8, + pub lines: *mut _PyCoLineInstrumentationData, + pub line_tools: *mut u8, + pub per_instruction_opcodes: *mut u8, + pub per_instruction_tools: *mut u8, +} + #[cfg(all(not(PyPy), not(Py_3_7)))] opaque_struct!(PyCodeObject); @@ -95,8 +134,7 @@ pub struct PyCodeObject { pub co_flags: c_int, #[cfg(not(Py_3_12))] pub co_warmup: c_int, - #[cfg(Py_3_12)] - pub _co_linearray_entry_size: c_short, + pub co_argcount: c_int, pub co_posonlyargcount: c_int, pub co_kwonlyargcount: c_int, @@ -107,9 +145,12 @@ pub struct PyCodeObject { #[cfg(Py_3_12)] pub co_framesize: c_int, pub co_nlocals: c_int, + #[cfg(not(Py_3_12))] pub co_nplaincellvars: c_int, pub co_ncellvars: c_int, pub co_nfreevars: c_int, + #[cfg(Py_3_12)] + pub co_version: u32, pub co_localsplusnames: *mut PyObject, pub co_localspluskinds: *mut PyObject, @@ -120,13 +161,15 @@ pub struct PyCodeObject { pub co_weakreflist: *mut PyObject, #[cfg(not(Py_3_12))] pub _co_code: *mut PyObject, - #[cfg(Py_3_12)] - pub _co_cached: *mut _PyCoCached, #[cfg(not(Py_3_12))] pub _co_linearray: *mut c_char, - pub _co_firsttraceable: c_int, #[cfg(Py_3_12)] - pub _co_linearray: *mut c_char, + pub _co_cached: *mut _PyCoCached, + #[cfg(Py_3_12)] + pub _co_instrumentation_version: u64, + #[cfg(Py_3_12)] + pub _co_monitoring: *mut _PyCoMonitoringData, + pub _co_firsttraceable: c_int, pub co_extra: *mut c_void, pub co_code_adaptive: [c_char; 1], } diff --git a/pyo3-ffi/src/cpython/funcobject.rs b/pyo3-ffi/src/cpython/funcobject.rs index 443988ddbd9..f5e2ae8a929 100644 --- a/pyo3-ffi/src/cpython/funcobject.rs +++ b/pyo3-ffi/src/cpython/funcobject.rs @@ -39,6 +39,8 @@ pub struct PyFunctionObject { pub func_weakreflist: *mut PyObject, pub func_module: *mut PyObject, pub func_annotations: *mut PyObject, + #[cfg(Py_3_12)] + pub func_typeparams: *mut PyObject, pub vectorcall: Option, #[cfg(Py_3_11)] pub func_version: u32, diff --git a/pyo3-ffi/src/cpython/genobject.rs b/pyo3-ffi/src/cpython/genobject.rs index 6b522bf8123..e409337ba79 100644 --- a/pyo3-ffi/src/cpython/genobject.rs +++ b/pyo3-ffi/src/cpython/genobject.rs @@ -15,6 +15,7 @@ pub struct PyGenObject { pub gi_frame: *mut PyFrameObject, #[cfg(not(Py_3_10))] pub gi_running: c_int, + #[cfg(not(Py_3_12))] pub gi_code: *mut PyObject, pub gi_weakreflist: *mut PyObject, pub gi_name: *mut PyObject, diff --git a/pyo3-ffi/src/cpython/object.rs b/pyo3-ffi/src/cpython/object.rs index 8209524fce7..c2519adc609 100644 --- a/pyo3-ffi/src/cpython/object.rs +++ b/pyo3-ffi/src/cpython/object.rs @@ -299,6 +299,8 @@ pub struct PyTypeObject { #[derive(Clone)] pub struct _specialization_cache { pub getitem: *mut PyObject, + #[cfg(Py_3_12)] + pub getitem_version: u32, } #[repr(C)] From afb56143e0bfade15b0b47c0a3ac3ef1ad232a36 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Tue, 25 Jul 2023 21:24:40 +0100 Subject: [PATCH 44/57] ci: stop allowing failure for pypy 3.10 --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 438f545b273..5a14617da67 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,7 +25,7 @@ on: jobs: build: - continue-on-error: ${{ endsWith(inputs.python-version, '-dev') || contains(fromJSON('["3.7", "pypy3.7", "pypy3.10"]'), inputs.python-version) }} + continue-on-error: ${{ endsWith(inputs.python-version, '-dev') || contains(fromJSON('["3.7", "pypy3.7"]'), inputs.python-version) }} runs-on: ${{ inputs.os }} steps: - uses: actions/checkout@v3 From 04c9c4177ceabfeaee7e7bac36a9db0242e14f24 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Fri, 28 Jul 2023 15:11:56 +0100 Subject: [PATCH 45/57] add PyAny::downcast_exact --- newsfragments/3346.added.md | 1 + src/types/any.rs | 38 +++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 newsfragments/3346.added.md diff --git a/newsfragments/3346.added.md b/newsfragments/3346.added.md new file mode 100644 index 00000000000..7e7085cb6d5 --- /dev/null +++ b/newsfragments/3346.added.md @@ -0,0 +1 @@ +Add `PyAny::downcast_exact`. diff --git a/src/types/any.rs b/src/types/any.rs index 5c6b84d6b66..5065cfd934c 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -903,6 +903,44 @@ impl PyAny { ::try_from(self) } + /// Downcast this `PyAny` to a concrete Python type or pyclass (but not a subclass of it). + /// + /// It is almost always better to use [`PyAny::downcast`] because it accounts for Python + /// subtyping. Use this method only when you do not want to allow subtypes. + /// + /// The advantage of this method over [`PyAny::downcast`] is that it is faster. The implementation + /// of `downcast_exact` uses the equivalent of the Python expression `type(self) is T`, whereas + /// `downcast` uses `isinstance(self, T)`. + /// + /// For extracting a Rust-only type, see [`PyAny::extract`](struct.PyAny.html#method.extract). + /// + /// # Example: Downcasting to a specific Python object but not a subtype + /// + /// ```rust + /// use pyo3::prelude::*; + /// use pyo3::types::{PyBool, PyLong}; + /// + /// Python::with_gil(|py| { + /// let b = PyBool::new(py, true); + /// assert!(b.is_instance_of::()); + /// let any: &PyAny = b.as_ref(); + /// + /// // `bool` is a subtype of `int`, so `downcast` will accept a `bool` as an `int` + /// // but `downcast_exact` will not. + /// assert!(any.downcast::().is_ok()); + /// assert!(any.downcast_exact::().is_err()); + /// + /// assert!(any.downcast_exact::().is_ok()); + /// }); + /// ``` + #[inline] + pub fn downcast_exact<'p, T>(&'p self) -> Result<&'p T, PyDowncastError<'_>> + where + T: PyTryFrom<'p>, + { + ::try_from_exact(self) + } + /// Converts this `PyAny` to a concrete Python type without checking validity. /// /// # Safety From 7090e6b4102d149b9ef7aa15c95449682ea865a6 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Thu, 27 Jul 2023 08:55:52 +0100 Subject: [PATCH 46/57] optimize `float` -> `f64` conversions on non-abi3 --- benches/bench_frompyobject.rs | 12 +++++++++- newsfragments/3345.changed.md | 1 + pyo3-ffi/src/cpython/floatobject.rs | 27 ++++++++++++++++++++++ pyo3-ffi/src/cpython/mod.rs | 2 ++ pyo3-ffi/src/floatobject.rs | 13 ----------- src/types/floatob.rs | 36 ++++++++++++++++++++--------- 6 files changed, 66 insertions(+), 25 deletions(-) create mode 100644 newsfragments/3345.changed.md create mode 100644 pyo3-ffi/src/cpython/floatobject.rs diff --git a/benches/bench_frompyobject.rs b/benches/bench_frompyobject.rs index 587a89bcdf3..c2dfbc0ea85 100644 --- a/benches/bench_frompyobject.rs +++ b/benches/bench_frompyobject.rs @@ -2,7 +2,7 @@ use criterion::{black_box, criterion_group, criterion_main, Bencher, Criterion}; use pyo3::{ prelude::*, - types::{PyList, PyString}, + types::{PyFloat, PyList, PyString}, }; #[derive(FromPyObject)] @@ -79,6 +79,15 @@ fn not_a_list_via_extract_enum(b: &mut Bencher<'_>) { }) } +fn f64_from_pyobject(b: &mut Bencher<'_>) { + Python::with_gil(|py| { + let obj = PyFloat::new(py, 1.234); + b.iter(|| { + let _: f64 = obj.extract().unwrap(); + }); + }) +} + fn criterion_benchmark(c: &mut Criterion) { c.bench_function("enum_from_pyobject", enum_from_pyobject); c.bench_function("list_via_downcast", list_via_downcast); @@ -86,6 +95,7 @@ fn criterion_benchmark(c: &mut Criterion) { c.bench_function("not_a_list_via_downcast", not_a_list_via_downcast); c.bench_function("not_a_list_via_extract", not_a_list_via_extract); c.bench_function("not_a_list_via_extract_enum", not_a_list_via_extract_enum); + c.bench_function("f64_from_pyobject", f64_from_pyobject); } criterion_group!(benches, criterion_benchmark); diff --git a/newsfragments/3345.changed.md b/newsfragments/3345.changed.md new file mode 100644 index 00000000000..35143e6abba --- /dev/null +++ b/newsfragments/3345.changed.md @@ -0,0 +1 @@ +Optimize conversion of `float` to `f64` (and `PyFloat::value`) on non-abi3 builds. diff --git a/pyo3-ffi/src/cpython/floatobject.rs b/pyo3-ffi/src/cpython/floatobject.rs new file mode 100644 index 00000000000..e33da0b91b9 --- /dev/null +++ b/pyo3-ffi/src/cpython/floatobject.rs @@ -0,0 +1,27 @@ +use crate::{PyFloat_Check, PyObject}; +use std::os::raw::c_double; + +#[repr(C)] +pub struct PyFloatObject { + pub ob_base: PyObject, + pub ob_fval: c_double, +} + +#[inline] +pub unsafe fn _PyFloat_CAST(op: *mut PyObject) -> *mut PyFloatObject { + debug_assert_eq!(PyFloat_Check(op), 1); + op.cast() +} + +#[inline] +pub unsafe fn PyFloat_AS_DOUBLE(op: *mut PyObject) -> c_double { + (*_PyFloat_CAST(op)).ob_fval +} + +// skipped PyFloat_Pack2 +// skipped PyFloat_Pack4 +// skipped PyFloat_Pack8 + +// skipped PyFloat_Unpack2 +// skipped PyFloat_Unpack4 +// skipped PyFloat_Unpack8 diff --git a/pyo3-ffi/src/cpython/mod.rs b/pyo3-ffi/src/cpython/mod.rs index 7c39a9e4dd6..30c5d3b9a65 100644 --- a/pyo3-ffi/src/cpython/mod.rs +++ b/pyo3-ffi/src/cpython/mod.rs @@ -29,6 +29,7 @@ pub(crate) mod pymem; pub(crate) mod pystate; pub(crate) mod pythonrun; // skipped sysmodule.h +pub(crate) mod floatobject; pub(crate) mod tupleobject; pub(crate) mod unicodeobject; pub(crate) mod weakrefobject; @@ -42,6 +43,7 @@ pub use self::compile::*; pub use self::descrobject::*; #[cfg(not(PyPy))] pub use self::dictobject::*; +pub use self::floatobject::*; pub use self::frameobject::*; pub use self::funcobject::*; pub use self::genobject::*; diff --git a/pyo3-ffi/src/floatobject.rs b/pyo3-ffi/src/floatobject.rs index 15f71ba2e72..a6e682cf8df 100644 --- a/pyo3-ffi/src/floatobject.rs +++ b/pyo3-ffi/src/floatobject.rs @@ -5,13 +5,6 @@ use std::os::raw::{c_double, c_int}; // TODO: remove (see https://github.com/PyO3/pyo3/pull/1341#issuecomment-751515985) opaque_struct!(PyFloatObject); -#[cfg(not(Py_LIMITED_API))] -#[repr(C)] -pub struct PyFloatObject { - pub ob_base: PyObject, - pub ob_fval: c_double, -} - #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { #[cfg_attr(PyPy, link_name = "PyPyFloat_Type")] @@ -43,12 +36,6 @@ extern "C" { pub fn PyFloat_AsDouble(arg1: *mut PyObject) -> c_double; } -#[cfg(not(Py_LIMITED_API))] -#[inline] -pub unsafe fn PyFloat_AS_DOUBLE(op: *mut PyObject) -> c_double { - (*(op as *mut PyFloatObject)).ob_fval -} - // skipped non-limited _PyFloat_Pack2 // skipped non-limited _PyFloat_Pack4 // skipped non-limited _PyFloat_Pack8 diff --git a/src/types/floatob.rs b/src/types/floatob.rs index 82db228dfa1..f56bcb1f9fd 100644 --- a/src/types/floatob.rs +++ b/src/types/floatob.rs @@ -1,7 +1,8 @@ #[cfg(feature = "experimental-inspect")] use crate::inspect::types::TypeInfo; use crate::{ - ffi, AsPyPointer, FromPyObject, IntoPy, PyAny, PyErr, PyObject, PyResult, Python, ToPyObject, + ffi, AsPyPointer, FromPyObject, IntoPy, PyAny, PyErr, PyNativeType, PyObject, PyResult, Python, + ToPyObject, }; use std::os::raw::c_double; @@ -29,7 +30,16 @@ impl PyFloat { /// Gets the value of this float. pub fn value(&self) -> c_double { - unsafe { ffi::PyFloat_AsDouble(self.as_ptr()) } + #[cfg(not(Py_LIMITED_API))] + unsafe { + // Safety: self is PyFloat object + ffi::PyFloat_AS_DOUBLE(self.as_ptr()) + } + + #[cfg(Py_LIMITED_API)] + unsafe { + ffi::PyFloat_AsDouble(self.as_ptr()) + } } } @@ -54,6 +64,15 @@ impl<'source> FromPyObject<'source> for f64 { // PyFloat_AsDouble returns -1.0 upon failure #![allow(clippy::float_cmp)] fn extract(obj: &'source PyAny) -> PyResult { + // On non-limited API, .value() uses PyFloat_AS_DOUBLE which + // allows us to have an optimized fast path for the case when + // we have exactly a `float` object (it's not worth going through + // `isinstance` machinery for subclasses). + #[cfg(not(Py_LIMITED_API))] + if let Ok(float) = obj.downcast_exact::() { + return Ok(float.value()); + } + let v = unsafe { ffi::PyFloat_AsDouble(obj.as_ptr()) }; if v == -1.0 { @@ -101,11 +120,7 @@ impl<'source> FromPyObject<'source> for f32 { #[cfg(test)] mod tests { - #[cfg(not(Py_LIMITED_API))] - use crate::ffi::PyFloat_AS_DOUBLE; - #[cfg(not(Py_LIMITED_API))] - use crate::AsPyPointer; - use crate::{Python, ToPyObject}; + use crate::{types::PyFloat, Python, ToPyObject}; macro_rules! num_to_py_object_and_back ( ($func_name:ident, $t1:ty, $t2:ty) => ( @@ -127,15 +142,14 @@ mod tests { num_to_py_object_and_back!(to_from_f32, f32, f32); num_to_py_object_and_back!(int_to_float, i32, f64); - #[cfg(not(Py_LIMITED_API))] #[test] - fn test_as_double_macro() { + fn test_float_value() { use assert_approx_eq::assert_approx_eq; Python::with_gil(|py| { let v = 1.23f64; - let obj = v.to_object(py); - assert_approx_eq!(v, unsafe { PyFloat_AS_DOUBLE(obj.as_ptr()) }); + let obj = PyFloat::new(py, 1.23); + assert_approx_eq!(v, obj.value()); }); } } From 9282f3180fa9b44df8afd3c037cbbef41a49ad2e Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Sat, 29 Jul 2023 07:39:47 +0000 Subject: [PATCH 47/57] Add a PySlice::full() constructor for :: --- src/types/slice.rs | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/types/slice.rs b/src/types/slice.rs index e82b535a45b..fdf9f7a856b 100644 --- a/src/types/slice.rs +++ b/src/types/slice.rs @@ -54,6 +54,14 @@ impl PySlice { } } + /// Constructs a new full slice that is equivalent to `::`. + pub fn full(py: Python<'_>) -> &PySlice { + unsafe { + let ptr = ffi::PySlice_New(ffi::Py_None(), ffi::Py_None(), ffi::Py_None()); + py.from_owned_ptr(ptr) + } + } + /// Retrieves the start, stop, and step indices from the slice object, /// assuming a sequence of length `length`, and stores the length of the /// slice in its `slicelength` member. @@ -116,6 +124,34 @@ mod tests { }); } + #[test] + fn test_py_slice_full() { + Python::with_gil(|py| { + let slice = PySlice::full(py); + assert!(slice.getattr("start").unwrap().is_none(),); + assert!(slice.getattr("stop").unwrap().is_none(),); + assert!(slice.getattr("step").unwrap().is_none(),); + assert_eq!( + slice.indices(0).unwrap(), + PySliceIndices { + start: 0, + stop: 0, + step: 1, + slicelength: 0, + }, + ); + assert_eq!( + slice.indices(42).unwrap(), + PySliceIndices { + start: 0, + stop: 42, + step: 1, + slicelength: 42, + }, + ); + }); + } + #[test] fn test_py_slice_indices_new() { let start = 0; From 79d5e4f8f48f58473c319453da04dc8a8e4b2a69 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Sat, 29 Jul 2023 07:43:17 +0000 Subject: [PATCH 48/57] Added newsfragment --- newsfragments/3353.added.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3353.added.md diff --git a/newsfragments/3353.added.md b/newsfragments/3353.added.md new file mode 100644 index 00000000000..2db4834e660 --- /dev/null +++ b/newsfragments/3353.added.md @@ -0,0 +1 @@ +Add `PySlice::full()` to construct a full slice (`::`). From 00a6ce6d919d087c3af4de98055b8cab3460f636 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Fri, 28 Jul 2023 15:59:01 +0100 Subject: [PATCH 49/57] fix compile failure for getter with return lifetime of self --- pyo3-macros-backend/src/pymethod.rs | 36 +++++++++++++---------------- tests/test_getter_setter.rs | 20 ++++++++++++++++ 2 files changed, 36 insertions(+), 20 deletions(-) diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index 9b7a8a5aa19..d5860f83809 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -620,7 +620,7 @@ pub fn impl_py_getter_def( let deprecations = property_type.deprecations(); let doc = property_type.doc(); - let getter_impl = match property_type { + let body = match property_type { PropertyType::Descriptor { field_index, field, .. } => { @@ -629,31 +629,28 @@ pub fn impl_py_getter_def( span: Span::call_site(), } .receiver(cls, ExtractErrorMode::Raise); - if let Some(ident) = &field.ident { + let field_token = if let Some(ident) = &field.ident { // named struct field - quote!(::std::clone::Clone::clone(&(#slf.#ident))) + ident.to_token_stream() } else { // tuple struct field - let index = syn::Index::from(field_index); - quote!(::std::clone::Clone::clone(&(#slf.#index))) - } - } - PropertyType::Function { - spec, self_type, .. - } => impl_call_getter(cls, spec, self_type)?, - }; - - let conversion = match property_type { - PropertyType::Descriptor { .. } => { + syn::Index::from(field_index).to_token_stream() + }; quote! { - let item: _pyo3::Py<_pyo3::PyAny> = _pyo3::IntoPy::into_py(item, _py); - ::std::result::Result::Ok(_pyo3::conversion::IntoPyPointer::into_ptr(item)) + ::std::result::Result::Ok( + _pyo3::conversion::IntoPyPointer::into_ptr( + _pyo3::IntoPy::<_pyo3::Py<_pyo3::PyAny>>::into_py(::std::clone::Clone::clone(&(#slf.#field_token)), _py) + ) + ) } } // Forward to `IntoPyCallbackOutput`, to handle `#[getter]`s returning results. - PropertyType::Function { .. } => { + PropertyType::Function { + spec, self_type, .. + } => { + let call = impl_call_getter(cls, spec, self_type)?; quote! { - _pyo3::callback::convert(_py, item) + _pyo3::callback::convert(_py, #call) } } }; @@ -688,8 +685,7 @@ pub fn impl_py_getter_def( _py: _pyo3::Python<'_>, _slf: *mut _pyo3::ffi::PyObject ) -> _pyo3::PyResult<*mut _pyo3::ffi::PyObject> { - let item = #getter_impl; - #conversion + #body } }; diff --git a/tests/test_getter_setter.rs b/tests/test_getter_setter.rs index 222421ce43f..a94c739d36a 100644 --- a/tests/test_getter_setter.rs +++ b/tests/test_getter_setter.rs @@ -216,3 +216,23 @@ fn cell_getter_setter() { ); }); } + +#[test] +fn borrowed_value_with_lifetime_of_self() { + #[pyclass] + struct BorrowedValue {} + + #[pymethods] + impl BorrowedValue { + #[getter] + fn value(&self) -> &str { + "value" + } + } + + Python::with_gil(|py| { + let inst = Py::new(py, BorrowedValue {}).unwrap().to_object(py); + + py_run!(py, inst, "assert inst.value == 'value'"); + }); +} From 0f796e77af205b9a4cff593387a1ab77c29936b2 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Sat, 29 Jul 2023 21:23:50 +0100 Subject: [PATCH 50/57] macros: `_py` -> `py` --- pyo3-macros-backend/src/method.rs | 8 ++++---- pyo3-macros-backend/src/pyclass.rs | 2 +- pyo3-macros-backend/src/pymethod.rs | 18 +++++++++--------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/pyo3-macros-backend/src/method.rs b/pyo3-macros-backend/src/method.rs index 9bd33cb5a8e..3d97e70ad56 100644 --- a/pyo3-macros-backend/src/method.rs +++ b/pyo3-macros-backend/src/method.rs @@ -116,12 +116,12 @@ impl FnType { } FnType::FnClass | FnType::FnNewClass => { quote! { - _pyo3::types::PyType::from_type_ptr(_py, _slf as *mut _pyo3::ffi::PyTypeObject), + _pyo3::types::PyType::from_type_ptr(py, _slf as *mut _pyo3::ffi::PyTypeObject), } } FnType::FnModule => { quote! { - _py.from_borrowed_ptr::<_pyo3::types::PyModule>(_slf), + py.from_borrowed_ptr::<_pyo3::types::PyModule>(_slf), } } } @@ -156,7 +156,7 @@ impl ExtractErrorMode { impl SelfType { pub fn receiver(&self, cls: &syn::Type, error_mode: ExtractErrorMode) -> TokenStream { - let py = syn::Ident::new("_py", Span::call_site()); + let py = syn::Ident::new("py", Span::call_site()); let slf = syn::Ident::new("_slf", Span::call_site()); match self { SelfType::Receiver { span, mutable } => { @@ -421,7 +421,7 @@ impl<'a> FnSpec<'a> { ) -> Result { let deprecations = &self.deprecations; let self_arg = self.tp.self_arg(cls, ExtractErrorMode::Raise); - let py = syn::Ident::new("_py", Span::call_site()); + let py = syn::Ident::new("py", Span::call_site()); let func_name = &self.name; let rust_call = |args: Vec| { diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index ca5d233eed0..b4c412e78ae 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -1070,7 +1070,7 @@ impl<'a> PyClassImplsBuilder<'a> { quote! { impl _pyo3::impl_::pyclass::PyClassWithFreeList for #cls { #[inline] - fn get_free_list(_py: _pyo3::Python<'_>) -> &mut _pyo3::impl_::freelist::FreeList<*mut _pyo3::ffi::PyObject> { + fn get_free_list(py: _pyo3::Python<'_>) -> &mut _pyo3::impl_::freelist::FreeList<*mut _pyo3::ffi::PyObject> { static mut FREELIST: *mut _pyo3::impl_::freelist::FreeList<*mut _pyo3::ffi::PyObject> = 0 as *mut _; unsafe { if FREELIST.is_null() { diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index d5860f83809..87d745dcbab 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -490,7 +490,7 @@ fn impl_call_setter( let name = &spec.name; let fncall = if py_arg.is_some() { - quote!(#cls::#name(#slf, _py, _val)) + quote!(#cls::#name(#slf, py, _val)) } else { quote!(#cls::#name(#slf, _val)) }; @@ -556,18 +556,18 @@ pub fn impl_py_setter_def( let associated_method = quote! { #cfg_attrs unsafe fn #wrapper_ident( - _py: _pyo3::Python<'_>, + py: _pyo3::Python<'_>, _slf: *mut _pyo3::ffi::PyObject, _value: *mut _pyo3::ffi::PyObject, ) -> _pyo3::PyResult<::std::os::raw::c_int> { - let _value = _py + let _value = py .from_borrowed_ptr_or_opt(_value) .ok_or_else(|| { _pyo3::exceptions::PyAttributeError::new_err("can't delete attribute") })?; let _val = _pyo3::FromPyObject::extract(_value)?; - _pyo3::callback::convert(_py, #setter_impl) + _pyo3::callback::convert(py, #setter_impl) } }; @@ -603,7 +603,7 @@ fn impl_call_getter( let name = &spec.name; let fncall = if py_arg.is_some() { - quote!(#cls::#name(#slf, _py)) + quote!(#cls::#name(#slf, py)) } else { quote!(#cls::#name(#slf)) }; @@ -650,7 +650,7 @@ pub fn impl_py_getter_def( } => { let call = impl_call_getter(cls, spec, self_type)?; quote! { - _pyo3::callback::convert(_py, #call) + _pyo3::callback::convert(py, #call) } } }; @@ -682,7 +682,7 @@ pub fn impl_py_getter_def( let associated_method = quote! { #cfg_attrs unsafe fn #wrapper_ident( - _py: _pyo3::Python<'_>, + py: _pyo3::Python<'_>, _slf: *mut _pyo3::ffi::PyObject ) -> _pyo3::PyResult<*mut _pyo3::ffi::PyObject> { #body @@ -1083,7 +1083,7 @@ impl SlotDef { spec.name.span() => format!("`{}` must be `unsafe fn`", method_name) ); } - let py = syn::Ident::new("_py", Span::call_site()); + let py = syn::Ident::new("py", Span::call_site()); let arg_types: &Vec<_> = &arguments.iter().map(|arg| arg.ffi_type()).collect(); let arg_idents: &Vec<_> = &(0..arguments.len()) .map(|i| format_ident!("arg{}", i)) @@ -1191,7 +1191,7 @@ impl SlotFragmentDef { let fragment_trait = format_ident!("PyClass{}SlotFragment", fragment); let method = syn::Ident::new(fragment, Span::call_site()); let wrapper_ident = format_ident!("__pymethod_{}__", fragment); - let py = syn::Ident::new("_py", Span::call_site()); + let py = syn::Ident::new("py", Span::call_site()); let arg_types: &Vec<_> = &arguments.iter().map(|arg| arg.ffi_type()).collect(); let arg_idents: &Vec<_> = &(0..arguments.len()) .map(|i| format_ident!("arg{}", i)) From 1fa46d0726b5f72845cdd1bf7d342f249d1ca617 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Fri, 28 Jul 2023 22:39:23 +0100 Subject: [PATCH 51/57] add macro quotes module for common snippets --- pyo3-macros-backend/src/lib.rs | 1 + pyo3-macros-backend/src/method.rs | 7 ++----- pyo3-macros-backend/src/pymethod.rs | 16 ++++++---------- pyo3-macros-backend/src/quotes.rs | 15 +++++++++++++++ 4 files changed, 24 insertions(+), 15 deletions(-) create mode 100644 pyo3-macros-backend/src/quotes.rs diff --git a/pyo3-macros-backend/src/lib.rs b/pyo3-macros-backend/src/lib.rs index 61cdbb630c0..745a8471c2b 100644 --- a/pyo3-macros-backend/src/lib.rs +++ b/pyo3-macros-backend/src/lib.rs @@ -19,6 +19,7 @@ mod pyclass; mod pyfunction; mod pyimpl; mod pymethod; +mod quotes; pub use frompyobject::build_derive_from_pyobject; pub use module::{process_functions_in_module, pymodule_impl, PyModuleOptions}; diff --git a/pyo3-macros-backend/src/method.rs b/pyo3-macros-backend/src/method.rs index 3d97e70ad56..b290fd55c13 100644 --- a/pyo3-macros-backend/src/method.rs +++ b/pyo3-macros-backend/src/method.rs @@ -3,6 +3,7 @@ use crate::deprecations::{Deprecation, Deprecations}; use crate::params::impl_arg_params; use crate::pyfunction::{DeprecatedArgs, FunctionSignature, PyFunctionArgPyO3Attributes}; use crate::pyfunction::{PyFunctionOptions, SignatureAttribute}; +use crate::quotes; use crate::utils::{self, PythonDoc}; use proc_macro2::{Span, TokenStream}; use quote::ToTokens; @@ -425,11 +426,7 @@ impl<'a> FnSpec<'a> { let func_name = &self.name; let rust_call = |args: Vec| { - quote! { - _pyo3::impl_::pymethods::OkWrap::wrap(function(#self_arg #(#args),*), #py) - .map(|obj| _pyo3::conversion::IntoPyPointer::into_ptr(obj)) - .map_err(::core::convert::Into::into) - } + quotes::map_result_into_ptr(quotes::ok_wrap(quote! { function(#self_arg #(#args),*) })) }; let rust_name = if let Some(cls) = cls { diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index 87d745dcbab..0658e28af7a 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -3,7 +3,7 @@ use std::borrow::Cow; use crate::attributes::NameAttribute; use crate::method::{CallingConvention, ExtractErrorMode}; use crate::utils::{ensure_not_async_fn, PythonDoc}; -use crate::{deprecations::Deprecations, utils}; +use crate::{deprecations::Deprecations, quotes, utils}; use crate::{ method::{FnArg, FnSpec, FnType, SelfType}, pyfunction::PyFunctionOptions, @@ -446,13 +446,13 @@ fn impl_py_class_attribute(cls: &syn::Type, spec: &FnSpec<'_>) -> syn::Result) -> _pyo3::PyResult<_pyo3::PyObject> { let function = #cls::#name; // Shadow the method name to avoid #3017 #deprecations - _pyo3::impl_::pymethods::OkWrap::wrap(#fncall, py) - .map_err(::core::convert::Into::into) + #body } }; @@ -636,13 +636,9 @@ pub fn impl_py_getter_def( // tuple struct field syn::Index::from(field_index).to_token_stream() }; - quote! { - ::std::result::Result::Ok( - _pyo3::conversion::IntoPyPointer::into_ptr( - _pyo3::IntoPy::<_pyo3::Py<_pyo3::PyAny>>::into_py(::std::clone::Clone::clone(&(#slf.#field_token)), _py) - ) - ) - } + quotes::map_result_into_ptr(quotes::ok_wrap(quote! { + ::std::clone::Clone::clone(&(#slf.#field_token)) + })) } // Forward to `IntoPyCallbackOutput`, to handle `#[getter]`s returning results. PropertyType::Function { diff --git a/pyo3-macros-backend/src/quotes.rs b/pyo3-macros-backend/src/quotes.rs new file mode 100644 index 00000000000..b3b51404cf6 --- /dev/null +++ b/pyo3-macros-backend/src/quotes.rs @@ -0,0 +1,15 @@ +use proc_macro2::TokenStream; +use quote::quote; + +pub(crate) fn ok_wrap(obj: TokenStream) -> TokenStream { + quote! { + _pyo3::impl_::pymethods::OkWrap::wrap(#obj, py) + .map_err(::core::convert::Into::into) + } +} + +pub(crate) fn map_result_into_ptr(result: TokenStream) -> TokenStream { + quote! { + #result.map(_pyo3::IntoPyPointer::into_ptr) + } +} From 1f7c1fc7fb249968e32fe178d5634754a23fa01b Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Sat, 29 Jul 2023 06:28:39 +0100 Subject: [PATCH 52/57] move benches to subdirectory --- .github/workflows/gh-pages.yml | 8 +- Cargo.toml | 60 -------------- Contributing.md | 4 +- noxfile.py | 49 ++++++++++-- pyo3-benches/Cargo.toml | 80 +++++++++++++++++++ .../benches}/bench_any.rs | 0 .../benches}/bench_call.rs | 0 pyo3-benches/benches/bench_comparisons.rs | 70 ++++++++++++++++ .../benches}/bench_decimal.rs | 0 .../benches}/bench_dict.rs | 0 .../benches}/bench_err.rs | 0 .../benches}/bench_extract.rs | 0 .../benches}/bench_frompyobject.rs | 0 .../benches}/bench_gil.rs | 0 .../benches}/bench_intern.rs | 0 .../benches}/bench_list.rs | 0 .../benches}/bench_pyclass.rs | 0 .../benches}/bench_pyobject.rs | 0 .../benches}/bench_set.rs | 0 .../benches}/bench_tuple.rs | 0 20 files changed, 198 insertions(+), 73 deletions(-) create mode 100644 pyo3-benches/Cargo.toml rename {benches => pyo3-benches/benches}/bench_any.rs (100%) rename {benches => pyo3-benches/benches}/bench_call.rs (100%) create mode 100644 pyo3-benches/benches/bench_comparisons.rs rename {benches => pyo3-benches/benches}/bench_decimal.rs (100%) rename {benches => pyo3-benches/benches}/bench_dict.rs (100%) rename {benches => pyo3-benches/benches}/bench_err.rs (100%) rename {benches => pyo3-benches/benches}/bench_extract.rs (100%) rename {benches => pyo3-benches/benches}/bench_frompyobject.rs (100%) rename {benches => pyo3-benches/benches}/bench_gil.rs (100%) rename {benches => pyo3-benches/benches}/bench_intern.rs (100%) rename {benches => pyo3-benches/benches}/bench_list.rs (100%) rename {benches => pyo3-benches/benches}/bench_pyclass.rs (100%) rename {benches => pyo3-benches/benches}/bench_pyobject.rs (100%) rename {benches => pyo3-benches/benches}/bench_set.rs (100%) rename {benches => pyo3-benches/benches}/bench_tuple.rs (100%) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 20d88873b43..46d43316333 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -60,7 +60,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - + - uses: actions/setup-python@v4 - uses: dtolnay/rust-toolchain@stable - uses: actions/cache@v3 @@ -74,8 +74,10 @@ jobs: - name: Run benchmarks run: | - for bench in call dict gil list pyclass pyobject set tuple; do - cargo bench --features hashbrown --bench "bench_$bench" -- --output-format bencher | tee -a output.txt + python -m pip install --upgrade pip && pip install nox + for bench in pyo3-benches/benches/*.rs; do + bench_name=$(basename "$bench" .rs) + nox -s bench -- --bench "$bench_name" -- --output-format bencher | tee -a output.txt done # Download previous benchmark result from cache (if exists) diff --git a/Cargo.toml b/Cargo.toml index d6dd041bb90..12990edcdfc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,7 +44,6 @@ serde = { version = "1.0", optional = true } [dev-dependencies] assert_approx_eq = "1.1.0" chrono = { version = "0.4" } -criterion = "0.3.5" # Required for "and $N others" normalization trybuild = ">=1.0.70" rustversion = "1.0" @@ -114,65 +113,6 @@ full = [ "rust_decimal", ] -[[bench]] -name = "bench_any" -harness = false - -[[bench]] -name = "bench_call" -harness = false - -[[bench]] -name = "bench_err" -harness = false - -[[bench]] -name = "bench_decimal" -harness = false -required-features = ["rust_decimal"] - -[[bench]] -name = "bench_dict" -harness = false - -[[bench]] -name = "bench_frompyobject" -harness = false -required-features = ["macros"] - -[[bench]] -name = "bench_gil" -harness = false - -[[bench]] -name = "bench_list" -harness = false - -[[bench]] -name = "bench_pyclass" -harness = false -required-features = ["macros"] - -[[bench]] -name = "bench_pyobject" -harness = false - -[[bench]] -name = "bench_set" -harness = false - -[[bench]] -name = "bench_tuple" -harness = false - -[[bench]] -name = "bench_intern" -harness = false - -[[bench]] -name = "bench_extract" -harness = false - [workspace] members = [ "pyo3-ffi", diff --git a/Contributing.md b/Contributing.md index 75b129edd1d..319a6397e09 100644 --- a/Contributing.md +++ b/Contributing.md @@ -165,9 +165,9 @@ CI tests both the most recent stable Rust version and the minimum supported Rust PyO3 has two sets of benchmarks for evaluating some aspects of its performance. The benchmark suite is currently very small - please open PRs with new benchmarks if you're interested in helping to expand it! -First, there are Rust-based benchmarks located in the `benches` subdirectory. As long as you have a nightly rust compiler available on your system, you can run these benchmarks with: +First, there are Rust-based benchmarks located in the `pyo3-benches` subdirectory. You can run these benchmarks with: - cargo +nightly bench + nox -s bench Second, there is a Python-based benchmark contained in the `pytests` subdirectory. You can read more about it [here](pytests). diff --git a/noxfile.py b/noxfile.py index 4d6a0176c87..d16227c36ec 100644 --- a/noxfile.py +++ b/noxfile.py @@ -75,7 +75,7 @@ def fmt(session: nox.Session): @nox.session(name="fmt-rust", venv_backend="none") def fmt_rust(session: nox.Session): _run_cargo(session, "fmt", "--all", "--check") - _run_cargo(session, "fmt", *_FFI_CHECK, "--all", "--check") + _run_cargo(session, "fmt", _FFI_CHECK, "--all", "--check") @nox.session(name="fmt-py") @@ -86,7 +86,7 @@ def fmt_py(session: nox.Session): @nox.session(name="clippy", venv_backend="none") def clippy(session: nox.Session) -> bool: - if not _clippy(session): + if not _clippy(session) and _clippy_additional_workspaces(session): session.error("one or more jobs failed") @@ -110,6 +110,33 @@ def _clippy(session: nox.Session, *, env: Dict[str, str] = None) -> bool: return success +def _clippy_additional_workspaces(session: nox.Session) -> bool: + # pyo3-benches and pyo3-ffi-check are in isolated workspaces so that their + # dependencies do not interact with MSRV + + success = True + try: + _run_cargo(session, "clippy", _BENCHES) + except Exception: + success = False + + # Run pyo3-ffi-check only on when not cross-compiling, because it needs to + # have Python headers to feed to bindgen which gets messy when cross-compiling. + target = os.environ.get("CARGO_BUILD_TARGET") + if target is None or _get_rust_default_target() == target: + try: + _build_docs_for_ffi_check(session) + _run_cargo(session, "clippy", _FFI_CHECK, "--workspace", "--all-targets") + except Exception: + success = False + return success + + +@nox.session(venv_backend="none") +def bench(session: nox.Session) -> bool: + _run_cargo(session, "bench", _BENCHES, *session.posargs) + + @nox.session(name="clippy-all", venv_backend="none") def clippy_all(session: nox.Session) -> None: success = True @@ -119,6 +146,7 @@ def _clippy_with_config(env: Dict[str, str]) -> None: success &= _clippy(session, env=env) _for_all_version_configs(session, _clippy_with_config) + success &= _clippy_additional_workspaces(session) if not success: session.error("one or more jobs failed") @@ -376,7 +404,7 @@ def address_sanitizer(session: nox.Session): "test", "--release", "-Zbuild-std", - f"--target={_get_rust_target()}", + f"--target={_get_rust_default_target()}", "--", "--test-threads=1", env={ @@ -533,9 +561,13 @@ def load_pkg_versions(): @nox.session(name="ffi-check") def ffi_check(session: nox.Session): - _run_cargo(session, "doc", *_FFI_CHECK, "-p", "pyo3-ffi", "--no-deps") - _run_cargo(session, "clippy", "--workspace", "--all-targets", *_FFI_CHECK) - _run_cargo(session, "run", *_FFI_CHECK) + _build_docs_for_ffi_check(session) + _run_cargo(session, "run", _FFI_CHECK) + + +def _build_docs_for_ffi_check(session: nox.Session) -> None: + # pyo3-ffi-check needs to scrape docs of pyo3-ffi + _run_cargo(session, "doc", _FFI_CHECK, "-p", "pyo3-ffi", "--no-deps") @lru_cache() @@ -554,7 +586,7 @@ def _get_rust_version() -> Tuple[int, int, int, List[str]]: return (*map(int, version_number.split(".")), extra) -def _get_rust_target() -> str: +def _get_rust_default_target() -> str: for line in _get_rust_info(): if line.startswith(_HOST_LINE_START): return line[len(_HOST_LINE_START) :].strip() @@ -692,4 +724,5 @@ def _job_with_config(implementation, version) -> bool: _job_with_config("PyPy", version) -_FFI_CHECK = ("--manifest-path", "pyo3-ffi-check/Cargo.toml") +_BENCHES = "--manifest-path=pyo3-benches/Cargo.toml" +_FFI_CHECK = "--manifest-path=pyo3-ffi-check/Cargo.toml" diff --git a/pyo3-benches/Cargo.toml b/pyo3-benches/Cargo.toml new file mode 100644 index 00000000000..4bb6b4d9bb6 --- /dev/null +++ b/pyo3-benches/Cargo.toml @@ -0,0 +1,80 @@ +[package] +name = "pyo3-benches" +version = "0.1.0" +description = "In-tree benchmarks for the PyO3 project" +authors = ["PyO3 Project and Contributors "] +edition = "2021" +publish = false + +[dependencies] +pyo3 = { path = "../", features = ["auto-initialize"] } + +[dev-dependencies] +criterion = "0.3.5" + +[[bench]] +name = "bench_any" +harness = false + +[[bench]] +name = "bench_call" +harness = false + +[[bench]] +name = "bench_comparisons" +harness = false + +[[bench]] +name = "bench_err" +harness = false + +[[bench]] +name = "bench_decimal" +harness = false +required-features = ["pyo3/rust_decimal"] + +[[bench]] +name = "bench_dict" +harness = false +required-features = ["pyo3/hashbrown"] + +[[bench]] +name = "bench_frompyobject" +harness = false +required-features = ["pyo3/macros"] + +[[bench]] +name = "bench_gil" +harness = false + +[[bench]] +name = "bench_list" +harness = false + +[[bench]] +name = "bench_pyclass" +harness = false +required-features = ["pyo3/macros"] + +[[bench]] +name = "bench_pyobject" +harness = false + +[[bench]] +name = "bench_set" +harness = false +required-features = ["pyo3/hashbrown"] + +[[bench]] +name = "bench_tuple" +harness = false + +[[bench]] +name = "bench_intern" +harness = false + +[[bench]] +name = "bench_extract" +harness = false + +[workspace] diff --git a/benches/bench_any.rs b/pyo3-benches/benches/bench_any.rs similarity index 100% rename from benches/bench_any.rs rename to pyo3-benches/benches/bench_any.rs diff --git a/benches/bench_call.rs b/pyo3-benches/benches/bench_call.rs similarity index 100% rename from benches/bench_call.rs rename to pyo3-benches/benches/bench_call.rs diff --git a/pyo3-benches/benches/bench_comparisons.rs b/pyo3-benches/benches/bench_comparisons.rs new file mode 100644 index 00000000000..bfa4ac63fa4 --- /dev/null +++ b/pyo3-benches/benches/bench_comparisons.rs @@ -0,0 +1,70 @@ +use criterion::{criterion_group, criterion_main, Bencher, Criterion}; + +use pyo3::{prelude::*, pyclass::CompareOp, Python}; + +#[pyclass] +struct OrderedDunderMethods(i64); + +#[pymethods] +impl OrderedDunderMethods { + fn __lt__(&self, other: &Self) -> bool { + self.0 < other.0 + } + + fn __le__(&self, other: &Self) -> bool { + self.0 <= other.0 + } + + fn __eq__(&self, other: &Self) -> bool { + self.0 == other.0 + } + + fn __ne__(&self, other: &Self) -> bool { + self.0 != other.0 + } + + fn __gt__(&self, other: &Self) -> bool { + self.0 > other.0 + } + + fn __ge__(&self, other: &Self) -> bool { + self.0 >= other.0 + } +} + +#[pyclass] +#[derive(PartialEq, Eq, PartialOrd, Ord)] +struct OrderedRichcmp(i64); + +#[pymethods] +impl OrderedRichcmp { + fn __richcmp__(&self, other: &Self, op: CompareOp) -> bool { + op.matches(self.cmp(other)) + } +} + +fn bench_ordered_dunder_methods(b: &mut Bencher<'_>) { + Python::with_gil(|py| { + let obj1 = Py::new(py, OrderedDunderMethods(0)).unwrap().into_ref(py); + let obj2 = Py::new(py, OrderedDunderMethods(1)).unwrap().into_ref(py); + + b.iter(|| obj2.gt(obj1).unwrap()); + }); +} + +fn bench_ordered_richcmp(b: &mut Bencher<'_>) { + Python::with_gil(|py| { + let obj1 = Py::new(py, OrderedRichcmp(0)).unwrap().into_ref(py); + let obj2 = Py::new(py, OrderedRichcmp(1)).unwrap().into_ref(py); + + b.iter(|| obj2.gt(obj1).unwrap()); + }); +} + +fn criterion_benchmark(c: &mut Criterion) { + c.bench_function("ordered_dunder_methods", bench_ordered_dunder_methods); + c.bench_function("ordered_richcmp", bench_ordered_richcmp); +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); diff --git a/benches/bench_decimal.rs b/pyo3-benches/benches/bench_decimal.rs similarity index 100% rename from benches/bench_decimal.rs rename to pyo3-benches/benches/bench_decimal.rs diff --git a/benches/bench_dict.rs b/pyo3-benches/benches/bench_dict.rs similarity index 100% rename from benches/bench_dict.rs rename to pyo3-benches/benches/bench_dict.rs diff --git a/benches/bench_err.rs b/pyo3-benches/benches/bench_err.rs similarity index 100% rename from benches/bench_err.rs rename to pyo3-benches/benches/bench_err.rs diff --git a/benches/bench_extract.rs b/pyo3-benches/benches/bench_extract.rs similarity index 100% rename from benches/bench_extract.rs rename to pyo3-benches/benches/bench_extract.rs diff --git a/benches/bench_frompyobject.rs b/pyo3-benches/benches/bench_frompyobject.rs similarity index 100% rename from benches/bench_frompyobject.rs rename to pyo3-benches/benches/bench_frompyobject.rs diff --git a/benches/bench_gil.rs b/pyo3-benches/benches/bench_gil.rs similarity index 100% rename from benches/bench_gil.rs rename to pyo3-benches/benches/bench_gil.rs diff --git a/benches/bench_intern.rs b/pyo3-benches/benches/bench_intern.rs similarity index 100% rename from benches/bench_intern.rs rename to pyo3-benches/benches/bench_intern.rs diff --git a/benches/bench_list.rs b/pyo3-benches/benches/bench_list.rs similarity index 100% rename from benches/bench_list.rs rename to pyo3-benches/benches/bench_list.rs diff --git a/benches/bench_pyclass.rs b/pyo3-benches/benches/bench_pyclass.rs similarity index 100% rename from benches/bench_pyclass.rs rename to pyo3-benches/benches/bench_pyclass.rs diff --git a/benches/bench_pyobject.rs b/pyo3-benches/benches/bench_pyobject.rs similarity index 100% rename from benches/bench_pyobject.rs rename to pyo3-benches/benches/bench_pyobject.rs diff --git a/benches/bench_set.rs b/pyo3-benches/benches/bench_set.rs similarity index 100% rename from benches/bench_set.rs rename to pyo3-benches/benches/bench_set.rs diff --git a/benches/bench_tuple.rs b/pyo3-benches/benches/bench_tuple.rs similarity index 100% rename from benches/bench_tuple.rs rename to pyo3-benches/benches/bench_tuple.rs From 0f1846c561781aa66ba4f6b1f46b337a4c214e41 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Sat, 29 Jul 2023 06:42:34 +0100 Subject: [PATCH 53/57] update criterion to 0.5.1 --- pyo3-benches/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyo3-benches/Cargo.toml b/pyo3-benches/Cargo.toml index 4bb6b4d9bb6..f04e5429a06 100644 --- a/pyo3-benches/Cargo.toml +++ b/pyo3-benches/Cargo.toml @@ -10,7 +10,7 @@ publish = false pyo3 = { path = "../", features = ["auto-initialize"] } [dev-dependencies] -criterion = "0.3.5" +criterion = "0.5.1" [[bench]] name = "bench_any" From 5564d61b7c4c63fec84d803418114382811a0711 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Fri, 21 Jul 2023 16:49:00 +0100 Subject: [PATCH 54/57] update object.h definitions for Python 3.12 --- newsfragments/3335.changed.md | 1 + newsfragments/3335.fixed.md | 1 + pyo3-ffi/src/cpython/object.rs | 12 +- pyo3-ffi/src/lib.rs | 10 +- pyo3-ffi/src/object.rs | 400 +++++++++++++++++------- pyo3-ffi/src/{buffer.rs => pybuffer.rs} | 3 + pyo3-ffi/src/pyport.rs | 16 + src/ffi/mod.rs | 2 +- src/ffi/tests.rs | 58 +++- 9 files changed, 374 insertions(+), 129 deletions(-) create mode 100644 newsfragments/3335.changed.md create mode 100644 newsfragments/3335.fixed.md rename pyo3-ffi/src/{buffer.rs => pybuffer.rs} (95%) diff --git a/newsfragments/3335.changed.md b/newsfragments/3335.changed.md new file mode 100644 index 00000000000..c35f1ddb1f2 --- /dev/null +++ b/newsfragments/3335.changed.md @@ -0,0 +1 @@ +Update FFI definitions of `object.h` for Python 3.12 and up. diff --git a/newsfragments/3335.fixed.md b/newsfragments/3335.fixed.md new file mode 100644 index 00000000000..bc7bb8b61d8 --- /dev/null +++ b/newsfragments/3335.fixed.md @@ -0,0 +1 @@ +Fix reference counting of immortal objects on Python 3.12 betas. diff --git a/pyo3-ffi/src/cpython/object.rs b/pyo3-ffi/src/cpython/object.rs index c2519adc609..abf8f1dc61c 100644 --- a/pyo3-ffi/src/cpython/object.rs +++ b/pyo3-ffi/src/cpython/object.rs @@ -1,4 +1,6 @@ use crate::object; +#[cfg(Py_3_8)] +use crate::vectorcallfunc; use crate::{PyObject, Py_ssize_t}; use std::mem; use std::os::raw::{c_char, c_int, c_uint, c_ulong, c_void}; @@ -112,14 +114,6 @@ mod bufferinfo { #[cfg(not(Py_3_11))] pub use self::bufferinfo::*; -#[cfg(Py_3_8)] -pub type vectorcallfunc = unsafe extern "C" fn( - callable: *mut PyObject, - args: *const *mut PyObject, - nargsf: libc::size_t, - kwnames: *mut PyObject, -) -> *mut PyObject; - #[repr(C)] #[derive(Copy, Clone)] pub struct PyNumberMethods { @@ -275,7 +269,7 @@ pub struct PyTypeObject { pub tp_version_tag: c_uint, pub tp_finalize: Option, #[cfg(Py_3_8)] - pub tp_vectorcall: Option, + pub tp_vectorcall: Option, #[cfg(Py_3_12)] pub tp_watched: c_char, #[cfg(any(all(PyPy, Py_3_8, not(Py_3_10)), all(not(PyPy), Py_3_8, not(Py_3_9))))] diff --git a/pyo3-ffi/src/lib.rs b/pyo3-ffi/src/lib.rs index 73ec459b90a..e4176778c7b 100644 --- a/pyo3-ffi/src/lib.rs +++ b/pyo3-ffi/src/lib.rs @@ -274,8 +274,6 @@ macro_rules! addr_of_mut_shim { pub use self::abstract_::*; pub use self::bltinmodule::*; pub use self::boolobject::*; -#[cfg(Py_3_11)] -pub use self::buffer::*; pub use self::bytearrayobject::*; pub use self::bytesobject::*; pub use self::ceval::*; @@ -308,6 +306,8 @@ pub use self::objimpl::*; pub use self::osmodule::*; #[cfg(not(any(PyPy, Py_LIMITED_API, Py_3_10)))] pub use self::pyarena::*; +#[cfg(Py_3_11)] +pub use self::pybuffer::*; pub use self::pycapsule::*; pub use self::pyerrors::*; pub use self::pyframe::*; @@ -335,8 +335,6 @@ mod abstract_; // skipped ast.h mod bltinmodule; mod boolobject; -#[cfg(Py_3_11)] -mod buffer; mod bytearrayobject; mod bytesobject; // skipped cellobject.h @@ -387,8 +385,9 @@ mod osmodule; // skipped py_curses.h #[cfg(not(any(PyPy, Py_LIMITED_API, Py_3_10)))] mod pyarena; +#[cfg(Py_3_11)] +mod pybuffer; mod pycapsule; -// skipped pydecimal.h // skipped pydtrace.h mod pyerrors; // skipped pyexpat.h @@ -402,6 +401,7 @@ mod pylifecycle; mod pymem; mod pyport; mod pystate; +// skipped pystats.h mod pythonrun; // skipped pystrhex.h // skipped pystrcmp.h diff --git a/pyo3-ffi/src/object.rs b/pyo3-ffi/src/object.rs index 7fabcdf6f60..6ff8beb5a9e 100644 --- a/pyo3-ffi/src/object.rs +++ b/pyo3-ffi/src/object.rs @@ -1,5 +1,3 @@ -// FFI note: this file changed a lot between 3.6 and 3.10. -// Some missing definitions may not be marked "skipped". use crate::pyport::{Py_hash_t, Py_ssize_t}; use std::mem; use std::os::raw::{c_char, c_int, c_uint, c_ulong, c_void}; @@ -14,11 +12,24 @@ pub use crate::cpython::object::PyTypeObject; // _PyObject_HEAD_EXTRA: conditionally defined in PyObject_HEAD_INIT // _PyObject_EXTRA_INIT: conditionally defined in PyObject_HEAD_INIT +#[cfg(Py_3_12)] +pub const _Py_IMMORTAL_REFCNT: Py_ssize_t = { + if cfg!(target_pointer_width = "64") { + c_uint::MAX as Py_ssize_t + } else { + // for 32-bit systems, use the lower 30 bits (see comment in CPython's object.h) + (c_uint::MAX >> 2) as Py_ssize_t + } +}; + pub const PyObject_HEAD_INIT: PyObject = PyObject { #[cfg(py_sys_config = "Py_TRACE_REFS")] _ob_next: std::ptr::null_mut(), #[cfg(py_sys_config = "Py_TRACE_REFS")] _ob_prev: std::ptr::null_mut(), + #[cfg(Py_3_12)] + ob_refcnt: PyObjectObRefcnt { ob_refcnt: 1 }, + #[cfg(not(Py_3_12))] ob_refcnt: 1, #[cfg(PyPy)] ob_pypy_link: 0, @@ -28,6 +39,27 @@ pub const PyObject_HEAD_INIT: PyObject = PyObject { // skipped PyObject_VAR_HEAD // skipped Py_INVALID_SIZE +#[repr(C)] +#[derive(Copy, Clone)] +#[cfg(Py_3_12)] +/// This union is anonymous in CPython, so the name was given by PyO3 because +/// Rust unions need a name. +pub union PyObjectObRefcnt { + pub ob_refcnt: Py_ssize_t, + #[cfg(target_pointer_width = "64")] + pub ob_refcnt_split: [crate::PY_UINT32_T; 2], +} + +#[cfg(Py_3_12)] +impl std::fmt::Debug for PyObjectObRefcnt { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", unsafe { self.ob_refcnt }) + } +} + +#[cfg(not(Py_3_12))] +pub type PyObjectObRefcnt = Py_ssize_t; + #[repr(C)] #[derive(Copy, Clone, Debug)] pub struct PyObject { @@ -35,14 +67,13 @@ pub struct PyObject { pub _ob_next: *mut PyObject, #[cfg(py_sys_config = "Py_TRACE_REFS")] pub _ob_prev: *mut PyObject, - pub ob_refcnt: Py_ssize_t, + pub ob_refcnt: PyObjectObRefcnt, #[cfg(PyPy)] pub ob_pypy_link: Py_ssize_t, pub ob_type: *mut PyTypeObject, } // skipped _PyObject_CAST -// skipped _PyObject_CAST_CONST #[repr(C)] #[derive(Debug, Copy, Clone)] @@ -52,18 +83,21 @@ pub struct PyVarObject { } // skipped _PyVarObject_CAST -// skipped _PyVarObject_CAST_CONST #[inline] pub unsafe fn Py_Is(x: *mut PyObject, y: *mut PyObject) -> c_int { (x == y).into() } -// skipped _Py_REFCNT: defined in Py_REFCNT +#[inline] +#[cfg(Py_3_12)] +pub unsafe fn Py_REFCNT(ob: *mut PyObject) -> Py_ssize_t { + (*ob).ob_refcnt.ob_refcnt +} #[inline] +#[cfg(not(Py_3_12))] pub unsafe fn Py_REFCNT(ob: *mut PyObject) -> Py_ssize_t { - assert!(!ob.is_null()); (*ob).ob_refcnt } @@ -72,9 +106,14 @@ pub unsafe fn Py_TYPE(ob: *mut PyObject) -> *mut PyTypeObject { (*ob).ob_type } +// PyLong_Type defined in longobject.rs +// PyBool_Type defined in boolobject.rs + #[inline] pub unsafe fn Py_SIZE(ob: *mut PyObject) -> Py_ssize_t { - (*(ob as *mut PyVarObject)).ob_size + debug_assert_ne!((*ob).ob_type, std::ptr::addr_of_mut!(crate::PyLong_Type)); + debug_assert_ne!((*ob).ob_type, std::ptr::addr_of_mut!(crate::PyBool_Type)); + (*ob.cast::()).ob_size } #[inline] @@ -82,6 +121,18 @@ pub unsafe fn Py_IS_TYPE(ob: *mut PyObject, tp: *mut PyTypeObject) -> c_int { (Py_TYPE(ob) == tp) as c_int } +#[inline(always)] +#[cfg(all(Py_3_12, target_pointer_width = "64"))] +pub unsafe fn _Py_IsImmortal(op: *mut PyObject) -> c_int { + (((*op).ob_refcnt.ob_refcnt as crate::PY_INT32_T) < 0) as c_int +} + +#[inline(always)] +#[cfg(all(Py_3_12, target_pointer_width = "32"))] +pub unsafe fn _Py_IsImmortal(op: *mut PyObject) -> c_int { + ((*op).ob_refcnt.ob_refcnt == _Py_IMMORTAL_REFCNT) as c_int +} + // skipped _Py_SET_REFCNT // skipped Py_SET_REFCNT // skipped _Py_SET_TYPE @@ -89,82 +140,51 @@ pub unsafe fn Py_IS_TYPE(ob: *mut PyObject, tp: *mut PyTypeObject) -> c_int { // skipped _Py_SET_SIZE // skipped Py_SET_SIZE -pub type unaryfunc = unsafe extern "C" fn(arg1: *mut PyObject) -> *mut PyObject; - -pub type binaryfunc = - unsafe extern "C" fn(arg1: *mut PyObject, arg2: *mut PyObject) -> *mut PyObject; - -pub type ternaryfunc = unsafe extern "C" fn( - arg1: *mut PyObject, - arg2: *mut PyObject, - arg3: *mut PyObject, -) -> *mut PyObject; - -pub type inquiry = unsafe extern "C" fn(arg1: *mut PyObject) -> c_int; - -pub type lenfunc = unsafe extern "C" fn(arg1: *mut PyObject) -> Py_ssize_t; - -pub type ssizeargfunc = - unsafe extern "C" fn(arg1: *mut PyObject, arg2: Py_ssize_t) -> *mut PyObject; - +pub type unaryfunc = unsafe extern "C" fn(*mut PyObject) -> *mut PyObject; +pub type binaryfunc = unsafe extern "C" fn(*mut PyObject, *mut PyObject) -> *mut PyObject; +pub type ternaryfunc = + unsafe extern "C" fn(*mut PyObject, *mut PyObject, *mut PyObject) -> *mut PyObject; +pub type inquiry = unsafe extern "C" fn(*mut PyObject) -> c_int; +pub type lenfunc = unsafe extern "C" fn(*mut PyObject) -> Py_ssize_t; +pub type ssizeargfunc = unsafe extern "C" fn(*mut PyObject, Py_ssize_t) -> *mut PyObject; pub type ssizessizeargfunc = - unsafe extern "C" fn(arg1: *mut PyObject, arg2: Py_ssize_t, arg3: Py_ssize_t) -> *mut PyObject; - -pub type ssizeobjargproc = - unsafe extern "C" fn(arg1: *mut PyObject, arg2: Py_ssize_t, arg3: *mut PyObject) -> c_int; - -pub type ssizessizeobjargproc = unsafe extern "C" fn( - arg1: *mut PyObject, - arg2: Py_ssize_t, - arg3: Py_ssize_t, - arg4: *mut PyObject, -) -> c_int; + unsafe extern "C" fn(*mut PyObject, Py_ssize_t, Py_ssize_t) -> *mut PyObject; +pub type ssizeobjargproc = unsafe extern "C" fn(*mut PyObject, Py_ssize_t, *mut PyObject) -> c_int; +pub type ssizessizeobjargproc = + unsafe extern "C" fn(*mut PyObject, Py_ssize_t, Py_ssize_t, arg4: *mut PyObject) -> c_int; +pub type objobjargproc = unsafe extern "C" fn(*mut PyObject, *mut PyObject, *mut PyObject) -> c_int; -pub type objobjargproc = - unsafe extern "C" fn(arg1: *mut PyObject, arg2: *mut PyObject, arg3: *mut PyObject) -> c_int; - -pub type objobjproc = unsafe extern "C" fn(arg1: *mut PyObject, arg2: *mut PyObject) -> c_int; +pub type objobjproc = unsafe extern "C" fn(*mut PyObject, *mut PyObject) -> c_int; pub type visitproc = unsafe extern "C" fn(object: *mut PyObject, arg: *mut c_void) -> c_int; pub type traverseproc = unsafe extern "C" fn(slf: *mut PyObject, visit: visitproc, arg: *mut c_void) -> c_int; -pub type freefunc = unsafe extern "C" fn(arg1: *mut c_void); -pub type destructor = unsafe extern "C" fn(arg1: *mut PyObject); -pub type getattrfunc = - unsafe extern "C" fn(arg1: *mut PyObject, arg2: *mut c_char) -> *mut PyObject; -pub type getattrofunc = - unsafe extern "C" fn(arg1: *mut PyObject, arg2: *mut PyObject) -> *mut PyObject; -pub type setattrfunc = - unsafe extern "C" fn(arg1: *mut PyObject, arg2: *mut c_char, arg3: *mut PyObject) -> c_int; -pub type setattrofunc = - unsafe extern "C" fn(arg1: *mut PyObject, arg2: *mut PyObject, arg3: *mut PyObject) -> c_int; -pub type reprfunc = unsafe extern "C" fn(arg1: *mut PyObject) -> *mut PyObject; -pub type hashfunc = unsafe extern "C" fn(arg1: *mut PyObject) -> Py_hash_t; -pub type richcmpfunc = - unsafe extern "C" fn(arg1: *mut PyObject, arg2: *mut PyObject, arg3: c_int) -> *mut PyObject; -pub type getiterfunc = unsafe extern "C" fn(arg1: *mut PyObject) -> *mut PyObject; -pub type iternextfunc = unsafe extern "C" fn(arg1: *mut PyObject) -> *mut PyObject; -pub type descrgetfunc = unsafe extern "C" fn( - arg1: *mut PyObject, - arg2: *mut PyObject, - arg3: *mut PyObject, -) -> *mut PyObject; -pub type descrsetfunc = - unsafe extern "C" fn(arg1: *mut PyObject, arg2: *mut PyObject, arg3: *mut PyObject) -> c_int; -pub type initproc = - unsafe extern "C" fn(arg1: *mut PyObject, arg2: *mut PyObject, arg3: *mut PyObject) -> c_int; -pub type newfunc = unsafe extern "C" fn( - arg1: *mut PyTypeObject, - arg2: *mut PyObject, - arg3: *mut PyObject, +pub type freefunc = unsafe extern "C" fn(*mut c_void); +pub type destructor = unsafe extern "C" fn(*mut PyObject); +pub type getattrfunc = unsafe extern "C" fn(*mut PyObject, *mut c_char) -> *mut PyObject; +pub type getattrofunc = unsafe extern "C" fn(*mut PyObject, *mut PyObject) -> *mut PyObject; +pub type setattrfunc = unsafe extern "C" fn(*mut PyObject, *mut c_char, *mut PyObject) -> c_int; +pub type setattrofunc = unsafe extern "C" fn(*mut PyObject, *mut PyObject, *mut PyObject) -> c_int; +pub type reprfunc = unsafe extern "C" fn(*mut PyObject) -> *mut PyObject; +pub type hashfunc = unsafe extern "C" fn(*mut PyObject) -> Py_hash_t; +pub type richcmpfunc = unsafe extern "C" fn(*mut PyObject, *mut PyObject, c_int) -> *mut PyObject; +pub type getiterfunc = unsafe extern "C" fn(*mut PyObject) -> *mut PyObject; +pub type iternextfunc = unsafe extern "C" fn(*mut PyObject) -> *mut PyObject; +pub type descrgetfunc = + unsafe extern "C" fn(*mut PyObject, *mut PyObject, *mut PyObject) -> *mut PyObject; +pub type descrsetfunc = unsafe extern "C" fn(*mut PyObject, *mut PyObject, *mut PyObject) -> c_int; +pub type initproc = unsafe extern "C" fn(*mut PyObject, *mut PyObject, *mut PyObject) -> c_int; +pub type newfunc = + unsafe extern "C" fn(*mut PyTypeObject, *mut PyObject, *mut PyObject) -> *mut PyObject; +pub type allocfunc = unsafe extern "C" fn(*mut PyTypeObject, Py_ssize_t) -> *mut PyObject; + +#[cfg(Py_3_8)] +pub type vectorcallfunc = unsafe extern "C" fn( + callable: *mut PyObject, + args: *const *mut PyObject, + nargsf: libc::size_t, + kwnames: *mut PyObject, ) -> *mut PyObject; -pub type allocfunc = - unsafe extern "C" fn(arg1: *mut PyTypeObject, arg2: Py_ssize_t) -> *mut PyObject; -#[cfg(Py_3_11)] -pub type getbufferproc = - unsafe extern "C" fn(arg1: *mut PyObject, arg2: *mut crate::Py_buffer, arg3: c_int) -> c_int; -#[cfg(Py_3_11)] -pub type releasebufferproc = unsafe extern "C" fn(arg1: *mut PyObject, arg2: *mut crate::Py_buffer); #[repr(C)] #[derive(Copy, Clone)] @@ -220,9 +240,32 @@ extern "C" { #[cfg(any(Py_3_10, all(Py_3_9, not(Py_LIMITED_API))))] #[cfg_attr(PyPy, link_name = "PyPyType_GetModuleState")] pub fn PyType_GetModuleState(arg1: *mut PyTypeObject) -> *mut c_void; -} -extern "C" { + #[cfg(Py_3_11)] + #[cfg_attr(PyPy, link_name = "PyPyType_GetName")] + pub fn PyType_GetName(arg1: *mut PyTypeObject) -> *mut PyObject; + + #[cfg(Py_3_11)] + #[cfg_attr(PyPy, link_name = "PyPyType_GetQualName")] + pub fn PyType_GetQualName(arg1: *mut PyTypeObject) -> *mut PyObject; + + #[cfg(Py_3_12)] + #[cfg_attr(PyPy, link_name = "PyPyType_FromMetaclass")] + pub fn PyType_FromMetaclass( + metaclass: *mut PyTypeObject, + module: *mut PyObject, + spec: *mut PyType_Spec, + bases: *mut PyObject, + ) -> *mut PyObject; + + #[cfg(Py_3_12)] + #[cfg_attr(PyPy, link_name = "PyPyObject_GetTypeData")] + pub fn PyObject_GetTypeData(obj: *mut PyObject, cls: *mut PyTypeObject) -> *mut c_void; + + #[cfg(Py_3_12)] + #[cfg_attr(PyPy, link_name = "PyPyObject_GetTypeDataSize")] + pub fn PyObject_GetTypeDataSize(cls: *mut PyTypeObject) -> Py_ssize_t; + #[cfg_attr(PyPy, link_name = "PyPyType_IsSubtype")] pub fn PyType_IsSubtype(a: *mut PyTypeObject, b: *mut PyTypeObject) -> c_int; } @@ -246,9 +289,7 @@ extern "C" { extern "C" { pub fn PyType_GetFlags(arg1: *mut PyTypeObject) -> c_ulong; -} -extern "C" { #[cfg_attr(PyPy, link_name = "PyPyType_Ready")] pub fn PyType_Ready(t: *mut PyTypeObject) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyType_GenericAlloc")] @@ -337,6 +378,15 @@ extern "C" { // Flag bits for printing: pub const Py_PRINT_RAW: c_int = 1; // No string quotes etc. +#[cfg(all(Py_3_12, not(Py_LIMITED_API)))] +pub const _Py_TPFLAGS_STATIC_BUILTIN: c_ulong = 1 << 1; + +#[cfg(all(Py_3_12, not(Py_LIMITED_API)))] +pub const Py_TPFLAGS_MANAGED_WEAKREF: c_ulong = 1 << 3; + +#[cfg(all(Py_3_11, not(Py_LIMITED_API)))] +pub const Py_TPFLAGS_MANAGED_DICT: c_ulong = 1 << 4; + #[cfg(all(Py_3_10, not(Py_LIMITED_API)))] pub const Py_TPFLAGS_SEQUENCE: c_ulong = 1 << 5; @@ -356,7 +406,7 @@ pub const Py_TPFLAGS_HEAPTYPE: c_ulong = 1 << 9; pub const Py_TPFLAGS_BASETYPE: c_ulong = 1 << 10; /// Set if the type implements the vectorcall protocol (PEP 590) -#[cfg(all(Py_3_8, not(Py_LIMITED_API)))] +#[cfg(any(Py_3_12, all(Py_3_8, not(Py_LIMITED_API))))] pub const Py_TPFLAGS_HAVE_VECTORCALL: c_ulong = 1 << 11; // skipped non-limited _Py_TPFLAGS_HAVE_VECTORCALL @@ -374,15 +424,14 @@ const Py_TPFLAGS_HAVE_STACKLESS_EXTENSION: c_ulong = 0; #[cfg(Py_3_8)] pub const Py_TPFLAGS_METHOD_DESCRIPTOR: c_ulong = 1 << 17; -/// This flag does nothing in Python 3.10+ -pub const Py_TPFLAGS_HAVE_VERSION_TAG: c_ulong = 1 << 18; - pub const Py_TPFLAGS_VALID_VERSION_TAG: c_ulong = 1 << 19; /* Type is abstract and cannot be instantiated */ pub const Py_TPFLAGS_IS_ABSTRACT: c_ulong = 1 << 20; // skipped non-limited / 3.10 Py_TPFLAGS_HAVE_AM_SEND +#[cfg(Py_3_12)] +pub const Py_TPFLAGS_ITEMS_AT_END: c_ulong = 1 << 23; /* These flags are used to determine if a type is a subclass. */ pub const Py_TPFLAGS_LONG_SUBCLASS: c_ulong = 1 << 24; @@ -394,37 +443,161 @@ pub const Py_TPFLAGS_DICT_SUBCLASS: c_ulong = 1 << 29; pub const Py_TPFLAGS_BASE_EXC_SUBCLASS: c_ulong = 1 << 30; pub const Py_TPFLAGS_TYPE_SUBCLASS: c_ulong = 1 << 31; -pub const Py_TPFLAGS_DEFAULT: c_ulong = - Py_TPFLAGS_HAVE_STACKLESS_EXTENSION | Py_TPFLAGS_HAVE_VERSION_TAG; +pub const Py_TPFLAGS_DEFAULT: c_ulong = if cfg!(Py_3_10) { + Py_TPFLAGS_HAVE_STACKLESS_EXTENSION +} else { + Py_TPFLAGS_HAVE_STACKLESS_EXTENSION | Py_TPFLAGS_HAVE_VERSION_TAG +}; pub const Py_TPFLAGS_HAVE_FINALIZE: c_ulong = 1; +pub const Py_TPFLAGS_HAVE_VERSION_TAG: c_ulong = 1 << 18; -// skipped _Py_RefTotal -// skipped _Py_NegativeRefCount +#[cfg(all(py_sys_config = "Py_REF_DEBUG", not(Py_LIMITED_API)))] +extern "C" { + pub fn _Py_NegativeRefCount(filename: *const c_char, lineno: c_int, op: *mut PyObject); + #[cfg(Py_3_12)] + #[link_name = "_Py_IncRefTotal_DO_NOT_USE_THIS"] + fn _Py_INC_REFTOTAL(); + #[cfg(Py_3_12)] + #[link_name = "_Py_DecRefTotal_DO_NOT_USE_THIS"] + fn _Py_DEC_REFTOTAL(); +} extern "C" { #[cfg_attr(PyPy, link_name = "_PyPy_Dealloc")] pub fn _Py_Dealloc(arg1: *mut PyObject); + + #[cfg_attr(PyPy, link_name = "PyPy_IncRef")] + pub fn Py_IncRef(o: *mut PyObject); + #[cfg_attr(PyPy, link_name = "PyPy_DecRef")] + pub fn Py_DecRef(o: *mut PyObject); + + #[cfg(Py_3_10)] + #[cfg_attr(PyPy, link_name = "_PyPy_IncRef")] + pub fn _Py_IncRef(o: *mut PyObject); + #[cfg(Py_3_10)] + #[cfg_attr(PyPy, link_name = "_PyPy_DecRef")] + pub fn _Py_DecRef(o: *mut PyObject); } -// Reference counting macros. -#[inline] +#[inline(always)] pub unsafe fn Py_INCREF(op: *mut PyObject) { - if cfg!(py_sys_config = "Py_REF_DEBUG") { - Py_IncRef(op) - } else { - (*op).ob_refcnt += 1 + #[cfg(any( + all(Py_LIMITED_API, Py_3_12), + all( + py_sys_config = "Py_REF_DEBUG", + Py_3_10, + not(all(Py_3_12, not(Py_LIMITED_API))) + ) + ))] + { + return _Py_IncRef(op); + } + + #[cfg(all(py_sys_config = "Py_REF_DEBUG", not(Py_3_10)))] + { + return Py_IncRef(op); + } + + #[cfg(any( + not(Py_LIMITED_API), + all(Py_LIMITED_API, not(Py_3_12)), + all(py_sys_config = "Py_REF_DEBUG", Py_3_12, not(Py_LIMITED_API)) + ))] + { + #[cfg(all(Py_3_12, target_pointer_width = "64"))] + { + let cur_refcnt = (*op).ob_refcnt.ob_refcnt_split[crate::PY_BIG_ENDIAN]; + let new_refcnt = cur_refcnt.wrapping_add(1); + if new_refcnt == 0 { + return; + } + (*op).ob_refcnt.ob_refcnt_split[crate::PY_BIG_ENDIAN] = new_refcnt; + } + + #[cfg(all(Py_3_12, target_pointer_width = "32"))] + { + if _Py_IsImmortal(op) != 0 { + return; + } + (*op).ob_refcnt.ob_refcnt += 1 + } + + #[cfg(not(Py_3_12))] + { + (*op).ob_refcnt += 1 + } + + // Skipped _Py_INCREF_STAT_INC - if anyone wants this, please file an issue + // or submit a PR supporting Py_STATS build option and pystats.h + + #[cfg(all(py_sys_config = "Py_REF_DEBUG", Py_3_12))] + _Py_INC_REFTOTAL(); } } -#[inline] +#[inline(always)] +#[cfg_attr( + all(py_sys_config = "Py_REF_DEBUG", Py_3_12, not(Py_LIMITED_API)), + track_caller +)] pub unsafe fn Py_DECREF(op: *mut PyObject) { - if cfg!(py_sys_config = "Py_REF_DEBUG") { - Py_DecRef(op) - } else { - (*op).ob_refcnt -= 1; - if (*op).ob_refcnt == 0 { - _Py_Dealloc(op) + #[cfg(any( + all(Py_LIMITED_API, Py_3_12), + all( + py_sys_config = "Py_REF_DEBUG", + Py_3_10, + not(all(Py_3_12, not(Py_LIMITED_API))) + ) + ))] + { + return _Py_DecRef(op); + } + + #[cfg(all(py_sys_config = "Py_REF_DEBUG", not(Py_3_10)))] + { + return Py_DecRef(op); + } + + #[cfg(any( + not(Py_LIMITED_API), + all(Py_LIMITED_API, not(Py_3_12)), + all(py_sys_config = "Py_REF_DEBUG", Py_3_12, not(Py_LIMITED_API)) + ))] + { + #[cfg(Py_3_12)] + if _Py_IsImmortal(op) != 0 { + return; + } + + // Skipped _Py_DECREF_STAT_INC - if anyone needs this, please file an issue + // or submit a PR supporting Py_STATS build option and pystats.h + + #[cfg(all(py_sys_config = "Py_REF_DEBUG", Py_3_12))] + _Py_DEC_REFTOTAL(); + + #[cfg(Py_3_12)] + { + (*op).ob_refcnt.ob_refcnt -= 1; + + #[cfg(py_sys_config = "Py_REF_DEBUG")] + if (*op).ob_refcnt.ob_refcnt < 0 { + let location = std::panic::Location::caller(); + _Py_NegativeRefcount(location.file(), location.line(), op); + } + + if (*op).ob_refcnt.ob_refcnt == 0 { + _Py_Dealloc(op); + } + } + + #[cfg(not(Py_3_12))] + { + (*op).ob_refcnt -= 1; + + if (*op).ob_refcnt == 0 { + _Py_Dealloc(op); + } } } } @@ -453,14 +626,9 @@ pub unsafe fn Py_XDECREF(op: *mut PyObject) { } extern "C" { - #[cfg_attr(PyPy, link_name = "PyPy_IncRef")] - pub fn Py_IncRef(o: *mut PyObject); - #[cfg_attr(PyPy, link_name = "PyPy_DecRef")] - pub fn Py_DecRef(o: *mut PyObject); - - #[cfg(Py_3_10)] + #[cfg(all(Py_3_10, Py_LIMITED_API))] pub fn Py_NewRef(obj: *mut PyObject) -> *mut PyObject; - #[cfg(Py_3_10)] + #[cfg(all(Py_3_10, Py_LIMITED_API))] pub fn Py_XNewRef(obj: *mut PyObject) -> *mut PyObject; } @@ -480,6 +648,18 @@ pub unsafe fn _Py_XNewRef(obj: *mut PyObject) -> *mut PyObject { obj } +#[cfg(all(Py_3_10, not(Py_LIMITED_API)))] +#[inline] +pub unsafe fn Py_NewRef(obj: *mut PyObject) -> *mut PyObject { + _Py_NewRef(obj) +} + +#[cfg(all(Py_3_10, not(Py_LIMITED_API)))] +#[inline] +pub unsafe fn Py_XNewRef(obj: *mut PyObject) -> *mut PyObject { + _Py_XNewRef(obj) +} + #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { #[cfg_attr(PyPy, link_name = "_PyPy_NoneStruct")] @@ -554,5 +734,5 @@ pub unsafe fn PyType_Check(op: *mut PyObject) -> c_int { #[inline] pub unsafe fn PyType_CheckExact(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == addr_of_mut_shim!(PyType_Type)) as c_int + Py_IS_TYPE(op, addr_of_mut_shim!(PyType_Type)) } diff --git a/pyo3-ffi/src/buffer.rs b/pyo3-ffi/src/pybuffer.rs similarity index 95% rename from pyo3-ffi/src/buffer.rs rename to pyo3-ffi/src/pybuffer.rs index bfa48c086cb..20f92fb6d2b 100644 --- a/pyo3-ffi/src/buffer.rs +++ b/pyo3-ffi/src/pybuffer.rs @@ -53,6 +53,9 @@ impl Py_buffer { } } +pub type getbufferproc = unsafe extern "C" fn(*mut PyObject, *mut crate::Py_buffer, c_int) -> c_int; +pub type releasebufferproc = unsafe extern "C" fn(*mut PyObject, *mut crate::Py_buffer); + /* Return 1 if the getbuffer function is available, otherwise return 0. */ extern "C" { #[cfg(not(PyPy))] diff --git a/pyo3-ffi/src/pyport.rs b/pyo3-ffi/src/pyport.rs index 4bf668dcbf5..741b0db7bf8 100644 --- a/pyo3-ffi/src/pyport.rs +++ b/pyo3-ffi/src/pyport.rs @@ -1,3 +1,9 @@ +pub type PY_UINT32_T = u32; +pub type PY_UINT64_T = u64; + +pub type PY_INT32_T = i32; +pub type PY_INT64_T = i64; + pub type Py_uintptr_t = ::libc::uintptr_t; pub type Py_intptr_t = ::libc::intptr_t; pub type Py_ssize_t = ::libc::ssize_t; @@ -7,3 +13,13 @@ pub type Py_uhash_t = ::libc::size_t; pub const PY_SSIZE_T_MIN: Py_ssize_t = std::isize::MIN as Py_ssize_t; pub const PY_SSIZE_T_MAX: Py_ssize_t = std::isize::MAX as Py_ssize_t; + +#[cfg(target_endian = "big")] +pub const PY_BIG_ENDIAN: usize = 1; +#[cfg(target_endian = "big")] +pub const PY_LITTLE_ENDIAN: usize = 0; + +#[cfg(target_endian = "little")] +pub const PY_BIG_ENDIAN: usize = 0; +#[cfg(target_endian = "little")] +pub const PY_LITTLE_ENDIAN: usize = 1; diff --git a/src/ffi/mod.rs b/src/ffi/mod.rs index 81d6f38e4b7..ce108223fbe 100644 --- a/src/ffi/mod.rs +++ b/src/ffi/mod.rs @@ -21,7 +21,7 @@ //! //! [capi]: https://docs.python.org/3/c-api/index.html -#[cfg(all(not(Py_LIMITED_API), test))] +#[cfg(test)] mod tests; // reexport raw bindings exposed in pyo3_ffi diff --git a/src/ffi/tests.rs b/src/ffi/tests.rs index f9edd8ee3ac..14f76cb4fee 100644 --- a/src/ffi/tests.rs +++ b/src/ffi/tests.rs @@ -1,10 +1,15 @@ use crate::ffi::*; -use crate::{types::PyDict, AsPyPointer, IntoPy, Py, PyAny, Python}; - -use crate::types::PyString; -#[cfg(not(Py_3_12))] +use crate::{AsPyPointer, Python}; + +#[cfg(not(Py_LIMITED_API))] +use crate::{ + types::{PyDict, PyString}, + IntoPy, Py, PyAny, +}; +#[cfg(not(any(Py_3_12, Py_LIMITED_API)))] use libc::wchar_t; +#[cfg(not(Py_LIMITED_API))] #[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons #[test] fn test_datetime_fromtimestamp() { @@ -25,6 +30,7 @@ fn test_datetime_fromtimestamp() { }) } +#[cfg(not(Py_LIMITED_API))] #[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons #[test] fn test_date_fromtimestamp() { @@ -45,6 +51,7 @@ fn test_date_fromtimestamp() { }) } +#[cfg(not(Py_LIMITED_API))] #[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons #[test] fn test_utc_timezone() { @@ -65,6 +72,7 @@ fn test_utc_timezone() { } #[test] +#[cfg(not(Py_LIMITED_API))] #[cfg(feature = "macros")] #[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons fn test_timezone_from_offset() { @@ -82,6 +90,7 @@ fn test_timezone_from_offset() { } #[test] +#[cfg(not(Py_LIMITED_API))] #[cfg(feature = "macros")] #[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons fn test_timezone_from_offset_and_name() { @@ -105,6 +114,7 @@ fn test_timezone_from_offset_and_name() { } #[test] +#[cfg(not(Py_LIMITED_API))] fn ascii_object_bitfield() { let ob_base: PyObject = unsafe { std::mem::zeroed() }; @@ -152,6 +162,7 @@ fn ascii_object_bitfield() { } #[test] +#[cfg(not(Py_LIMITED_API))] #[cfg_attr(Py_3_10, allow(deprecated))] fn ascii() { Python::with_gil(|py| { @@ -193,6 +204,7 @@ fn ascii() { } #[test] +#[cfg(not(Py_LIMITED_API))] #[cfg_attr(Py_3_10, allow(deprecated))] fn ucs4() { Python::with_gil(|py| { @@ -236,6 +248,7 @@ fn ucs4() { } #[test] +#[cfg(not(Py_LIMITED_API))] #[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons #[cfg(not(PyPy))] fn test_get_tzinfo() { @@ -276,3 +289,40 @@ fn test_get_tzinfo() { ); }) } + +#[test] +fn test_inc_dec_ref() { + Python::with_gil(|py| { + let obj = py.eval("object()", None, None).unwrap(); + + let ref_count = obj.get_refcnt(); + let ptr = obj.as_ptr(); + + unsafe { Py_INCREF(ptr) }; + + assert_eq!(obj.get_refcnt(), ref_count + 1); + + unsafe { Py_DECREF(ptr) }; + + assert_eq!(obj.get_refcnt(), ref_count); + }) +} + +#[test] +#[cfg(Py_3_12)] +fn test_inc_dec_ref_immortal() { + Python::with_gil(|py| { + let obj = py.None(); + + let ref_count = obj.get_refcnt(py); + let ptr = obj.as_ptr(); + + unsafe { Py_INCREF(ptr) }; + + assert_eq!(obj.get_refcnt(py), ref_count); + + unsafe { Py_DECREF(ptr) }; + + assert_eq!(obj.get_refcnt(py), ref_count); + }) +} From ecfddd5ab9ac0bf2aa6e199f6a43ff374e0ec546 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Tue, 25 Jul 2023 05:03:00 +0100 Subject: [PATCH 55/57] update tests of refcounting to use a non-immortal object --- src/types/dict.rs | 8 ++++---- src/types/iterator.rs | 25 ++++++++++++------------- src/types/list.rs | 24 ++++++++++++------------ 3 files changed, 28 insertions(+), 29 deletions(-) diff --git a/src/types/dict.rs b/src/types/dict.rs index 74e2d79adb5..80d187c175c 100644 --- a/src/types/dict.rs +++ b/src/types/dict.rs @@ -591,14 +591,14 @@ mod tests { fn test_set_item_refcnt() { Python::with_gil(|py| { let cnt; + let obj = py.eval("object()", None, None).unwrap(); { let _pool = unsafe { crate::GILPool::new() }; - let none = py.None(); - cnt = none.get_refcnt(py); - let _dict = [(10, none)].into_py_dict(py); + cnt = obj.get_refcnt(); + let _dict = [(10, obj)].into_py_dict(py); } { - assert_eq!(cnt, py.None().get_refcnt(py)); + assert_eq!(cnt, obj.get_refcnt()); } }); } diff --git a/src/types/iterator.rs b/src/types/iterator.rs index 71ca5d305b4..7b411bde765 100644 --- a/src/types/iterator.rs +++ b/src/types/iterator.rs @@ -155,31 +155,30 @@ mod tests { #[test] fn iter_item_refcnt() { Python::with_gil(|py| { - let obj; - let none; let count; - { + let obj = py.eval("object()", None, None).unwrap(); + let list = { let _pool = unsafe { GILPool::new() }; - let l = PyList::empty(py); - none = py.None(); - l.append(10).unwrap(); - l.append(&none).unwrap(); - count = none.get_refcnt(py); - obj = l.to_object(py); - } + let list = PyList::empty(py); + list.append(10).unwrap(); + list.append(obj).unwrap(); + count = obj.get_refcnt(); + list.to_object(py) + }; { let _pool = unsafe { GILPool::new() }; - let inst = obj.as_ref(py); + let inst = list.as_ref(py); let mut it = inst.iter().unwrap(); assert_eq!( 10_i32, it.next().unwrap().unwrap().extract::<'_, i32>().unwrap() ); - assert!(it.next().unwrap().unwrap().is_none()); + assert!(it.next().unwrap().unwrap().is(obj)); + assert!(it.next().is_none()); } - assert_eq!(count, none.get_refcnt(py)); + assert_eq!(count, obj.get_refcnt()); }); } diff --git a/src/types/list.rs b/src/types/list.rs index e86086467f9..91de4eb418b 100644 --- a/src/types/list.rs +++ b/src/types/list.rs @@ -395,18 +395,18 @@ mod tests { #[test] fn test_set_item_refcnt() { Python::with_gil(|py| { + let obj = py.eval("object()", None, None).unwrap(); let cnt; { let _pool = unsafe { crate::GILPool::new() }; let v = vec![2]; let ob = v.to_object(py); let list: &PyList = ob.downcast(py).unwrap(); - let none = py.None(); - cnt = none.get_refcnt(py); - list.set_item(0, none).unwrap(); + cnt = obj.get_refcnt(); + list.set_item(0, obj).unwrap(); } - assert_eq!(cnt, py.None().get_refcnt(py)); + assert_eq!(cnt, obj.get_refcnt()); }); } @@ -431,15 +431,15 @@ mod tests { fn test_insert_refcnt() { Python::with_gil(|py| { let cnt; + let obj = py.eval("object()", None, None).unwrap(); { let _pool = unsafe { crate::GILPool::new() }; let list = PyList::empty(py); - let none = py.None(); - cnt = none.get_refcnt(py); - list.insert(0, none).unwrap(); + cnt = obj.get_refcnt(); + list.insert(0, obj).unwrap(); } - assert_eq!(cnt, py.None().get_refcnt(py)); + assert_eq!(cnt, obj.get_refcnt()); }); } @@ -457,14 +457,14 @@ mod tests { fn test_append_refcnt() { Python::with_gil(|py| { let cnt; + let obj = py.eval("object()", None, None).unwrap(); { let _pool = unsafe { crate::GILPool::new() }; let list = PyList::empty(py); - let none = py.None(); - cnt = none.get_refcnt(py); - list.append(none).unwrap(); + cnt = obj.get_refcnt(); + list.append(obj).unwrap(); } - assert_eq!(cnt, py.None().get_refcnt(py)); + assert_eq!(cnt, obj.get_refcnt()); }); } From e8d8840e1514060e4eb1fdeb22d59311528e62ef Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Fri, 28 Jul 2023 16:31:09 +0100 Subject: [PATCH 56/57] fix msrv issues for 0.19.2 patch release --- noxfile.py | 4 ++++ pyo3-ffi/src/cpython/abstract_.rs | 8 ++++---- pyo3-ffi/src/object.rs | 4 ++-- pyo3-macros-backend/src/method.rs | 4 ++-- pytests/src/pyclasses.rs | 1 + src/test_hygiene/pyclass.rs | 2 +- 6 files changed, 14 insertions(+), 9 deletions(-) diff --git a/noxfile.py b/noxfile.py index d16227c36ec..8ab8d9ec545 100644 --- a/noxfile.py +++ b/noxfile.py @@ -499,6 +499,7 @@ def set_minimal_package_versions(session: nox.Session): "trybuild": "1.0.76", # pins to avoid syn 2.0 (which requires Rust 1.56) "ghost": "0.1.8", + "serde_json": "1.0.99", "serde": "1.0.156", "serde_derive": "1.0.156", "cxx": "1.0.92", @@ -508,6 +509,9 @@ def set_minimal_package_versions(session: nox.Session): "js-sys": "0.3.61", "wasm-bindgen": "0.2.84", "syn": "1.0.109", + # proc-macro2 1.0.66+ is edition 2021 + "quote": "1.0.30", + "proc-macro2": "1.0.65", } # run cargo update first to ensure that everything is at highest # possible version, so that this matches what CI will resolve to. diff --git a/pyo3-ffi/src/cpython/abstract_.rs b/pyo3-ffi/src/cpython/abstract_.rs index dd029f3324a..d2e3ca9d67a 100644 --- a/pyo3-ffi/src/cpython/abstract_.rs +++ b/pyo3-ffi/src/cpython/abstract_.rs @@ -41,11 +41,11 @@ extern "C" { ) -> *mut PyObject; } -#[cfg(all(Py_3_8))] +#[cfg(Py_3_8)] const PY_VECTORCALL_ARGUMENTS_OFFSET: Py_ssize_t = 1 << (8 * std::mem::size_of::() as Py_ssize_t - 1); -#[cfg(all(Py_3_8))] +#[cfg(Py_3_8)] #[inline(always)] pub unsafe fn PyVectorcall_NARGS(n: size_t) -> Py_ssize_t { assert!(n <= (PY_SSIZE_T_MAX as size_t)); @@ -113,7 +113,7 @@ extern "C" { kwnames: *mut PyObject, ) -> *mut PyObject; - #[cfg(all(Py_3_8))] + #[cfg(Py_3_8)] #[cfg_attr(all(not(PyPy), not(Py_3_9)), link_name = "_PyObject_VectorcallDict")] #[cfg_attr(all(PyPy, not(Py_3_9)), link_name = "_PyPyObject_VectorcallDict")] #[cfg_attr(all(PyPy, Py_3_9), link_name = "PyPyObject_VectorcallDict")] @@ -124,7 +124,7 @@ extern "C" { kwdict: *mut PyObject, ) -> *mut PyObject; - #[cfg(all(Py_3_8))] + #[cfg(Py_3_8)] #[cfg_attr(not(any(Py_3_9, PyPy)), link_name = "_PyVectorcall_Call")] #[cfg_attr(PyPy, link_name = "PyPyVectorcall_Call")] pub fn PyVectorcall_Call( diff --git a/pyo3-ffi/src/object.rs b/pyo3-ffi/src/object.rs index 6ff8beb5a9e..4c53dc7a441 100644 --- a/pyo3-ffi/src/object.rs +++ b/pyo3-ffi/src/object.rs @@ -111,8 +111,8 @@ pub unsafe fn Py_TYPE(ob: *mut PyObject) -> *mut PyTypeObject { #[inline] pub unsafe fn Py_SIZE(ob: *mut PyObject) -> Py_ssize_t { - debug_assert_ne!((*ob).ob_type, std::ptr::addr_of_mut!(crate::PyLong_Type)); - debug_assert_ne!((*ob).ob_type, std::ptr::addr_of_mut!(crate::PyBool_Type)); + debug_assert_ne!((*ob).ob_type, addr_of_mut_shim!(crate::PyLong_Type)); + debug_assert_ne!((*ob).ob_type, addr_of_mut_shim!(crate::PyBool_Type)); (*ob.cast::()).ob_size } diff --git a/pyo3-macros-backend/src/method.rs b/pyo3-macros-backend/src/method.rs index b290fd55c13..2e2f75d18af 100644 --- a/pyo3-macros-backend/src/method.rs +++ b/pyo3-macros-backend/src/method.rs @@ -254,8 +254,8 @@ pub fn parse_method_receiver(arg: &syn::FnArg) -> Result { ) => { bail_spanned!(recv.span() => RECEIVER_BY_VALUE_ERR); } - syn::FnArg::Receiver(recv @ syn::Receiver { mutability, .. }) => Ok(SelfType::Receiver { - mutable: mutability.is_some(), + syn::FnArg::Receiver(recv) => Ok(SelfType::Receiver { + mutable: recv.mutability.is_some(), span: recv.span(), }), syn::FnArg::Typed(syn::PatType { ty, .. }) => { diff --git a/pytests/src/pyclasses.rs b/pytests/src/pyclasses.rs index 3ee61b343b0..b9c7a5beb48 100644 --- a/pytests/src/pyclasses.rs +++ b/pytests/src/pyclasses.rs @@ -16,6 +16,7 @@ impl EmptyClass { /// This is for demonstrating how to return a value from __next__ #[pyclass] +#[derive(Default)] struct PyClassIter { count: usize, } diff --git a/src/test_hygiene/pyclass.rs b/src/test_hygiene/pyclass.rs index 0b535abe860..4d07009cad6 100644 --- a/src/test_hygiene/pyclass.rs +++ b/src/test_hygiene/pyclass.rs @@ -56,6 +56,6 @@ pub struct Foo4 { field: i32, #[pyo3(get, set)] - #[cfg(any(not(FALSE)))] + #[cfg(not(FALSE))] field: u32, } From 931e23dde6080602acaca00d5ca5884c6109efd3 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Sun, 30 Jul 2023 17:54:44 +0100 Subject: [PATCH 57/57] release: 0.19.2 --- CHANGELOG.md | 33 ++++++++++++++++++- Cargo.toml | 8 ++--- README.md | 4 +-- examples/Cargo.toml | 2 +- examples/decorator/.template/pre-script.rhai | 2 +- .../maturin-starter/.template/pre-script.rhai | 2 +- examples/plugin/.template/pre-script.rhai | 2 +- .../.template/pre-script.rhai | 2 +- examples/word-count/.template/pre-script.rhai | 2 +- newsfragments/3269.fixed.md | 1 - newsfragments/3295.added.md | 1 - newsfragments/3297.added.md | 1 - newsfragments/3297.fixed.md | 1 - newsfragments/3300.fixed.md | 1 - newsfragments/3304.added.md | 1 - newsfragments/3306.changed.md | 1 - newsfragments/3313.fixed.md | 1 - newsfragments/3326.fixed.md | 1 - newsfragments/3328.fixed.md | 1 - newsfragments/3334.added.md | 1 - newsfragments/3335.changed.md | 1 - newsfragments/3335.fixed.md | 1 - newsfragments/3339.added.md | 1 - newsfragments/3342.changed.md | 1 - newsfragments/3345.changed.md | 1 - newsfragments/3346.added.md | 1 - newsfragments/3353.added.md | 1 - pyo3-build-config/Cargo.toml | 2 +- pyo3-ffi/Cargo.toml | 4 +-- pyo3-macros-backend/Cargo.toml | 2 +- pyo3-macros/Cargo.toml | 4 +-- pyproject.toml | 2 +- 32 files changed, 51 insertions(+), 38 deletions(-) delete mode 100644 newsfragments/3269.fixed.md delete mode 100644 newsfragments/3295.added.md delete mode 100644 newsfragments/3297.added.md delete mode 100644 newsfragments/3297.fixed.md delete mode 100644 newsfragments/3300.fixed.md delete mode 100644 newsfragments/3304.added.md delete mode 100644 newsfragments/3306.changed.md delete mode 100644 newsfragments/3313.fixed.md delete mode 100644 newsfragments/3326.fixed.md delete mode 100644 newsfragments/3328.fixed.md delete mode 100644 newsfragments/3334.added.md delete mode 100644 newsfragments/3335.changed.md delete mode 100644 newsfragments/3335.fixed.md delete mode 100644 newsfragments/3339.added.md delete mode 100644 newsfragments/3342.changed.md delete mode 100644 newsfragments/3345.changed.md delete mode 100644 newsfragments/3346.added.md delete mode 100644 newsfragments/3353.added.md diff --git a/CHANGELOG.md b/CHANGELOG.md index fe87c5c3ba7..31a9e023cf6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,36 @@ To see unreleased changes, please see the [CHANGELOG on the main branch guide](h +## [0.19.2] - 2023-08-01 + +### Added + +- Add FFI definitions `PyState_AddModule`, `PyState_RemoveModule` and `PyState_FindModule` for PyPy 3.9 and up. [#3295](https://github.com/PyO3/pyo3/pull/3295) +- Add FFI definitions `_PyObject_CallFunction_SizeT` and `_PyObject_CallMethod_SizeT`. [#3297](https://github.com/PyO3/pyo3/pull/3297) +- Add a "performance" section to the guide collecting performance-related tricks and problems. [#3304](https://github.com/PyO3/pyo3/pull/3304) +- Add `PyErr::Display` for all Python versions, and FFI symbol `PyErr_DisplayException` for Python 3.12. [#3334](https://github.com/PyO3/pyo3/pull/3334) +- Add FFI definition `PyType_GetDict()` for Python 3.12. [#3339](https://github.com/PyO3/pyo3/pull/3339) +- Add `PyAny::downcast_exact`. [#3346](https://github.com/PyO3/pyo3/pull/3346) +- Add `PySlice::full()` to construct a full slice (`::`). [#3353](https://github.com/PyO3/pyo3/pull/3353) + +### Changed + +- Update `PyErr` for 3.12 betas to avoid deprecated ffi methods. [#3306](https://github.com/PyO3/pyo3/pull/3306) +- Update FFI definitions of `object.h` for Python 3.12.0b4. [#3335](https://github.com/PyO3/pyo3/pull/3335) +- Update `pyo3::ffi` struct definitions to be compatible with 3.12.0b4. [#3342](https://github.com/PyO3/pyo3/pull/3342) +- Optimize conversion of `float` to `f64` (and `PyFloat::value`) on non-abi3 builds. [#3345](https://github.com/PyO3/pyo3/pull/3345) + +### Fixed + +- Fix timezone conversion bug for FixedOffset datetimes that were being incorrectly converted to and from UTC. [#3269](https://github.com/PyO3/pyo3/pull/3269) +- Fix `SystemError` raised in `PyUnicodeDecodeError_Create` on PyPy 3.10. [#3297](https://github.com/PyO3/pyo3/pull/3297) +- Correct FFI definition `Py_EnterRecursiveCall` to return `c_int` (was incorrectly returning `()`). [#3300](https://github.com/PyO3/pyo3/pull/3300) +- Fix case where `PyErr::matches` and `PyErr::is_instance` returned results inconsistent with `PyErr::get_type`. [#3313](https://github.com/PyO3/pyo3/pull/3313) +- Fix loss of panic message in `PanicException` when unwinding after the exception was "normalized". [#3326](https://github.com/PyO3/pyo3/pull/3326) +- Fix `PyErr::from_value` and `PyErr::into_value` losing traceback on conversion. [#3328](https://github.com/PyO3/pyo3/pull/3328) +- Fix reference counting of immortal objects on Python 3.12.0b4. [#3335](https://github.com/PyO3/pyo3/pull/3335) + + ## [0.19.1] - 2023-07-03 ### Packaging @@ -1503,7 +1533,8 @@ Yanked - Initial release -[Unreleased]: https://github.com/pyo3/pyo3/compare/v0.19.1...HEAD +[Unreleased]: https://github.com/pyo3/pyo3/compare/v0.19.2...HEAD +[0.19.2]: https://github.com/pyo3/pyo3/compare/v0.19.1...v0.19.2 [0.19.1]: https://github.com/pyo3/pyo3/compare/v0.19.0...v0.19.1 [0.19.0]: https://github.com/pyo3/pyo3/compare/v0.18.3...v0.19.0 [0.18.3]: https://github.com/pyo3/pyo3/compare/v0.18.2...v0.18.3 diff --git a/Cargo.toml b/Cargo.toml index 12990edcdfc..50e04bbd1c8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3" -version = "0.19.1" +version = "0.19.2" description = "Bindings to Python interpreter" authors = ["PyO3 Project and Contributors "] readme = "README.md" @@ -20,10 +20,10 @@ parking_lot = ">= 0.11, < 0.13" memoffset = "0.9" # ffi bindings to the python interpreter, split into a separate crate so they can be used independently -pyo3-ffi = { path = "pyo3-ffi", version = "=0.19.1" } +pyo3-ffi = { path = "pyo3-ffi", version = "=0.19.2" } # support crates for macros feature -pyo3-macros = { path = "pyo3-macros", version = "=0.19.1", optional = true } +pyo3-macros = { path = "pyo3-macros", version = "=0.19.2", optional = true } indoc = { version = "1.0.3", optional = true } unindent = { version = "0.1.4", optional = true } @@ -57,7 +57,7 @@ rust_decimal = { version = "1.8.0", features = ["std"] } widestring = "0.5.1" [build-dependencies] -pyo3-build-config = { path = "pyo3-build-config", version = "0.19.1", features = ["resolve-config"] } +pyo3-build-config = { path = "pyo3-build-config", version = "0.19.2", features = ["resolve-config"] } [features] default = ["macros"] diff --git a/README.md b/README.md index 2dc7b821065..27868be0c2e 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ name = "string_sum" crate-type = ["cdylib"] [dependencies] -pyo3 = { version = "0.19.1", features = ["extension-module"] } +pyo3 = { version = "0.19.2", features = ["extension-module"] } ``` **`src/lib.rs`** @@ -137,7 +137,7 @@ Start a new project with `cargo new` and add `pyo3` to the `Cargo.toml` like th ```toml [dependencies.pyo3] -version = "0.19.1" +version = "0.19.2" features = ["auto-initialize"] ``` diff --git a/examples/Cargo.toml b/examples/Cargo.toml index bdc78d6a034..ab627206ae8 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -5,7 +5,7 @@ publish = false edition = "2018" [dev-dependencies] -pyo3 = { version = "0.19.1", path = "..", features = ["auto-initialize", "extension-module"] } +pyo3 = { version = "0.19.2", path = "..", features = ["auto-initialize", "extension-module"] } [[example]] name = "decorator" diff --git a/examples/decorator/.template/pre-script.rhai b/examples/decorator/.template/pre-script.rhai index 088ea73bfbe..d3341677b1f 100644 --- a/examples/decorator/.template/pre-script.rhai +++ b/examples/decorator/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.19.1"); +variable::set("PYO3_VERSION", "0.19.2"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/examples/maturin-starter/.template/pre-script.rhai b/examples/maturin-starter/.template/pre-script.rhai index 088ea73bfbe..d3341677b1f 100644 --- a/examples/maturin-starter/.template/pre-script.rhai +++ b/examples/maturin-starter/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.19.1"); +variable::set("PYO3_VERSION", "0.19.2"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/examples/plugin/.template/pre-script.rhai b/examples/plugin/.template/pre-script.rhai index 158e1522040..6ab231df25e 100644 --- a/examples/plugin/.template/pre-script.rhai +++ b/examples/plugin/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.19.1"); +variable::set("PYO3_VERSION", "0.19.2"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/plugin_api/Cargo.toml", "plugin_api/Cargo.toml"); file::delete(".template"); diff --git a/examples/setuptools-rust-starter/.template/pre-script.rhai b/examples/setuptools-rust-starter/.template/pre-script.rhai index 19ea7cc8520..b06e0f272b1 100644 --- a/examples/setuptools-rust-starter/.template/pre-script.rhai +++ b/examples/setuptools-rust-starter/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.19.1"); +variable::set("PYO3_VERSION", "0.19.2"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/setup.cfg", "setup.cfg"); file::delete(".template"); diff --git a/examples/word-count/.template/pre-script.rhai b/examples/word-count/.template/pre-script.rhai index a4bfa7ce13b..1349490d477 100644 --- a/examples/word-count/.template/pre-script.rhai +++ b/examples/word-count/.template/pre-script.rhai @@ -1,3 +1,3 @@ -variable::set("PYO3_VERSION", "0.19.1"); +variable::set("PYO3_VERSION", "0.19.2"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::delete(".template"); diff --git a/newsfragments/3269.fixed.md b/newsfragments/3269.fixed.md deleted file mode 100644 index 0f1e1af7b80..00000000000 --- a/newsfragments/3269.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fix timezone conversion bug for FixedOffset datetimes that were being incorrectly converted to and from UTC. diff --git a/newsfragments/3295.added.md b/newsfragments/3295.added.md deleted file mode 100644 index 82987455381..00000000000 --- a/newsfragments/3295.added.md +++ /dev/null @@ -1 +0,0 @@ -Add FFI definitions `PyState_AddModule`, `PyState_RemoveModule` and `PyState_FindModule` for PyPy 3.9 and up. diff --git a/newsfragments/3297.added.md b/newsfragments/3297.added.md deleted file mode 100644 index 65d54832e2d..00000000000 --- a/newsfragments/3297.added.md +++ /dev/null @@ -1 +0,0 @@ -Add FFI definitions `_PyObject_CallFunction_SizeT` and `_PyObject_CallMethod_SizeT`. diff --git a/newsfragments/3297.fixed.md b/newsfragments/3297.fixed.md deleted file mode 100644 index 65e211b42be..00000000000 --- a/newsfragments/3297.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fix `SystemError` raised in `PyUnicodeDecodeError_Create` on PyPy 3.10. diff --git a/newsfragments/3300.fixed.md b/newsfragments/3300.fixed.md deleted file mode 100644 index e7c09ff5d8f..00000000000 --- a/newsfragments/3300.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Correct FFI definition `Py_EnterRecursiveCall` to return `c_int` (was incorrectly returning `()`). diff --git a/newsfragments/3304.added.md b/newsfragments/3304.added.md deleted file mode 100644 index f3b62eba58e..00000000000 --- a/newsfragments/3304.added.md +++ /dev/null @@ -1 +0,0 @@ -Added a "performance" section to the guide collecting performance-related tricks and problems. diff --git a/newsfragments/3306.changed.md b/newsfragments/3306.changed.md deleted file mode 100644 index a2bbc985512..00000000000 --- a/newsfragments/3306.changed.md +++ /dev/null @@ -1 +0,0 @@ -Update `PyErr` for 3.12 betas to avoid deprecated ffi methods. diff --git a/newsfragments/3313.fixed.md b/newsfragments/3313.fixed.md deleted file mode 100644 index e5fc5c0e153..00000000000 --- a/newsfragments/3313.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fix case where `PyErr::matches` and `PyErr::is_instance` returned results inconsistent with `PyErr::get_type`. diff --git a/newsfragments/3326.fixed.md b/newsfragments/3326.fixed.md deleted file mode 100644 index 8353c598f72..00000000000 --- a/newsfragments/3326.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fix loss of panic message in `PanicException` when unwinding after the exception was "normalized". diff --git a/newsfragments/3328.fixed.md b/newsfragments/3328.fixed.md deleted file mode 100644 index 7db0293b4d0..00000000000 --- a/newsfragments/3328.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fixed `PyErr.from_value()` and `PyErr.into_value()` to both maintain original traceback instead of losing it on conversion. \ No newline at end of file diff --git a/newsfragments/3334.added.md b/newsfragments/3334.added.md deleted file mode 100644 index fd3824ad0dc..00000000000 --- a/newsfragments/3334.added.md +++ /dev/null @@ -1 +0,0 @@ -Add `PyErr::Display` for all Python versions, and FFI symbol `PyErr_DisplayException` for Python 3.12. diff --git a/newsfragments/3335.changed.md b/newsfragments/3335.changed.md deleted file mode 100644 index c35f1ddb1f2..00000000000 --- a/newsfragments/3335.changed.md +++ /dev/null @@ -1 +0,0 @@ -Update FFI definitions of `object.h` for Python 3.12 and up. diff --git a/newsfragments/3335.fixed.md b/newsfragments/3335.fixed.md deleted file mode 100644 index bc7bb8b61d8..00000000000 --- a/newsfragments/3335.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fix reference counting of immortal objects on Python 3.12 betas. diff --git a/newsfragments/3339.added.md b/newsfragments/3339.added.md deleted file mode 100644 index 88884bfd841..00000000000 --- a/newsfragments/3339.added.md +++ /dev/null @@ -1 +0,0 @@ -Define `PyType_GetDict()` FFI for CPython 3.12 or later. diff --git a/newsfragments/3342.changed.md b/newsfragments/3342.changed.md deleted file mode 100644 index 71435df06ab..00000000000 --- a/newsfragments/3342.changed.md +++ /dev/null @@ -1 +0,0 @@ -Update `pyo3::ffi` struct definitions to be compatible with 3.12.0b4. diff --git a/newsfragments/3345.changed.md b/newsfragments/3345.changed.md deleted file mode 100644 index 35143e6abba..00000000000 --- a/newsfragments/3345.changed.md +++ /dev/null @@ -1 +0,0 @@ -Optimize conversion of `float` to `f64` (and `PyFloat::value`) on non-abi3 builds. diff --git a/newsfragments/3346.added.md b/newsfragments/3346.added.md deleted file mode 100644 index 7e7085cb6d5..00000000000 --- a/newsfragments/3346.added.md +++ /dev/null @@ -1 +0,0 @@ -Add `PyAny::downcast_exact`. diff --git a/newsfragments/3353.added.md b/newsfragments/3353.added.md deleted file mode 100644 index 2db4834e660..00000000000 --- a/newsfragments/3353.added.md +++ /dev/null @@ -1 +0,0 @@ -Add `PySlice::full()` to construct a full slice (`::`). diff --git a/pyo3-build-config/Cargo.toml b/pyo3-build-config/Cargo.toml index 50445fc938a..4ba6b113880 100644 --- a/pyo3-build-config/Cargo.toml +++ b/pyo3-build-config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-build-config" -version = "0.19.1" +version = "0.19.2" description = "Build configuration for the PyO3 ecosystem" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] diff --git a/pyo3-ffi/Cargo.toml b/pyo3-ffi/Cargo.toml index 4e910d3fa5a..16d1e2dffe3 100644 --- a/pyo3-ffi/Cargo.toml +++ b/pyo3-ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-ffi" -version = "0.19.1" +version = "0.19.2" description = "Python-API bindings for the PyO3 ecosystem" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -38,4 +38,4 @@ generate-import-lib = ["pyo3-build-config/python3-dll-a"] [build-dependencies] -pyo3-build-config = { path = "../pyo3-build-config", version = "0.19.1", features = ["resolve-config"] } +pyo3-build-config = { path = "../pyo3-build-config", version = "0.19.2", features = ["resolve-config"] } diff --git a/pyo3-macros-backend/Cargo.toml b/pyo3-macros-backend/Cargo.toml index 4644fc6e764..96d8b475118 100644 --- a/pyo3-macros-backend/Cargo.toml +++ b/pyo3-macros-backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-macros-backend" -version = "0.19.1" +version = "0.19.2" description = "Code generation for PyO3 package" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] diff --git a/pyo3-macros/Cargo.toml b/pyo3-macros/Cargo.toml index 347b8d78ae5..b4e1aceff0e 100644 --- a/pyo3-macros/Cargo.toml +++ b/pyo3-macros/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-macros" -version = "0.19.1" +version = "0.19.2" description = "Proc macros for PyO3 package" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -22,4 +22,4 @@ abi3 = ["pyo3-macros-backend/abi3"] proc-macro2 = { version = "1", default-features = false } quote = "1" syn = { version = "1.0.85", features = ["full", "extra-traits"] } -pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.19.1" } +pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.19.2" } diff --git a/pyproject.toml b/pyproject.toml index ae99ebf75f1..1892c2aecf5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ exclude = ''' [tool.towncrier] filename = "CHANGELOG.md" -version = "0.19.1" +version = "0.19.2" start_string = "\n" template = ".towncrier.template.md" title_format = "## [{version}] - {project_date}"