fix vectorcall argument handling

Fixes #4093

- Make PY_VECTORCALL_ARGUMENTS_OFFSET a size_t like on CPython
- Change the assert in PyVectorcall_NARGS to check the recovered
  number, not the original value (which is > PY_SSIZE_T_MAX on
  purpose)

newsfragments/4104.fixed.md

@@ -0,0 +1 @@
+Changes definitions of `PY_VECTORCALL_ARGUMENTS_OFFSET` and `PyVectorcall_NARGS` to fix a false-positive assertion.

pyo3-ffi/src/cpython/abstract_.rs

@@ -42,14 +42,15 @@ extern "C" {
 }
 
 #[cfg(Py_3_8)]
-const PY_VECTORCALL_ARGUMENTS_OFFSET: Py_ssize_t =
-    1 << (8 * std::mem::size_of::<Py_ssize_t>() as Py_ssize_t - 1);
+const PY_VECTORCALL_ARGUMENTS_OFFSET: size_t =
+    1 << (8 * std::mem::size_of::<size_t>() as size_t - 1);
 
 #[cfg(Py_3_8)]
 #[inline(always)]
 pub unsafe fn PyVectorcall_NARGS(n: size_t) -> Py_ssize_t {
+    let n = n & !PY_VECTORCALL_ARGUMENTS_OFFSET;
     assert!(n <= (PY_SSIZE_T_MAX as size_t));
-    (n as Py_ssize_t) & !PY_VECTORCALL_ARGUMENTS_OFFSET
+    n as Py_ssize_t
 }
 
 #[cfg(all(Py_3_8, not(any(PyPy, GraalPy))))]
@@ -184,7 +185,7 @@ pub unsafe fn PyObject_CallOneArg(func: *mut PyObject, arg: *mut PyObject) -> *m
     let args = args_array.as_ptr().offset(1); // For PY_VECTORCALL_ARGUMENTS_OFFSET
     let tstate = PyThreadState_GET();
     let nargsf = 1 | PY_VECTORCALL_ARGUMENTS_OFFSET;
-    _PyObject_VectorcallTstate(tstate, func, args, nargsf as size_t, std::ptr::null_mut())
+    _PyObject_VectorcallTstate(tstate, func, args, nargsf, std::ptr::null_mut())
 }
 
 extern "C" {
@@ -206,7 +207,7 @@ pub unsafe fn PyObject_CallMethodNoArgs(
     PyObject_VectorcallMethod(
         name,
         &self_,
-        1 | PY_VECTORCALL_ARGUMENTS_OFFSET as size_t,
+        1 | PY_VECTORCALL_ARGUMENTS_OFFSET,
         std::ptr::null_mut(),
     )
 }
@@ -223,7 +224,7 @@ pub unsafe fn PyObject_CallMethodOneArg(
     PyObject_VectorcallMethod(
         name,
         args.as_ptr(),
-        2 | PY_VECTORCALL_ARGUMENTS_OFFSET as size_t,
+        2 | PY_VECTORCALL_ARGUMENTS_OFFSET,
         std::ptr::null_mut(),
     )
 }