Use stable Python C API for building Rust extension (Qiskit#10120)

* Use stable Python C API for building Rust extension

This commit tweaks the rust extension code to start using the PyO3 abi3
flag to build binaries that are compatible with all python versions, not
just a single release. Previously, we were building against the version
specific C API and that resulted in needing abinary file for each
supported python version on each supported platform/architecture. By
using the abi3 feature flag and marking the wheels as being built with
the limited api we can reduce our packaging overhead to just having one
wheel file per supported platform/architecture.

The only real code change needed here was to update the memory
marginalization function. PyO3's abi3 feature is incompatible with
returning a big int object from rust (the C API they use for that
conversion isn't part of the stable C API). So this commit updates the
function to convert to create a python int manually using the PyO3 api
where that was being done before.

Co-authored-by: Jake Lishman <[email protected]>

* Set minimum version on abi3 flag to Python 3.8

* Fix lint

* Use py_limited_api="auto" on RustExtension

According to the docs for the setuptools-rust RustExtension class:
https://setuptools-rust.readthedocs.io/en/latest/reference.html#setuptools_rust.RustExtension
The best setting to use for the py_limited_api argument is `"auto"` as
this will use the setting in the PyO3 module to determine the correct
value to set. This commit updates the setup.py to follow the
recommendation in the docs.

* Update handling of phase input to expval rust calls

The pauli_expval module in Rust that Statevector and DensityMatrix
leverage when computing defines the input type of the phase argument as
Complex64. Previously, the quantum info code in the Statevector and
DensityMatrix classes were passing in a 1 element ndarray for this
parameter. When using the the version specific Python C API in pyo3 it
would convert the single element array to a scalar value. However when
using abi3 this handling was not done (or was not done correctly) and
this caused the tests to fail. This commit updates the quantum info
module to pass the phase as a complex value instead of a 1 element numpy
array to bypass this behavior change in PyO3 when using abi3.

Co-authored-by: Jake Lishman <[email protected]>

* Set py_limited_api explicitly to True

* DNM: Test cibuildwheel works with abi3

* Add abi3audit to cibuildwheel repair step

* Force setuptools to use abi3 tag

* Add wheel to sdist build

* Workaround abiaudit3 not moving wheels and windows not having a default repair command

* Add source of setup.py hack

* Add comment about pending pyo3 abi3 bigint support

* Revert "DNM: Test cibuildwheel works with abi3"

This reverts commit 8ca24cf.

* Add release note

* Simplify setting abi3 tag in built wheels

* Update releasenotes/notes/use-abi3-4a935e0557d3833b.yaml

Co-authored-by: Jake Lishman <[email protected]>

* Update release note

* Update releasenotes/notes/use-abi3-4a935e0557d3833b.yaml

---------

Co-authored-by: Jake Lishman <[email protected]>
Co-authored-by: Jake Lishman <[email protected]>

.azure/wheels.yml

@@ -25,7 +25,7 @@ jobs:
       - bash: |
           set -e
           python -m pip install --upgrade pip
-          python -m pip install cibuildwheel==2.11.2
+          python -m pip install cibuildwheel==2.13.0
           python -m pip install -U twine
           cibuildwheel --output-dir wheelhouse .
 

.github/workflows/wheels.yml

@@ -24,7 +24,7 @@ jobs:
         with:
           platforms: all
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.11.2
+        uses: pypa/cibuildwheel@v2.13.0
         env:
           CIBW_ARCHS_LINUX: s390x
           CIBW_TEST_SKIP: "cp*"
@@ -57,7 +57,7 @@ jobs:
         with:
           platforms: all
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.11.2
+        uses: pypa/cibuildwheel@v2.13.0
         env:
           CIBW_ARCHS_LINUX: ppc64le
           CIBW_TEST_SKIP: "cp*"
@@ -90,7 +90,7 @@ jobs:
         with:
           platforms: all
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.11.2
+        uses: pypa/cibuildwheel@v2.13.0
         env:
           CIBW_ARCHS_LINUX: aarch64
       - uses: actions/upload-artifact@v3

crates/accelerate/Cargo.toml

@@ -26,7 +26,7 @@ rustworkx-core = "0.13"
 # can be done in the workspace and inherited once we hit Rust 1.64.
 [dependencies.pyo3]
 version = "0.19.0"
-features = ["extension-module", "hashbrown", "indexmap", "num-complex", "num-bigint"]
+features = ["extension-module", "hashbrown", "indexmap", "num-complex", "num-bigint", "abi3-py38"]
 
 [dependencies.ndarray]
 version = "^0.15.6"

crates/accelerate/src/results/marginalization.rs

@@ -152,19 +152,24 @@ pub fn marginal_memory(
             .collect()
     };
     if return_int {
-        if out_mem.len() < parallel_threshold || !run_in_parallel {
-            Ok(out_mem
-                .iter()
-                .map(|x| BigUint::parse_bytes(x.as_bytes(), 2).unwrap())
-                .collect::<Vec<BigUint>>()
-                .to_object(py))
-        } else {
-            Ok(out_mem
-                .par_iter()
-                .map(|x| BigUint::parse_bytes(x.as_bytes(), 2).unwrap())
-                .collect::<Vec<BigUint>>()
-                .to_object(py))
-        }
+        // Replace with:
+        //
+        // .iter()
+        // .map(|x| BigUint::parse_bytes(x.as_bytes(), 2).unwrap())
+        // .collect::<Vec<BigUint>>()
+        // .to_object(py))
+        //
+        // (also this can be done in parallel, see
+        // https://github.com/Qiskit/qiskit-terra/pull/10120 for more
+        // details)
+        //
+        // After PyO3/pyo3#3198 is included in a pyo3 release.
+        let int_pyobject = py.import("builtins")?.getattr("int")?;
+        Ok(out_mem
+            .iter()
+            .map(|x| int_pyobject.call1((x, 2u8)).unwrap())
+            .collect::<Vec<_>>()
+            .to_object(py))
     } else {
         Ok(out_mem.to_object(py))
     }

crates/qasm2/Cargo.toml

@@ -13,4 +13,4 @@ crate-type = ["cdylib"]
 
 [dependencies]
 hashbrown = "0.13.2"
-pyo3 = { version = "0.19.0", features = ["extension-module"] }
+pyo3 = { version = "0.19.0", features = ["extension-module", "abi3-py38"] }

pyproject.toml

@@ -22,6 +22,13 @@ environment = 'RUSTUP_TOOLCHAIN="stable"'
 [tool.cibuildwheel.linux]
 before-all = "yum install -y wget && {package}/tools/install_rust.sh"
 environment = 'PATH="$PATH:$HOME/.cargo/bin" CARGO_NET_GIT_FETCH_WITH_CLI="true" RUSTUP_TOOLCHAIN="stable"'
+repair-wheel-command = "auditwheel repair -w {dest_dir} {wheel} && pipx run abi3audit --strict --report {wheel}"
+
+[tool.cibuildwheel.macos]
+repair-wheel-command = "delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel} && pipx run abi3audit --strict --report {wheel}"
+
+[tool.cibuildwheel.windows]
+repair-wheel-command = "cp {wheel} {dest_dir}/. && pipx run abi3audit --strict --report {wheel}"
 
 [tool.ruff]
 select = [

qiskit/quantum_info/states/densitymatrix.py

@@ -382,6 +382,7 @@ def _expectation_value_pauli(self, pauli, qargs=None):
 
         x_max = qubits[pauli.x][-1]
         y_phase = (-1j) ** pauli._count_y()
+        y_phase = y_phase[0]
         return pauli_phase * density_expval_pauli_with_x(
             data, self.num_qubits, z_mask, x_mask, y_phase, x_max
         )

qiskit/quantum_info/states/statevector.py

@@ -476,6 +476,7 @@ def _expectation_value_pauli(self, pauli, qargs=None):
 
         x_max = qubits[pauli.x][-1]
         y_phase = (-1j) ** pauli._count_y()
+        y_phase = y_phase[0]
 
         return pauli_phase * expval_pauli_with_x(
             self.data, self.num_qubits, z_mask, x_mask, y_phase, x_max

releasenotes/notes/use-abi3-4a935e0557d3833b.yaml

@@ -0,0 +1,17 @@
+---
+upgrade:
+  - |
+    By default Qiskit builds its compiled extensions using the
+    `Python Stable ABI <https://docs.python.org/3/c-api/stable.html>`__
+    with support back to the oldest version of Python supported by Qiskit
+    (currently 3.8). This means that moving forward there
+    will be a single precompiled wheels that are shipped on release that
+    works with all of Qiskit's supported Python versions. There isn't any
+    expected runtime performance difference using the limited API so it is
+    enabled by default for all builds now.
+    Previously, the compiled extensions were built using the version specific API and
+    would only work with a single Python version. This change was made
+    to reduce the number of package files we need to build and publish in each
+    release. When building Qiskit from source there should be no changes
+    necessary to the build process except that the default tags in the output
+    filenames will be different to reflect the use of the limited API.

setup.py

@@ -110,9 +110,13 @@
             debug=rust_debug,
         ),
         RustExtension(
-            "qiskit._qasm2", "crates/qasm2/Cargo.toml", binding=Binding.PyO3, debug=rust_debug
+            "qiskit._qasm2",
+            "crates/qasm2/Cargo.toml",
+            binding=Binding.PyO3,
+            debug=rust_debug,
         ),
     ],
+    options={"bdist_wheel": {"py_limited_api": "cp38"}},
     zip_safe=False,
     entry_points={
         "qiskit.unitary_synthesis": [