Merge pull request #2364 from messense/import-lib

Add support for generating non-abi3 python import libraries for Windows targets

.github/workflows/ci.yml

@@ -277,11 +277,17 @@ jobs:
         env:
           XWIN_ARCH: x86_64
         run: |
+          set -ex
           sudo apt-get install -y mingw-w64 llvm
           rustup target add x86_64-pc-windows-gnu x86_64-pc-windows-msvc
           python -m pip install cargo-xwin
+          # abi3
           cargo build --manifest-path examples/maturin-starter/Cargo.toml --features abi3 --target x86_64-pc-windows-gnu
           cargo xwin build --manifest-path examples/maturin-starter/Cargo.toml --features abi3 --target x86_64-pc-windows-msvc
+          # non-abi3
+          export PYO3_CROSS_PYTHON_VERSION=3.9
+          cargo build --manifest-path examples/maturin-starter/Cargo.toml --features generate-import-lib --target x86_64-pc-windows-gnu
+          cargo xwin build --manifest-path examples/maturin-starter/Cargo.toml --features generate-import-lib --target x86_64-pc-windows-msvc
       - name: Test cross compile to Windows with maturin
         if: ${{ matrix.platform.os == 'ubuntu-latest' && matrix.python-version == '3.8' }}
         uses: messense/maturin-action@v1

Architecture.md

@@ -193,9 +193,9 @@ Some of the functionality of `pyo3-build-config`:
     `PYO3_CROSS_PYTHON_IMPLEMENTATION`) or system files.
     When cross compiling extension modules it is often possible to make it work without any
     additional user input.
-  - When an experimental feature `generate-abi3-import-lib` is enabled, the `pyo3-ffi` build script can
+  - When an experimental feature `generate-import-lib` is enabled, the `pyo3-ffi` build script can
     generate `python3.dll` import libraries for Windows targets automatically via an external
-    [`python3-dll-a`] crate. This enables the users to cross compile abi3 extensions for Windows without
+    [`python3-dll-a`] crate. This enables the users to cross compile Python extensions for Windows without
     having to install any Windows Python libraries.
 
 <!-- External Links -->

CHANGELOG.md

@@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Implement `ToPyObject` for `[T; N]`. [#2313](https://github.com/PyO3/pyo3/pull/2313)
 - Added the internal `IntoPyResult` trait to give better error messages when function return types do not implement `IntoPy`. [#2326](https://github.com/PyO3/pyo3/pull/2326)
 - Add `PyDictKeys`, `PyDictValues` and `PyDictItems` Rust types to represent `dict_keys`, `dict_values` and `dict_items` types. [#2358](https://github.com/PyO3/pyo3/pull/2358)
+- Add an experimental `generate-import-lib` feature to support auto-generating non-abi3 python import libraries for Windows targets. [#2364](https://github.com/PyO3/pyo3/pull/2364)
 
 ### Changed
 
@@ -21,6 +22,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - The deprecated `pyproto` feature is now disabled by default. [#2322](https://github.com/PyO3/pyo3/pull/2322)
 - Deprecate `ToBorrowedObject` trait (it is only used as a wrapper for `ToPyObject`). [#2333](https://github.com/PyO3/pyo3/pull/2333)
 - `impl<T, const N: usize> IntoPy<PyObject> for [T; N]` now requires `T: IntoPy` rather than `T: ToPyObject`. [#2326](https://github.com/PyO3/pyo3/pull/2326)
+- Deprecate experimental `generate-abi3-import-lib` feature in favor of the new `generate-import-lib` feature. [#2364](https://github.com/PyO3/pyo3/pull/2364)
 
 ### Fixed
 

Cargo.toml

@@ -80,7 +80,9 @@ abi3-py39 = ["abi3-py310", "pyo3-build-config/abi3-py39", "pyo3-ffi/abi3-py39"]
 abi3-py310 = ["abi3", "pyo3-build-config/abi3-py310", "pyo3-ffi/abi3-py310"]
 
 # Automatically generates `python3.dll` import libraries for Windows targets.
-generate-abi3-import-lib = ["pyo3-ffi/generate-abi3-import-lib"]
+generate-import-lib = ["pyo3-ffi/generate-import-lib"]
+# Deprecated, replaced by `generate-import-lib`
+generate-abi3-import-lib = ["generate-import-lib"]
 
 # Changes `Python::with_gil` and `Python::acquire_gil` to automatically initialize the
 # Python interpreter if needed.

examples/maturin-starter/Cargo.toml

@@ -11,6 +11,7 @@ crate-type = ["cdylib"]
 pyo3 = { path = "../../", features = ["extension-module"] }
 
 [features]
-abi3 = ["pyo3/abi3-py37", "pyo3/generate-abi3-import-lib"]
+abi3 = ["pyo3/abi3-py37", "generate-import-lib"]
+generate-import-lib = ["pyo3/generate-import-lib"]
 
 [workspace]

guide/src/building_and_distribution.md

@@ -163,7 +163,7 @@ PyO3 will still attempt to compile `abi3` extension modules after displaying a w
 On Unix-like systems this works unconditionally; on Windows you must also set the `RUSTFLAGS` environment variable
 to contain `-L native=/path/to/python/libs` so that the linker can find `python3.lib`.
 
-If the `python3.dll` import library is not available, an experimental `generate-abi3-import-lib` crate
+If the `python3.dll` import library is not available, an experimental `generate-import-lib` crate
 feature may be enabled, and the required library will be created and used by PyO3 automatically.
 
 *Note*: MSVC targets require LLVM binutils (`llvm-dlltool`) to be available in `PATH` for
@@ -243,12 +243,12 @@ When cross-compiling, PyO3's build script cannot execute the target Python inter
 * `PYO3_CROSS_PYTHON_VERSION`: Major and minor version (e.g. 3.9) of the target Python installation. This variable is only needed if PyO3 cannot determine the version to target from `abi3-py3*` features, or if `PYO3_CROSS_LIB_DIR` is not set, or if there are multiple versions of Python present in `PYO3_CROSS_LIB_DIR`.
 * `PYO3_CROSS_PYTHON_IMPLEMENTATION`: Python implementation name ("CPython" or "PyPy") of the target Python installation. CPython is assumed by default when this variable is not set, unless `PYO3_CROSS_LIB_DIR` is set for a Unix-like target and PyO3 can get the interpreter configuration from `_sysconfigdata*.py`.
 
-An experimental `pyo3` crate feature `generate-abi3-import-lib` enables the user to cross-compile
-"abi3" extension modules for Windows targets without setting the `PYO3_CROSS_LIB_DIR` environment
+An experimental `pyo3` crate feature `generate-import-lib` enables the user to cross-compile
+extension modules for Windows targets without setting the `PYO3_CROSS_LIB_DIR` environment
 variable or providing any Windows Python library files. It uses an external [`python3-dll-a`] crate
-to generate import libraries for the Stable ABI Python DLL for MinGW-w64 and MSVC compile targets.
-*Note*: MSVC targets require LLVM binutils to be available on the host system.
-More specifically, `python3-dll-a` requires `llvm-dlltool` executable to be present in `PATH` when
+to generate import libraries for the Python DLL for MinGW-w64 and MSVC compile targets.
+*Note*: MSVC targets require LLVM binutils or MSVC build tools to be available on the host system.
+More specifically, `python3-dll-a` requires `llvm-dlltool` or `lib.exe` executable to be present in `PATH` when
 targeting `*-pc-windows-msvc`.
 
 An example might look like the following (assuming your target's sysroot is at `/home/pyo3/cross/sysroot` and that your target is `armv7`):
@@ -278,7 +278,7 @@ cargo build --target x86_64-pc-windows-gnu
 Any of the `abi3-py3*` features can be enabled instead of setting `PYO3_CROSS_PYTHON_VERSION` in the above examples.
 
 `PYO3_CROSS_LIB_DIR` can often be omitted when cross compiling extension modules for Unix and macOS targets,
-or when cross compiling "abi3" extension modules for Windows and the experimental `generate-abi3-import-lib`
+or when cross compiling extension modules for Windows and the experimental `generate-import-lib`
 crate feature is enabled.
 
 The following resources may also be useful for cross-compiling:

guide/src/features.md

@@ -30,12 +30,12 @@ These features are extensions of the `abi3` feature to specify the exact minimum
 
 See the [building and distribution](building_and_distribution.md#minimum-python-version-for-abi3) section for further detail.
 
-### `generate-abi3-import-lib`
+### `generate-import-lib`
 
-This experimental feature is used to generate import libraries for the Stable ABI Python DLL
+This experimental feature is used to generate import libraries for Python DLL
 for MinGW-w64 and MSVC (cross-)compile targets.
 
-Enabling it allows to (cross-)compile `abi3` extension modules to any Windows targets
+Enabling it allows to (cross-)compile extension modules to any Windows targets
 without having to install the Windows Python distribution files for the target.
 
 See the [building and distribution](building_and_distribution.md#building-abi3-extensions-without-a-python-interpreter)

pyo3-build-config/Cargo.toml

@@ -12,11 +12,11 @@ edition = "2018"
 
 [dependencies]
 once_cell = "1"
-python3-dll-a = { version = "0.2", optional = true }
+python3-dll-a = { version = "0.2.2", optional = true }
 target-lexicon = "0.12"
 
 [build-dependencies]
-python3-dll-a = { version = "0.2", optional = true }
+python3-dll-a = { version = "0.2.2", optional = true }
 target-lexicon = "0.12"
 
 [features]

pyo3-build-config/src/impl_.rs

@@ -3,8 +3,8 @@
 
 // Optional python3.dll import library generator for Windows
 #[cfg(feature = "python3-dll-a")]
-#[path = "abi3_import_lib.rs"]
-mod abi3_import_lib;
+#[path = "import_lib.rs"]
+mod import_lib;
 
 use std::{
     collections::{HashMap, HashSet},
@@ -1391,13 +1391,11 @@ fn default_cross_compile(cross_compile_config: &CrossCompileConfig) -> Result<In
         .unwrap_or(PythonImplementation::CPython);
 
     let lib_name = if cross_compile_config.target.operating_system == OperatingSystem::Windows {
-        let mingw = cross_compile_config.target.environment == Environment::Gnu;
-
         Some(default_lib_name_windows(
             version,
             implementation,
             abi3,
-            mingw,
+            false,
         ))
     } else if is_linking_libpython_for_target(&cross_compile_config.target) {
         Some(default_lib_name_unix(version, implementation, None))
@@ -1409,8 +1407,9 @@ fn default_cross_compile(cross_compile_config: &CrossCompileConfig) -> Result<In
 
     // Auto generate python3.dll import libraries for Windows targets.
     #[cfg(feature = "python3-dll-a")]
-    if abi3 && lib_dir.is_none() {
-        lib_dir = self::abi3_import_lib::generate_abi3_import_lib(&cross_compile_config.target)?;
+    if lib_dir.is_none() {
+        let py_version = if abi3 { None } else { Some(version) };
+        lib_dir = self::import_lib::generate_import_lib(&cross_compile_config.target, py_version)?;
     }
 
     Ok(InterpreterConfig {
@@ -1723,7 +1722,12 @@ pub fn make_interpreter_config() -> Result<InterpreterConfig> {
     // Auto generate python3.dll import libraries for Windows targets.
     #[cfg(feature = "python3-dll-a")]
     {
-        interpreter_config.lib_dir = self::abi3_import_lib::generate_abi3_import_lib(&host)?;
+        let py_version = if interpreter_config.abi3 {
+            None
+        } else {
+            Some(interpreter_config.version)
+        };
+        interpreter_config.lib_dir = self::import_lib::generate_import_lib(&host, py_version)?;
     }
 
     Ok(interpreter_config)
@@ -2130,7 +2134,7 @@ mod tests {
                 version: PythonVersion { major: 3, minor: 8 },
                 shared: true,
                 abi3: false,
-                lib_name: Some("python3.8".into()),
+                lib_name: Some("python38".into()),
                 lib_dir: Some("/usr/lib/mingw".into()),
                 executable: None,
                 pointer_width: None,

pyo3-build-config/src/abi3_import_lib.rs → pyo3-build-config/src/import_lib.rs

@@ -3,25 +3,28 @@
 use std::env;
 use std::path::PathBuf;
 
-use python3_dll_a::generate_implib_for_target;
+use python3_dll_a::ImportLibraryGenerator;
 
 use crate::errors::{Context, Result};
 
-use super::{Architecture, OperatingSystem, Triple};
+use super::{Architecture, OperatingSystem, PythonVersion, Triple};
 
-/// Generates the `python3.dll` import library for Windows targets.
+/// Generates the `python3.dll` or `pythonXY.dll` import library for Windows targets.
 ///
 /// Places the generated import library into the build script output directory
 /// and returns the full library directory path.
 ///
 /// Does nothing if the target OS is not Windows.
-pub(super) fn generate_abi3_import_lib(target: &Triple) -> Result<Option<String>> {
+pub(super) fn generate_import_lib(
+    target: &Triple,
+    py_version: Option<PythonVersion>,
+) -> Result<Option<String>> {
     if target.operating_system != OperatingSystem::Windows {
         return Ok(None);
     }
 
-    let out_dir = env::var_os("OUT_DIR")
-        .expect("generate_abi3_import_lib() must be called from a build script");
+    let out_dir =
+        env::var_os("OUT_DIR").expect("generate_import_lib() must be called from a build script");
 
     // Put the newly created import library into the build script output directory.
     let mut out_lib_dir = PathBuf::from(out_dir);
@@ -36,7 +39,9 @@ pub(super) fn generate_abi3_import_lib(target: &Triple) -> Result<Option<String>
 
     let env = target.environment.to_string();
 
-    generate_implib_for_target(&out_lib_dir, &arch, &env)
+    ImportLibraryGenerator::new(&arch, &env)
+        .version(py_version.map(|v| (v.major, v.minor)))
+        .generate(&out_lib_dir)
         .context("failed to generate python3.dll import library")?;
 
     let out_lib_dir_string = out_lib_dir

pyo3-ffi/Cargo.toml

@@ -33,7 +33,9 @@ abi3-py39 = ["abi3-py310", "pyo3-build-config/abi3-py39"]
 abi3-py310 = ["abi3", "pyo3-build-config/abi3-py310"]
 
 # Automatically generates `python3.dll` import libraries for Windows targets.
-generate-abi3-import-lib = ["pyo3-build-config/python3-dll-a"]
+generate-import-lib = ["pyo3-build-config/python3-dll-a"]
+# Deprecated, replaced by `generate-import-lib`
+generate-abi3-import-lib = ["generate-import-lib"]
 
 
 [build-dependencies]