Merge pull request #899 from messense/pyo3-config-file

Add support for `PYO3_CONFIG_FILE`

.github/workflows/test.yml

@@ -124,6 +124,12 @@ jobs:
           sudo apt-get install -y mingw-w64
           rustup target add x86_64-pc-windows-gnu
           cargo run -- build --no-sdist -m test-crates/pyo3-pure/Cargo.toml --target x86_64-pc-windows-gnu
+      - name: test compiling with PYO3_CONFIG_FILE
+        shell: bash
+        run: |
+          rustup target add x86_64-unknown-linux-gnu
+          export PYO3_CONFIG_FILE=$(pwd)/test-crates/pyo3-mixed/pyo3-config.txt
+          cargo run -- build --no-sdist -m test-crates/pyo3-mixed/Cargo.toml --target x86_64-unknown-linux-gnu --zig
 
   test-alpine:
     name: Test Alpine Linux

Changelog.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * Re-export `__all__` for pure Rust projects in [#886](https://github.com/PyO3/maturin/pull/886)
 * Stop setting `RUSTFLAGS` environment variable to an empty string in [#887](https://github.com/PyO3/maturin/pull/887)
 * Add hardcoded well-known sysconfigs for effortless cross compiling in [#896](https://github.com/PyO3/maturin/pull/896)
+* Add support for `PYO3_CONFIG_FILE` in [#899](https://github.com/PyO3/maturin/pull/899)
 
 ## [0.12.14] - 2022-04-25
 

src/build_options.rs

@@ -656,7 +656,18 @@ pub fn find_interpreter(
     match bridge {
         BridgeModel::Bindings(binding_name, _) => {
             let mut interpreters = Vec::new();
-            if binding_name.starts_with("pyo3") && target.is_unix() && target.cross_compiling() {
+            if let Some(config_file) = env::var_os("PYO3_CONFIG_FILE") {
+                if !binding_name.starts_with("pyo3") {
+                    bail!("Only pyo3 bindings can be configured with PYO3_CONFIG_FILE");
+                }
+                let interpreter_config =
+                    InterpreterConfig::from_pyo3_config(config_file.as_ref(), target)
+                        .context("Invalid PYO3_CONFIG_FILE")?;
+                interpreters.push(PythonInterpreter::from_config(interpreter_config));
+            } else if binding_name.starts_with("pyo3")
+                && target.is_unix()
+                && target.cross_compiling()
+            {
                 if let Some(cross_lib_dir) = std::env::var_os("PYO3_CROSS_LIB_DIR") {
                     let host_interpreters =
                         find_host_interpreter(bridge, interpreter, target, min_python_minor)?;

src/python_interpreter/config.rs

@@ -1,8 +1,13 @@
 use super::InterpreterKind;
 use crate::target::{Arch, Os};
+use crate::Target;
+use anyhow::{format_err, Context, Result};
+use fs_err as fs;
 use once_cell::sync::Lazy;
 use serde::Deserialize;
 use std::collections::HashMap;
+use std::io::{BufRead, BufReader};
+use std::path::Path;
 
 /// Wellknown Python interpreter sysconfig values
 static WELLKNOWN_SYSCONFIG: Lazy<HashMap<Os, HashMap<Arch, Vec<InterpreterConfig>>>> =
@@ -72,6 +77,123 @@ impl InterpreterConfig {
         None
     }
 
+    /// Construct a new InterpreterConfig from a pyo3 config file
+    pub fn from_pyo3_config(config_file: &Path, target: &Target) -> Result<Self> {
+        let config_file = fs::File::open(config_file)?;
+        let reader = BufReader::new(config_file);
+        let lines = reader.lines();
+
+        macro_rules! parse_value {
+            ($variable:ident, $value:ident) => {
+                $variable = Some($value.trim().parse().context(format!(
+                    concat!(
+                        "failed to parse ",
+                        stringify!($variable),
+                        " from config value '{}'"
+                    ),
+                    $value
+                ))?)
+            };
+        }
+
+        let mut implementation = None;
+        let mut version = None;
+        let mut abiflags = None;
+        let mut ext_suffix = None;
+        let mut abi_tag = None;
+        let mut pointer_width = None;
+
+        for (i, line) in lines.enumerate() {
+            let line = line.context("failed to read line from config")?;
+            let (key, value) = line
+                .split_once('=')
+                .with_context(|| format!("expected key=value pair on line {}", i + 1))?;
+            match key {
+                "implementation" => parse_value!(implementation, value),
+                "version" => parse_value!(version, value),
+                "abiflags" => parse_value!(abiflags, value),
+                "ext_suffix" => parse_value!(ext_suffix, value),
+                "abi_tag" => parse_value!(abi_tag, value),
+                "pointer_width" => parse_value!(pointer_width, value),
+                _ => continue,
+            }
+        }
+        let version: String = version.context("missing value for version")?;
+        let (ver_major, ver_minor) = version
+            .split_once('.')
+            .context("Invalid python interpreter version")?;
+        let major = ver_major.parse::<usize>().with_context(|| {
+            format!(
+                "Invalid python interpreter major version '{}', expect a digit",
+                ver_major
+            )
+        })?;
+        let minor = ver_minor.parse::<usize>().with_context(|| {
+            format!(
+                "Invalid python interpreter minor version '{}', expect a digit",
+                ver_minor
+            )
+        })?;
+        let implementation = implementation.unwrap_or_else(|| "cpython".to_string());
+        let interpreter_kind = implementation.parse().map_err(|e| format_err!("{}", e))?;
+        let abi_tag = match interpreter_kind {
+            InterpreterKind::CPython => {
+                if (major, minor) >= (3, 8) {
+                    abi_tag.unwrap_or_else(|| format!("{}{}", major, minor))
+                } else {
+                    abi_tag.unwrap_or_else(|| format!("{}{}m", major, minor))
+                }
+            }
+            InterpreterKind::PyPy => abi_tag.unwrap_or_else(|| "pp73".to_string()),
+        };
+        let file_ext = if target.is_windows() { "pyd" } else { "so" };
+        let ext_suffix = if target.is_linux() || target.is_macos() {
+            // See https://github.com/pypa/auditwheel/issues/349
+            let target_env = if (major, minor) >= (3, 11) {
+                target.target_env().to_string()
+            } else {
+                "gnu".to_string()
+            };
+            match interpreter_kind {
+                InterpreterKind::CPython => ext_suffix.unwrap_or_else(|| {
+                    // Eg: .cpython-38-x86_64-linux-gnu.so
+                    format!(
+                        ".cpython-{}-{}-{}-{}.{}",
+                        abi_tag,
+                        target.get_python_arch(),
+                        target.get_python_os(),
+                        target_env,
+                        file_ext,
+                    )
+                }),
+                InterpreterKind::PyPy => ext_suffix.unwrap_or_else(|| {
+                    // Eg: .pypy38-pp73-x86_64-linux-gnu.so
+                    format!(
+                        ".pypy{}{}-{}-{}-{}-{}.{}",
+                        major,
+                        minor,
+                        abi_tag,
+                        target.get_python_arch(),
+                        target.get_python_os(),
+                        target_env,
+                        file_ext,
+                    )
+                }),
+            }
+        } else {
+            ext_suffix.context("missing value for ext_suffix")?
+        };
+        Ok(Self {
+            major,
+            minor,
+            interpreter_kind,
+            abiflags: abiflags.unwrap_or_default(),
+            ext_suffix,
+            abi_tag: Some(abi_tag),
+            pointer_width,
+        })
+    }
+
     /// Generate pyo3 config file content
     pub fn pyo3_config_file(&self) -> String {
         let mut content = format!(

src/python_interpreter/mod.rs

@@ -10,7 +10,7 @@ use std::io;
 use std::ops::Deref;
 use std::path::{Path, PathBuf};
 use std::process::{Command, Stdio};
-use std::str;
+use std::str::{self, FromStr};
 
 mod config;
 
@@ -248,6 +248,18 @@ impl fmt::Display for InterpreterKind {
     }
 }
 
+impl FromStr for InterpreterKind {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s.to_ascii_lowercase().as_str() {
+            "cpython" => Ok(InterpreterKind::CPython),
+            "pypy" => Ok(InterpreterKind::PyPy),
+            unknown => Err(format!("Unknown interpreter kind '{}'", unknown)),
+        }
+    }
+}
+
 /// The output format of [GET_INTERPRETER_METADATA]
 #[derive(Deserialize)]
 struct IntepreterMetadataMessage {

src/target.rs

@@ -296,6 +296,19 @@ impl Target {
         Ok(tag)
     }
 
+    /// Returns the name python uses in `sys.platform` for this architecture.
+    pub fn get_python_arch(&self) -> &str {
+        match self.arch {
+            Arch::Aarch64 => "aarch64",
+            Arch::Armv7L => "armv7l",
+            Arch::Powerpc64Le => "powerpc64le",
+            Arch::Powerpc64 => "powerpc64",
+            Arch::X86 => "i386",
+            Arch::X86_64 => "x86_64",
+            Arch::S390X => "s390x",
+        }
+    }
+
     /// Returns the name python uses in `sys.platform` for this os
     pub fn get_python_os(&self) -> &str {
         match self.os {
@@ -364,6 +377,11 @@ impl Target {
         self.arch
     }
 
+    /// Returns target environment
+    pub fn target_env(&self) -> Environment {
+        self.env
+    }
+
     /// Returns true if the current platform is linux
     pub fn is_linux(&self) -> bool {
         self.os == Os::Linux

test-crates/pyo3-mixed/pyo3-config.txt

@@ -0,0 +1,6 @@
+implementation=CPython
+version=3.10
+shared=true
+abi3=false
+suppress_build_script_link_lines=false
+pointer_width=64