Better cross compiling support for PyO3 binding on Unix

.github/workflows/test.yml

@@ -143,3 +143,24 @@ jobs:
 
       # Fix permissions from docker for caching
       - run: sudo chown $(id -u):$(id -g) -R target test-crates/*/target
+
+  test-cross-compile:
+    name: Test Cross Compile
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        platform: [
+          { target: "aarch64-unknown-linux-gnu", arch: "aarch64" },
+          { target: "armv7-unknown-linux-gnueabihf", arch: "armv7" },
+        ]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Build Wheels
+        run: |
+          echo 'curl -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable --profile minimal
+          source ~/.cargo/env
+          rustup target add ${{ matrix.platform.target }}
+          export PYO3_CROSS_LIB_DIR=/opt/python/cp36-cp36m/lib
+          cargo run --target x86_64-unknown-linux-gnu -- build -i python3.9 --release --out dist --no-sdist --target ${{ matrix.platform.target }} -m test-crates/pyo3-mixed/Cargo.toml
+          ' > build-wheel.sh
+          docker run --rm -v "$PWD":/io -w /io messense/manylinux2014-cross:${{ matrix.platform.arch }} bash build-wheel.sh

src/build_options.rs

@@ -1,5 +1,6 @@
 use crate::auditwheel::Manylinux;
 use crate::build_context::{BridgeModel, ProjectLayout};
+use crate::cross_compile::{find_sysconfigdata, is_cross_compiling, parse_sysconfigdata};
 use crate::python_interpreter::InterpreterKind;
 use crate::BuildContext;
 use crate::CargoToml;
@@ -420,8 +421,8 @@ pub fn find_interpreter(
     min_python_minor: Option<usize>,
 ) -> Result<Vec<PythonInterpreter>> {
     match bridge {
-        BridgeModel::Bindings(_) => {
-            let interpreter = if !interpreter.is_empty() {
+        BridgeModel::Bindings(binding_name) => {
+            let mut interpreter = if !interpreter.is_empty() {
                 PythonInterpreter::check_executables(&interpreter, &target, &bridge)
                     .context("The given list of python interpreters is invalid")?
             } else {
@@ -433,6 +434,56 @@ pub fn find_interpreter(
                 bail!("Couldn't find any python interpreters. Please specify at least one with -i");
             }
 
+            if binding_name == "pyo3" && target.is_unix() && is_cross_compiling(target)? {
+                if let Some(cross_lib_dir) = std::env::var_os("PYO3_CROSS_LIB_DIR") {
+                    println!("⚠ Cross-compiling is poorly supported");
+                    let host_python = &interpreter[0];
+                    println!(
+                        "🐍 Using host {} for cross-compiling preparation",
+                        host_python
+                    );
+                    // pyo3
+                    env::set_var("PYO3_PYTHON", &host_python.executable);
+                    // rust-cpython, and legacy pyo3 versions
+                    env::set_var("PYTHON_SYS_EXECUTABLE", &host_python.executable);
+
+                    let sysconfig_path = find_sysconfigdata(cross_lib_dir.as_ref())?;
+                    let sysconfig_data =
+                        parse_sysconfigdata(&host_python.executable, sysconfig_path)?;
+                    let major = sysconfig_data
+                        .get("version_major")
+                        .context("version_major is not defined")?
+                        .parse::<usize>()
+                        .context("Could not parse value of version_major")?;
+                    let minor = sysconfig_data
+                        .get("version_minor")
+                        .context("version_minor is not defined")?
+                        .parse::<usize>()
+                        .context("Could not parse value of version_minor")?;
+                    let abiflags = sysconfig_data
+                        .get("ABIFLAGS")
+                        .map(ToString::to_string)
+                        .unwrap_or_default();
+                    let ext_suffix = sysconfig_data
+                        .get("EXT_SUFFIX")
+                        .context("syconfig didn't define an `EXT_SUFFIX` ಠ_ಠ")?;
+                    let abi_tag = sysconfig_data
+                        .get("SOABI")
+                        .and_then(|abi| abi.split('-').nth(1).map(ToString::to_string));
+                    interpreter = vec![PythonInterpreter {
+                        major,
+                        minor,
+                        abiflags,
+                        target: target.clone(),
+                        executable: PathBuf::new(),
+                        ext_suffix: ext_suffix.to_string(),
+                        interpreter_kind: InterpreterKind::CPython,
+                        abi_tag,
+                        libs_dir: PathBuf::from(cross_lib_dir),
+                    }];
+                }
+            }
+
             println!(
                 "🐍 Found {}",
                 interpreter
@@ -525,7 +576,6 @@ fn extract_cargo_metadata_args(cargo_extra_args: &[String]) -> Result<Vec<String
         ("--all-features", false),
         ("--no-default-features", false),
     ];
-
     let mut cargo_metadata_extra_args = vec![];
     let mut args_iter = cargo_extra_args.iter();
     // We do manual iteration so we can take and skip the value of an option that is in the next

src/compile.rs

@@ -213,12 +213,15 @@ fn compile_target(
     }
 
     if let Some(python_interpreter) = python_interpreter {
-        if bindings_crate.is_bindings("pyo3") {
-            build_command.env("PYO3_PYTHON", &python_interpreter.executable);
-        }
+        // `python_interpreter.executable` could be empty when cross compiling
+        if python_interpreter.executable != PathBuf::new() {
+            if bindings_crate.is_bindings("pyo3") {
+                build_command.env("PYO3_PYTHON", &python_interpreter.executable);
+            }
 
-        // rust-cpython, and legacy pyo3 versions
-        build_command.env("PYTHON_SYS_EXECUTABLE", &python_interpreter.executable);
+            // rust-cpython, and legacy pyo3 versions
+            build_command.env("PYTHON_SYS_EXECUTABLE", &python_interpreter.executable);
+        }
     }
 
     let mut cargo_build = build_command.spawn().context("Failed to run cargo")?;

src/cross_compile.rs

@@ -0,0 +1,220 @@
+use crate::Target;
+use anyhow::{bail, format_err, Result};
+use fs_err::{self as fs, DirEntry};
+use std::collections::HashMap;
+use std::env;
+use std::io;
+use std::path::{Path, PathBuf};
+use std::process::{Command, Stdio};
+
+pub fn is_cross_compiling(target: &Target) -> Result<bool> {
+    let target_triple = target.target_triple();
+    let host = platforms::Platform::guess_current()
+        .map(|platform| platform.target_triple)
+        .ok_or_else(|| format_err!("Couldn't guess the current host platform"))?;
+    if target_triple == host {
+        // Not cross-compiling
+        return Ok(false);
+    }
+
+    if target_triple == "x86_64-apple-darwin" && host == "aarch64-apple-darwin" {
+        // Not cross-compiling to compile for x86-64 Python from macOS arm64
+        return Ok(false);
+    }
+    if target_triple == "aarch64-apple-darwin" && host == "x86_64-apple-darwin" {
+        // Not cross-compiling to compile for arm64 Python from macOS x86_64
+        return Ok(false);
+    }
+
+    if let Some(target_without_env) = target_triple
+        .rfind('-')
+        .map(|index| &target_triple[0..index])
+    {
+        if host.starts_with(target_without_env) {
+            // Not cross-compiling if arch-vendor-os is all the same
+            // e.g. x86_64-unknown-linux-musl on x86_64-unknown-linux-gnu host
+            return Ok(false);
+        }
+    }
+
+    Ok(true)
+}
+
+/// Parse sysconfigdata file
+///
+/// The sysconfigdata is simply a dictionary containing all the build time variables used for the
+/// python executable and library. Here it is read and added to a script to extract only what is
+/// necessary. This necessitates a python interpreter for the host machine to work.
+pub fn parse_sysconfigdata(
+    interpreter: &Path,
+    config_path: impl AsRef<Path>,
+) -> Result<HashMap<String, String>> {
+    let mut script = fs::read_to_string(config_path)?;
+    script += r#"
+print("version_major", build_time_vars["VERSION"][0])  # 3
+print("version_minor", build_time_vars["VERSION"][2])  # E.g., 8
+KEYS = [
+    "ABIFLAGS",
+    "EXT_SUFFIX",
+    "SOABI",
+]
+for key in KEYS:
+    print(key, build_time_vars.get(key, ""))
+"#;
+    let output = run_python_script(interpreter, &script)?;
+
+    Ok(parse_script_output(&output))
+}
+
+fn parse_script_output(output: &str) -> HashMap<String, String> {
+    output
+        .lines()
+        .filter_map(|line| {
+            let mut i = line.splitn(2, ' ');
+            Some((i.next()?.into(), i.next()?.into()))
+        })
+        .collect()
+}
+
+/// Run a python script using the specified interpreter binary.
+fn run_python_script(interpreter: &Path, script: &str) -> Result<String> {
+    let out = Command::new(interpreter)
+        .env("PYTHONIOENCODING", "utf-8")
+        .stdin(Stdio::piped())
+        .stdout(Stdio::piped())
+        .stderr(Stdio::inherit())
+        .spawn()
+        .and_then(|mut child| {
+            use std::io::Write;
+            child
+                .stdin
+                .as_mut()
+                .expect("piped stdin")
+                .write_all(script.as_bytes())?;
+            child.wait_with_output()
+        });
+
+    match out {
+        Err(err) => {
+            if err.kind() == io::ErrorKind::NotFound {
+                bail!(
+                    "Could not find any interpreter at {}, \
+                     are you sure you have Python installed on your PATH?",
+                    interpreter.display()
+                );
+            } else {
+                bail!(
+                    "Failed to run the Python interpreter at {}: {}",
+                    interpreter.display(),
+                    err
+                );
+            }
+        }
+        Ok(ok) if !ok.status.success() => bail!("Python script failed"),
+        Ok(ok) => Ok(String::from_utf8(ok.stdout)?),
+    }
+}
+
+fn starts_with(entry: &DirEntry, pat: &str) -> bool {
+    let name = entry.file_name();
+    name.to_string_lossy().starts_with(pat)
+}
+fn ends_with(entry: &DirEntry, pat: &str) -> bool {
+    let name = entry.file_name();
+    name.to_string_lossy().ends_with(pat)
+}
+
+/// Finds the `_sysconfigdata*.py` file in the library path.
+///
+/// From the python source for `_sysconfigdata*.py` is always going to be located at
+/// `build/lib.{PLATFORM}-{PY_MINOR_VERSION}` when built from source. The [exact line][1] is defined as:
+///
+/// ```py
+/// pybuilddir = 'build/lib.%s-%s' % (get_platform(), sys.version_info[:2])
+/// ```
+///
+/// Where get_platform returns a kebab-case formated string containing the os, the architecture and
+/// possibly the os' kernel version (not the case on linux). However, when installed using a package
+/// manager, the `_sysconfigdata*.py` file is installed in the `${PREFIX}/lib/python3.Y/` directory.
+/// The `_sysconfigdata*.py` is generally in a sub-directory of the location of `libpython3.Y.so`.
+/// So we must find the file in the following possible locations:
+///
+/// ```sh
+/// # distribution from package manager, lib_dir should include lib/
+/// ${INSTALL_PREFIX}/lib/python3.Y/_sysconfigdata*.py
+/// ${INSTALL_PREFIX}/lib/libpython3.Y.so
+/// ${INSTALL_PREFIX}/lib/python3.Y/config-3.Y-${HOST_TRIPLE}/libpython3.Y.so
+///
+/// # Built from source from host
+/// ${CROSS_COMPILED_LOCATION}/build/lib.linux-x86_64-Y/_sysconfigdata*.py
+/// ${CROSS_COMPILED_LOCATION}/libpython3.Y.so
+///
+/// # if cross compiled, kernel release is only present on certain OS targets.
+/// ${CROSS_COMPILED_LOCATION}/build/lib.{OS}(-{OS-KERNEL-RELEASE})?-{ARCH}-Y/_sysconfigdata*.py
+/// ${CROSS_COMPILED_LOCATION}/libpython3.Y.so
+/// ```
+///
+/// [1]: https://github.com/python/cpython/blob/3.5/Lib/sysconfig.py#L389
+pub fn find_sysconfigdata(lib_dir: &Path) -> Result<PathBuf> {
+    let sysconfig_paths = search_lib_dir(lib_dir);
+    let mut sysconfig_paths = sysconfig_paths
+        .iter()
+        .filter_map(|p| fs::canonicalize(p).ok())
+        .collect::<Vec<PathBuf>>();
+    sysconfig_paths.dedup();
+    if sysconfig_paths.is_empty() {
+        bail!(
+            "Could not find either libpython.so or _sysconfigdata*.py in {}",
+            lib_dir.display()
+        );
+    } else if sysconfig_paths.len() > 1 {
+        bail!(
+            "Detected multiple possible python versions, please set the PYO3_PYTHON_VERSION \
+            variable to the wanted version on your system\nsysconfigdata paths = {:?}",
+            sysconfig_paths
+        )
+    }
+
+    Ok(sysconfig_paths.remove(0))
+}
+
+/// recursive search for _sysconfigdata, returns all possibilities of sysconfigdata paths
+fn search_lib_dir(path: impl AsRef<Path>) -> Vec<PathBuf> {
+    let mut sysconfig_paths = vec![];
+    let version_pat = if let Some(v) =
+        env::var_os("PYO3_CROSS_PYTHON_VERSION").map(|s| s.into_string().unwrap())
+    {
+        format!("python{}", v)
+    } else {
+        "python3.".into()
+    };
+    for f in fs::read_dir(path.as_ref()).expect("Path does not exist") {
+        let sysc = match &f {
+            Ok(f) if starts_with(f, "_sysconfigdata") && ends_with(f, "py") => vec![f.path()],
+            Ok(f) if starts_with(f, "build") => search_lib_dir(f.path()),
+            Ok(f) if starts_with(f, "lib.") => {
+                let name = f.file_name();
+                // check if right target os
+                let os = env::var("CARGO_CFG_TARGET_OS").unwrap();
+                if !name
+                    .to_string_lossy()
+                    .contains(if os == "android" { "linux" } else { &os })
+                {
+                    continue;
+                }
+                // Check if right arch
+                if !name
+                    .to_string_lossy()
+                    .contains(&env::var("CARGO_CFG_TARGET_ARCH").unwrap())
+                {
+                    continue;
+                }
+                search_lib_dir(f.path())
+            }
+            Ok(f) if starts_with(f, &version_pat) => search_lib_dir(f.path()),
+            _ => continue,
+        };
+        sysconfig_paths.extend(sysc);
+    }
+    sysconfig_paths
+}

src/lib.rs

@@ -55,6 +55,7 @@ mod build_context;
 mod build_options;
 mod cargo_toml;
 mod compile;
+mod cross_compile;
 mod develop;
 mod metadata;
 mod module_writer;