diff --git a/src/build_options.rs b/src/build_options.rs index 16b2bf2ea..b794f68e2 100644 --- a/src/build_options.rs +++ b/src/build_options.rs @@ -8,11 +8,13 @@ use crate::PythonInterpreter; use crate::Target; use anyhow::{bail, format_err, Context, Result}; use cargo_metadata::{Metadata, MetadataCommand, Node}; +use fs_err::{self as fs, DirEntry}; use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; use std::env; use std::io; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; use structopt::StructOpt; /// High level API for building wheels from a crate which is also used for the CLI @@ -384,8 +386,8 @@ pub fn find_interpreter( target: &Target, ) -> Result> { match bridge { - BridgeModel::Bindings(_) => { - let interpreter = if !interpreter.is_empty() { + BridgeModel::Bindings(binding_name) => { + let mut interpreter = if !interpreter.is_empty() { PythonInterpreter::check_executables(&interpreter, &target, &bridge) .context("The given list of python interpreters is invalid")? } else { @@ -406,6 +408,49 @@ pub fn find_interpreter( .join(", ") ); + if binding_name == "pyo3" && target.is_unix() && is_cross_compiling(target)? { + if let Some(cross_lib_dir) = std::env::var_os("PYO3_CROSS_LIB_DIR") { + println!("⚠ Cross-compiling is poorly supported"); + let host_python = &interpreter[0]; + println!( + "🐍 Using host {} for cross-compiling preparation", + host_python + ); + let sysconfig_path = find_sysconfigdata(cross_lib_dir.as_ref())?; + let sysconfig_data = + parse_sysconfigdata(&host_python.executable, sysconfig_path)?; + let major = sysconfig_data + .get("version_major") + .context("version_major is not defined")? + .parse::() + .context("Could not parse value of version_major")?; + let minor = sysconfig_data + .get("version_minor") + .context("version_minor is not defined")? + .parse::() + .context("Could not parse value of version_minor")?; + let abiflags = sysconfig_data + .get("ABIFLAGS") + .map(ToString::to_string) + .unwrap_or_default(); + let ext_suffix = sysconfig_data.get("EXT_SUFFIX").cloned(); + let abi_tag = sysconfig_data + .get("SOABI") + .and_then(|abi| abi.split('-').nth(1).map(ToString::to_string)); + interpreter = vec![PythonInterpreter { + major, + minor, + abiflags, + target: target.clone(), + executable: host_python.executable.clone(), + ext_suffix, + interpreter_kind: InterpreterKind::CPython, + abi_tag, + libs_dir: PathBuf::from(cross_lib_dir), + }]; + } + } + Ok(interpreter) } BridgeModel::Cffi => { @@ -469,6 +514,39 @@ fn split_extra_args(given_args: &[String]) -> Result> { Ok(splitted_args) } +fn is_cross_compiling(target: &Target) -> Result { + let target_triple = target.target_triple(); + let host = platforms::Platform::guess_current() + .map(|platform| platform.target_triple) + .ok_or_else(|| format_err!("Couldn't guess the current host platform"))?; + if target_triple == host { + // Not cross-compiling + return Ok(false); + } + + if target_triple == "x86_64-apple-darwin" && host == "aarch64-apple-darwin" { + // Not cross-compiling to compile for x86-64 Python from macOS arm64 + return Ok(false); + } + if target_triple == "aarch64-apple-darwin" && host == "x86_64-apple-darwin" { + // Not cross-compiling to compile for arm64 Python from macOS x86_64 + return Ok(false); + } + + if let Some(target_without_env) = target_triple + .rfind('-') + .map(|index| &target_triple[0..index]) + { + if host.starts_with(target_without_env) { + // Not cross-compiling if arch-vendor-os is all the same + // e.g. x86_64-unknown-linux-musl on x86_64-unknown-linux-gnu host + return Ok(false); + } + } + + Ok(true) +} + /// We need to pass feature flags to cargo metadata /// (s. https://github.com/PyO3/maturin/issues/211), but we can't pass /// all the extra args, as e.g. `--target` isn't supported. @@ -498,6 +576,185 @@ fn extra_feature_args(cargo_extra_args: &[String]) -> Vec { cargo_metadata_extra_args } +/// Parse sysconfigdata file +/// +/// The sysconfigdata is simply a dictionary containing all the build time variables used for the +/// python executable and library. Here it is read and added to a script to extract only what is +/// necessary. This necessitates a python interpreter for the host machine to work. +fn parse_sysconfigdata( + interpreter: &Path, + config_path: impl AsRef, +) -> Result> { + let mut script = fs::read_to_string(config_path)?; + script += r#" +print("version_major", build_time_vars["VERSION"][0]) # 3 +print("version_minor", build_time_vars["VERSION"][2]) # E.g., 8 +KEYS = [ + "ABIFLAGS", + "EXT_SUFFIX", + "SOABI", +] +for key in KEYS: + print(key, build_time_vars.get(key, "")) +"#; + let output = run_python_script(interpreter, &script)?; + + Ok(parse_script_output(&output)) +} + +fn parse_script_output(output: &str) -> HashMap { + output + .lines() + .filter_map(|line| { + let mut i = line.splitn(2, ' '); + Some((i.next()?.into(), i.next()?.into())) + }) + .collect() +} + +/// Run a python script using the specified interpreter binary. +fn run_python_script(interpreter: &Path, script: &str) -> Result { + let out = Command::new(interpreter) + .env("PYTHONIOENCODING", "utf-8") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()) + .spawn() + .and_then(|mut child| { + use std::io::Write; + child + .stdin + .as_mut() + .expect("piped stdin") + .write_all(script.as_bytes())?; + child.wait_with_output() + }); + + match out { + Err(err) => { + if err.kind() == io::ErrorKind::NotFound { + bail!( + "Could not find any interpreter at {}, \ + are you sure you have Python installed on your PATH?", + interpreter.display() + ); + } else { + bail!( + "Failed to run the Python interpreter at {}: {}", + interpreter.display(), + err + ); + } + } + Ok(ok) if !ok.status.success() => bail!("Python script failed"), + Ok(ok) => Ok(String::from_utf8(ok.stdout)?), + } +} + +fn starts_with(entry: &DirEntry, pat: &str) -> bool { + let name = entry.file_name(); + name.to_string_lossy().starts_with(pat) +} +fn ends_with(entry: &DirEntry, pat: &str) -> bool { + let name = entry.file_name(); + name.to_string_lossy().ends_with(pat) +} + +/// Finds the `_sysconfigdata*.py` file in the library path. +/// +/// From the python source for `_sysconfigdata*.py` is always going to be located at +/// `build/lib.{PLATFORM}-{PY_MINOR_VERSION}` when built from source. The [exact line][1] is defined as: +/// +/// ```py +/// pybuilddir = 'build/lib.%s-%s' % (get_platform(), sys.version_info[:2]) +/// ``` +/// +/// Where get_platform returns a kebab-case formated string containing the os, the architecture and +/// possibly the os' kernel version (not the case on linux). However, when installed using a package +/// manager, the `_sysconfigdata*.py` file is installed in the `${PREFIX}/lib/python3.Y/` directory. +/// The `_sysconfigdata*.py` is generally in a sub-directory of the location of `libpython3.Y.so`. +/// So we must find the file in the following possible locations: +/// +/// ```sh +/// # distribution from package manager, lib_dir should include lib/ +/// ${INSTALL_PREFIX}/lib/python3.Y/_sysconfigdata*.py +/// ${INSTALL_PREFIX}/lib/libpython3.Y.so +/// ${INSTALL_PREFIX}/lib/python3.Y/config-3.Y-${HOST_TRIPLE}/libpython3.Y.so +/// +/// # Built from source from host +/// ${CROSS_COMPILED_LOCATION}/build/lib.linux-x86_64-Y/_sysconfigdata*.py +/// ${CROSS_COMPILED_LOCATION}/libpython3.Y.so +/// +/// # if cross compiled, kernel release is only present on certain OS targets. +/// ${CROSS_COMPILED_LOCATION}/build/lib.{OS}(-{OS-KERNEL-RELEASE})?-{ARCH}-Y/_sysconfigdata*.py +/// ${CROSS_COMPILED_LOCATION}/libpython3.Y.so +/// ``` +/// +/// [1]: https://github.com/python/cpython/blob/3.5/Lib/sysconfig.py#L389 +fn find_sysconfigdata(lib_dir: &Path) -> Result { + let sysconfig_paths = search_lib_dir(lib_dir); + let mut sysconfig_paths = sysconfig_paths + .iter() + .filter_map(|p| fs::canonicalize(p).ok()) + .collect::>(); + sysconfig_paths.dedup(); + if sysconfig_paths.is_empty() { + bail!( + "Could not find either libpython.so or _sysconfigdata*.py in {}", + lib_dir.display() + ); + } else if sysconfig_paths.len() > 1 { + bail!( + "Detected multiple possible python versions, please set the PYO3_PYTHON_VERSION \ + variable to the wanted version on your system\nsysconfigdata paths = {:?}", + sysconfig_paths + ) + } + + Ok(sysconfig_paths.remove(0)) +} + +/// recursive search for _sysconfigdata, returns all possibilities of sysconfigdata paths +fn search_lib_dir(path: impl AsRef) -> Vec { + let mut sysconfig_paths = vec![]; + let version_pat = if let Some(v) = + env::var_os("PYO3_CROSS_PYTHON_VERSION").map(|s| s.into_string().unwrap()) + { + format!("python{}", v) + } else { + "python3.".into() + }; + for f in fs::read_dir(path.as_ref()).expect("Path does not exist") { + let sysc = match &f { + Ok(f) if starts_with(f, "_sysconfigdata") && ends_with(f, "py") => vec![f.path()], + Ok(f) if starts_with(f, "build") => search_lib_dir(f.path()), + Ok(f) if starts_with(f, "lib.") => { + let name = f.file_name(); + // check if right target os + let os = env::var("CARGO_CFG_TARGET_OS").unwrap(); + if !name + .to_string_lossy() + .contains(if os == "android" { "linux" } else { &os }) + { + continue; + } + // Check if right arch + if !name + .to_string_lossy() + .contains(&env::var("CARGO_CFG_TARGET_ARCH").unwrap()) + { + continue; + } + search_lib_dir(f.path()) + } + Ok(f) if starts_with(f, &version_pat) => search_lib_dir(f.path()), + _ => continue, + }; + sysconfig_paths.extend(sysc); + } + sysconfig_paths +} + #[cfg(test)] mod test { use std::path::Path; diff --git a/src/compile.rs b/src/compile.rs index 9322657ba..e78e35c18 100644 --- a/src/compile.rs +++ b/src/compile.rs @@ -211,12 +211,15 @@ fn compile_target( } if let Some(python_interpreter) = python_interpreter { - if bindings_crate.is_bindings("pyo3") { - build_command.env("PYO3_PYTHON", &python_interpreter.executable); - } + // `python_interpreter.executable` could be empty when cross compiling + if python_interpreter.executable != PathBuf::new() { + if bindings_crate.is_bindings("pyo3") { + build_command.env("PYO3_PYTHON", &python_interpreter.executable); + } - // rust-cpython, and legacy pyo3 versions - build_command.env("PYTHON_SYS_EXECUTABLE", &python_interpreter.executable); + // rust-cpython, and legacy pyo3 versions + build_command.env("PYTHON_SYS_EXECUTABLE", &python_interpreter.executable); + } } let mut cargo_build = build_command.spawn().context("Failed to run cargo")?;