Skip to content

Commit

Permalink
Better cross compiling support for PyO3 binding on Unix
Browse files Browse the repository at this point in the history
  • Loading branch information
messense committed Feb 27, 2021
1 parent a3e19ee commit 41845d5
Show file tree
Hide file tree
Showing 2 changed files with 268 additions and 8 deletions.
263 changes: 260 additions & 3 deletions src/build_options.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,13 @@ use crate::PythonInterpreter;
use crate::Target;
use anyhow::{bail, format_err, Context, Result};
use cargo_metadata::{Metadata, MetadataCommand, Node};
use fs_err::{self as fs, DirEntry};
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
use std::env;
use std::io;
use std::path::PathBuf;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use structopt::StructOpt;

/// High level API for building wheels from a crate which is also used for the CLI
Expand Down Expand Up @@ -384,8 +386,8 @@ pub fn find_interpreter(
target: &Target,
) -> Result<Vec<PythonInterpreter>> {
match bridge {
BridgeModel::Bindings(_) => {
let interpreter = if !interpreter.is_empty() {
BridgeModel::Bindings(binding_name) => {
let mut interpreter = if !interpreter.is_empty() {
PythonInterpreter::check_executables(&interpreter, &target, &bridge)
.context("The given list of python interpreters is invalid")?
} else {
Expand All @@ -406,6 +408,49 @@ pub fn find_interpreter(
.join(", ")
);

if binding_name == "pyo3" && target.is_unix() && is_cross_compiling(target)? {
if let Some(cross_lib_dir) = std::env::var_os("PYO3_CROSS_LIB_DIR") {
println!("⚠ Cross-compiling is poorly supported");
let host_python = &interpreter[0];
println!(
"🐍 Using host {} for cross-compiling preparation",
host_python
);
let sysconfig_path = find_sysconfigdata(cross_lib_dir.as_ref())?;
let sysconfig_data =
parse_sysconfigdata(&host_python.executable, sysconfig_path)?;
let major = sysconfig_data
.get("version_major")
.context("version_major is not defined")?
.parse::<usize>()
.context("Could not parse value of version_major")?;
let minor = sysconfig_data
.get("version_minor")
.context("version_minor is not defined")?
.parse::<usize>()
.context("Could not parse value of version_minor")?;
let abiflags = sysconfig_data
.get("ABIFLAGS")
.map(ToString::to_string)
.unwrap_or_default();
let ext_suffix = sysconfig_data.get("EXT_SUFFIX").cloned();
let abi_tag = sysconfig_data
.get("SOABI")
.and_then(|abi| abi.split('-').nth(1).map(ToString::to_string));
interpreter = vec![PythonInterpreter {
major,
minor,
abiflags,
target: target.clone(),
executable: host_python.executable.clone(),
ext_suffix,
interpreter_kind: InterpreterKind::CPython,
abi_tag,
libs_dir: PathBuf::from(cross_lib_dir),
}];
}
}

Ok(interpreter)
}
BridgeModel::Cffi => {
Expand Down Expand Up @@ -469,6 +514,39 @@ fn split_extra_args(given_args: &[String]) -> Result<Vec<String>> {
Ok(splitted_args)
}

fn is_cross_compiling(target: &Target) -> Result<bool> {
let target_triple = target.target_triple();
let host = platforms::Platform::guess_current()
.map(|platform| platform.target_triple)
.ok_or_else(|| format_err!("Couldn't guess the current host platform"))?;
if target_triple == host {
// Not cross-compiling
return Ok(false);
}

if target_triple == "x86_64-apple-darwin" && host == "aarch64-apple-darwin" {
// Not cross-compiling to compile for x86-64 Python from macOS arm64
return Ok(false);
}
if target_triple == "aarch64-apple-darwin" && host == "x86_64-apple-darwin" {
// Not cross-compiling to compile for arm64 Python from macOS x86_64
return Ok(false);
}

if let Some(target_without_env) = target_triple
.rfind('-')
.map(|index| &target_triple[0..index])
{
if host.starts_with(target_without_env) {
// Not cross-compiling if arch-vendor-os is all the same
// e.g. x86_64-unknown-linux-musl on x86_64-unknown-linux-gnu host
return Ok(false);
}
}

Ok(true)
}

/// We need to pass feature flags to cargo metadata
/// (s. https://github.com/PyO3/maturin/issues/211), but we can't pass
/// all the extra args, as e.g. `--target` isn't supported.
Expand Down Expand Up @@ -498,6 +576,185 @@ fn extra_feature_args(cargo_extra_args: &[String]) -> Vec<String> {
cargo_metadata_extra_args
}

/// Parse sysconfigdata file
///
/// The sysconfigdata is simply a dictionary containing all the build time variables used for the
/// python executable and library. Here it is read and added to a script to extract only what is
/// necessary. This necessitates a python interpreter for the host machine to work.
fn parse_sysconfigdata(
interpreter: &Path,
config_path: impl AsRef<Path>,
) -> Result<HashMap<String, String>> {
let mut script = fs::read_to_string(config_path)?;
script += r#"
print("version_major", build_time_vars["VERSION"][0]) # 3
print("version_minor", build_time_vars["VERSION"][2]) # E.g., 8
KEYS = [
"ABIFLAGS",
"EXT_SUFFIX",
"SOABI",
]
for key in KEYS:
print(key, build_time_vars.get(key, ""))
"#;
let output = run_python_script(interpreter, &script)?;

Ok(parse_script_output(&output))
}

fn parse_script_output(output: &str) -> HashMap<String, String> {
output
.lines()
.filter_map(|line| {
let mut i = line.splitn(2, ' ');
Some((i.next()?.into(), i.next()?.into()))
})
.collect()
}

/// Run a python script using the specified interpreter binary.
fn run_python_script(interpreter: &Path, script: &str) -> Result<String> {
let out = Command::new(interpreter)
.env("PYTHONIOENCODING", "utf-8")
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::inherit())
.spawn()
.and_then(|mut child| {
use std::io::Write;
child
.stdin
.as_mut()
.expect("piped stdin")
.write_all(script.as_bytes())?;
child.wait_with_output()
});

match out {
Err(err) => {
if err.kind() == io::ErrorKind::NotFound {
bail!(
"Could not find any interpreter at {}, \
are you sure you have Python installed on your PATH?",
interpreter.display()
);
} else {
bail!(
"Failed to run the Python interpreter at {}: {}",
interpreter.display(),
err
);
}
}
Ok(ok) if !ok.status.success() => bail!("Python script failed"),
Ok(ok) => Ok(String::from_utf8(ok.stdout)?),
}
}

fn starts_with(entry: &DirEntry, pat: &str) -> bool {
let name = entry.file_name();
name.to_string_lossy().starts_with(pat)
}
fn ends_with(entry: &DirEntry, pat: &str) -> bool {
let name = entry.file_name();
name.to_string_lossy().ends_with(pat)
}

/// Finds the `_sysconfigdata*.py` file in the library path.
///
/// From the python source for `_sysconfigdata*.py` is always going to be located at
/// `build/lib.{PLATFORM}-{PY_MINOR_VERSION}` when built from source. The [exact line][1] is defined as:
///
/// ```py
/// pybuilddir = 'build/lib.%s-%s' % (get_platform(), sys.version_info[:2])
/// ```
///
/// Where get_platform returns a kebab-case formated string containing the os, the architecture and
/// possibly the os' kernel version (not the case on linux). However, when installed using a package
/// manager, the `_sysconfigdata*.py` file is installed in the `${PREFIX}/lib/python3.Y/` directory.
/// The `_sysconfigdata*.py` is generally in a sub-directory of the location of `libpython3.Y.so`.
/// So we must find the file in the following possible locations:
///
/// ```sh
/// # distribution from package manager, lib_dir should include lib/
/// ${INSTALL_PREFIX}/lib/python3.Y/_sysconfigdata*.py
/// ${INSTALL_PREFIX}/lib/libpython3.Y.so
/// ${INSTALL_PREFIX}/lib/python3.Y/config-3.Y-${HOST_TRIPLE}/libpython3.Y.so
///
/// # Built from source from host
/// ${CROSS_COMPILED_LOCATION}/build/lib.linux-x86_64-Y/_sysconfigdata*.py
/// ${CROSS_COMPILED_LOCATION}/libpython3.Y.so
///
/// # if cross compiled, kernel release is only present on certain OS targets.
/// ${CROSS_COMPILED_LOCATION}/build/lib.{OS}(-{OS-KERNEL-RELEASE})?-{ARCH}-Y/_sysconfigdata*.py
/// ${CROSS_COMPILED_LOCATION}/libpython3.Y.so
/// ```
///
/// [1]: https://github.com/python/cpython/blob/3.5/Lib/sysconfig.py#L389
fn find_sysconfigdata(lib_dir: &Path) -> Result<PathBuf> {
let sysconfig_paths = search_lib_dir(lib_dir);
let mut sysconfig_paths = sysconfig_paths
.iter()
.filter_map(|p| fs::canonicalize(p).ok())
.collect::<Vec<PathBuf>>();
sysconfig_paths.dedup();
if sysconfig_paths.is_empty() {
bail!(
"Could not find either libpython.so or _sysconfigdata*.py in {}",
lib_dir.display()
);
} else if sysconfig_paths.len() > 1 {
bail!(
"Detected multiple possible python versions, please set the PYO3_PYTHON_VERSION \
variable to the wanted version on your system\nsysconfigdata paths = {:?}",
sysconfig_paths
)
}

Ok(sysconfig_paths.remove(0))
}

/// recursive search for _sysconfigdata, returns all possibilities of sysconfigdata paths
fn search_lib_dir(path: impl AsRef<Path>) -> Vec<PathBuf> {
let mut sysconfig_paths = vec![];
let version_pat = if let Some(v) =
env::var_os("PYO3_CROSS_PYTHON_VERSION").map(|s| s.into_string().unwrap())
{
format!("python{}", v)
} else {
"python3.".into()
};
for f in fs::read_dir(path.as_ref()).expect("Path does not exist") {
let sysc = match &f {
Ok(f) if starts_with(f, "_sysconfigdata") && ends_with(f, "py") => vec![f.path()],
Ok(f) if starts_with(f, "build") => search_lib_dir(f.path()),
Ok(f) if starts_with(f, "lib.") => {
let name = f.file_name();
// check if right target os
let os = env::var("CARGO_CFG_TARGET_OS").unwrap();
if !name
.to_string_lossy()
.contains(if os == "android" { "linux" } else { &os })
{
continue;
}
// Check if right arch
if !name
.to_string_lossy()
.contains(&env::var("CARGO_CFG_TARGET_ARCH").unwrap())
{
continue;
}
search_lib_dir(f.path())
}
Ok(f) if starts_with(f, &version_pat) => search_lib_dir(f.path()),
_ => continue,
};
sysconfig_paths.extend(sysc);
}
sysconfig_paths
}

#[cfg(test)]
mod test {
use std::path::Path;
Expand Down
13 changes: 8 additions & 5 deletions src/compile.rs
Original file line number Diff line number Diff line change
Expand Up @@ -211,12 +211,15 @@ fn compile_target(
}

if let Some(python_interpreter) = python_interpreter {
if bindings_crate.is_bindings("pyo3") {
build_command.env("PYO3_PYTHON", &python_interpreter.executable);
}
// `python_interpreter.executable` could be empty when cross compiling
if python_interpreter.executable != PathBuf::new() {
if bindings_crate.is_bindings("pyo3") {
build_command.env("PYO3_PYTHON", &python_interpreter.executable);
}

// rust-cpython, and legacy pyo3 versions
build_command.env("PYTHON_SYS_EXECUTABLE", &python_interpreter.executable);
// rust-cpython, and legacy pyo3 versions
build_command.env("PYTHON_SYS_EXECUTABLE", &python_interpreter.executable);
}
}

let mut cargo_build = build_command.spawn().context("Failed to run cargo")?;
Expand Down

0 comments on commit 41845d5

Please sign in to comment.