From 9a4121930a7c6c7e92cb6a920447c937aad532f1 Mon Sep 17 00:00:00 2001 From: Rene Leveille Date: Sat, 8 Aug 2020 22:37:38 -0400 Subject: [PATCH] appy suggestions from PR --- CHANGELOG.md | 4 +- Cargo.toml | 7 +- build.rs | 186 ++++++++++++++++++++----- guide/src/building_and_distribution.md | 3 +- 4 files changed, 160 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96304a1a7a3..899ff195408 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,6 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Change `PyIterator::from_object` to return `PyResult` instead of `Result`. [#1051](https://github.com/PyO3/pyo3/pull/1051) - Implement `Send + Sync` for `PyErr`. `PyErr::new`, `PyErr::from_type`, `PyException::py_err` and `PyException::into` have had these bounds added to their arguments. [#1067](https://github.com/PyO3/pyo3/pull/1067) - Change `#[pyproto]` to return NotImplemented for operators for which Python can try a reversed operation. [1072](https://github.com/PyO3/pyo3/pull/1072) -- Change method for getting cross-compilation configurations using the file which provides the `sysconfig` module with its data, `_sysconfig_*.py` located in the lib directory. [1077](https://github.com/PyO3/pyo3/issues/1077) ### Removed - Remove `PyString::as_bytes`. [#1023](https://github.com/PyO3/pyo3/pull/1023) @@ -36,7 +35,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Don't rely on the order of structmembers to compute offsets in PyCell. Related to [#1058](https://github.com/PyO3/pyo3/pull/1058). [#1059](https://github.com/PyO3/pyo3/pull/1059) - Allows `&Self` as a `#[pymethods]` argument again. [#1071](https://github.com/PyO3/pyo3/pull/1071) -- Linking against libpython when compiling for android even with `extension-module` set[#1082](https://github.com/PyO3/pyo3/issues/1082) +- Fix python configuration detection when cross-compiling. [1077](https://github.com/PyO3/pyo3/issues/1077) +- Link against libpython on android with `extension-module` set. [#1082](https://github.com/PyO3/pyo3/issues/1082) ## [0.11.1] - 2020-06-30 ### Added diff --git a/Cargo.toml b/Cargo.toml index 134e3d00aed..5a1eff2cbf2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,8 +36,8 @@ trybuild = "1.0.23" rustversion = "1.0" [build-dependencies] -walkdir = "~2.3" -regex = "~1.3" +walkdir = {version = "~2.3", optional = true } +regex = { version = "~1.3", optional = true } [features] default = ["macros"] @@ -58,6 +58,9 @@ extension-module = [] # are welcome. # abi3 = [] +# Use this feature when cross-compiling the library +cross-compile = ["walkdir", "regex"] + [workspace] members = [ "pyo3cls", diff --git a/build.rs b/build.rs index ae45729b9bc..05a30269acb 100644 --- a/build.rs +++ b/build.rs @@ -2,7 +2,7 @@ use std::{ collections::HashMap, convert::AsRef, env, fmt, - fs::File, + fs::{self, File}, io::{self, BufRead, BufReader}, path::{Path, PathBuf}, process::{Command, Stdio}, @@ -76,6 +76,57 @@ impl FromStr for PythonInterpreterKind { } } +struct PythonPaths { + lib_dir: String, + include_dir: Option, +} + +impl PythonPaths { + fn both() -> Result { + Ok(PythonPaths { + include_dir: Some(PythonPaths::validate_variable("PYO3_CROSS_INCLUDE_DIR")?), + ..PythonPaths::lib_only()? + }) + } + + fn lib_only() -> Result { + Ok(PythonPaths { + lib_dir: PythonPaths::validate_variable("PYO3_CROSS_LIB_DIR")?, + include_dir: None, + }) + } + + fn validate_variable(var: &str) -> Result { + let path = match env::var(var) { + Ok(v) => v, + Err(_) => bail!( + "Must provide {} environment variable when cross-compiling", + var + ), + }; + + if fs::metadata(&path).is_err() { + bail!("{} value of {} does not exist", var, path) + } + + Ok(path) + } +} + +fn cross_compiling() -> Result> { + if env::var("TARGET")? == env::var("HOST")? { + return Ok(None); + } + + if env::var("CARGO_CFG_TARGET_FAMILY")? == "windows" { + Ok(Some(PythonPaths::both()?)) + } else if cfg!(feature = "cross-compile") { + Ok(Some(PythonPaths::lib_only()?)) + } else { + bail!("Cross compiling PyO3 for a unix platform requires the cross-compile feature to be enabled") + } +} + /// A list of python interpreter compile-time preprocessor defines that /// we will pick up and pass to rustc via --cfg=py_sys_config={varname}; /// this allows using them conditional cfg attributes in the .rs files, so @@ -134,11 +185,57 @@ fn fix_config_map(mut config_map: HashMap) -> HashMap) -> Result> { let config_reader = BufReader::new(File::open(config_path)?); let mut entries = HashMap::new(); - let entry_re = regex::Regex::new(r#"'([a-zA-Z_0-9]*)': ((?:"|')([\S ]*)(?:"|')|\d+)($|,$)"#)?; - let subsequent_re = regex::Regex::new(r#"\s+(?:"|')([\S ]*)(?:"|')($|,)"#)?; + + let entry_re = regex::Regex::new(r#"'([a-zA-Z_0-9]*)': ((?:"|')([\S ]*)(?:"|')|\d+)($|,|})"#)?; + let subsequent_re = regex::Regex::new(r#"\s+(?:"|')([\S ]*)(?:"|')($|,|})"#)?; let mut previous_finished = None; for maybe_line in config_reader.lines() { let line = maybe_line?; @@ -174,13 +271,35 @@ fn parse_sysconfigdata(config_path: impl AsRef) -> Result Result<(InterpreterConfig, HashMap)> { - // find info from sysconfig - // first find sysconfigdata file - let sysconfig_re = regex::Regex::new(r"_sysconfigdata_m?_linux_([a-z_\-0-9]*)?\.py$")?; - let mut walker = walkdir::WalkDir::new(&python_lib_dir).into_iter(); + let sysconfig_re = + regex::Regex::new(r"_sysconfigdata_(?:u|d|m|)_[a-z0-9]+_([a-z_\-0-9]*)?\.py$")?; + let mut walker = walkdir::WalkDir::new(&python_paths.lib_dir).into_iter(); let sysconfig_path = loop { let entry = match walker.next() { Some(Ok(entry)) => entry, @@ -224,7 +343,7 @@ fn load_cross_compile_from_sysconfigdata( let interpreter_config = InterpreterConfig { version: python_version, - libdir: Some(python_lib_dir.to_owned()), + libdir: Some(python_paths.lib_dir.to_owned()), shared, ld_version, base_prefix: "".to_string(), @@ -236,9 +355,9 @@ fn load_cross_compile_from_sysconfigdata( } fn load_cross_compile_from_headers( - python_include_dir: &str, - python_lib_dir: &str, + python_paths: PythonPaths, ) -> Result<(InterpreterConfig, HashMap)> { + let python_include_dir = python_paths.include_dir.unwrap(); let python_include_dir = Path::new(&python_include_dir); let patchlevel_defines = parse_header_defines(python_include_dir.join("patchlevel.h"))?; @@ -279,7 +398,7 @@ fn load_cross_compile_from_headers( let interpreter_config = InterpreterConfig { version: python_version, - libdir: Some(python_lib_dir.to_owned()), + libdir: Some(python_paths.lib_dir.to_owned()), shared, ld_version: format!("{}.{}", major, minor), base_prefix: "".to_string(), @@ -290,17 +409,27 @@ fn load_cross_compile_from_headers( Ok((interpreter_config, fix_config_map(config_map))) } +#[allow(unused_variables)] fn load_cross_compile_info( - python_include_dir: String, - python_lib_dir: String, + python_paths: PythonPaths, ) -> Result<(InterpreterConfig, HashMap)> { - // Try to configure from the sysconfigdata file which is more accurate for the information - // provided at python's compile time - match load_cross_compile_from_sysconfigdata(&python_lib_dir) { - Ok(ret) => Ok(ret), - // If the config could not be loaded by sysconfigdata, failover to configuring from headers - Err(_) => load_cross_compile_from_headers(&python_include_dir, &python_lib_dir), + let target_family = env::var("CARGO_CFG_TARGET_FAMILY")?; + // Because compiling for windows on linux still includes the unix target family + if target_family == "unix" && cfg!(feature = "cross-compile") { + // Configure for unix platforms using the sysconfigdata file + #[cfg(feature = "cross-compile")] + { + return load_cross_compile_from_sysconfigdata(python_paths); + } + } else if target_family == "windows" { + // Must configure by headers on windows platform + return load_cross_compile_from_headers(python_paths); } + + // If you get here you were on unix without cross-compile capabilities + bail!( + "Cross compiling PyO3 for a unix platform requires the cross-compile feature to be enabled" + ); } /// Examine python's compile flags to pass to cfg by launching @@ -692,24 +821,11 @@ fn main() -> Result<()> { // // Detecting if cross-compiling by checking if the target triple is different from the host // rustc's triple. - let cross_compiling = env::var("TARGET") != env::var("HOST"); - let (interpreter_config, mut config_map) = if cross_compiling { + let (interpreter_config, mut config_map) = if let Some(paths) = cross_compiling()? { // If cross compiling we need the path to the cross-compiled include dir and lib dir, else // fail quickly and loudly - let python_include_dir = match env::var("PYO3_CROSS_INCLUDE_DIR") { - Ok(v) => v, - Err(_) => bail!( - "Must provide PYO3_CROSS_INCLUDE_DIR environment variable when cross-compiling" - ), - }; - let python_lib_dir = match env::var("PYO3_CROSS_LIB_DIR") { - Ok(v) => v, - Err(_) => { - bail!("Must provide PYO3_CROSS_LIB_DIR environment variable when cross-compiling") - } - }; - load_cross_compile_info(python_include_dir, python_lib_dir)? + load_cross_compile_info(paths)? } else { find_interpreter_and_get_config()? }; diff --git a/guide/src/building_and_distribution.md b/guide/src/building_and_distribution.md index 7c36fd3822f..d8d9c2798fa 100644 --- a/guide/src/building_and_distribution.md +++ b/guide/src/building_and_distribution.md @@ -51,8 +51,9 @@ See https://github.com/japaric/rust-cross for a primer on cross compiling Rust i After you've obtained the above, you can build a cross compiled PyO3 module by setting a few extra environment variables: -* `PYO3_CROSS_INCLUDE_DIR`: This variable must be set to the directory containing the headers for the target's Python interpreter. +* `PYO3_CROSS_INCLUDE_DIR`: This variable must be set to the directory containing the headers for the target's Python interpreter. It is only necessary if compiling for Windows platforms * `PYO3_CROSS_LIB_DIR`: This variable must be set to the directory containing the target's libpython DSO. +* If compiling for unix platforms, the `cross-compile` feature must be set. An example might look like the following (assuming your target's sysroot is at `/home/pyo3/cross/sysroot` and that your target is `armv7`):