Skip to content

Commit

Permalink
pyo3-build-config: Add PYO3_CROSS_PYTHON_IMPLEMENTATION env var
Browse files Browse the repository at this point in the history 
Adds a new cross-compile target interpreter configuration
environment variable.

This feature allows PyO3 to target PyPy on both Windows and Unix
cross compile targets.
  • Loading branch information
@ravenexp
ravenexp committed Apr 4, 2022
1 parent d3dcbd7 commit 8067536
Show file tree
Hide file tree
Showing 3 changed files with 130 additions and 26 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

### Added

- Add `PYO3_CROSS_PYTHON_IMPLEMENTATION` environment variable for selecting the default cross Python implementation. [#2272](https://github.com/PyO3/pyo3/pull/2272)
- Add new public `pyo3-build-config` API using the types from `target_lexicon` crate. Deprecate `cross_compiling()`. [#2253](https://github.com/PyO3/pyo3/pull/2253)
- Allow dependent crates to access config values from `pyo3-build-config` via cargo link dep env vars. [#2092](https://github.com/PyO3/pyo3/pull/2092)
- Added methods on `InterpreterConfig` to run Python scripts using the configured executable. [#2092](https://github.com/PyO3/pyo3/pull/2092)
Expand Down
1 change: 1 addition & 0 deletions guide/src/building_and_distribution.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,7 @@ When cross-compiling, PyO3's build script cannot execute the target Python inter
* `PYO3_CROSS`: If present this variable forces PyO3 to configure as a cross-compilation.
* `PYO3_CROSS_LIB_DIR`: This variable can be set to the directory containing the target's libpython DSO and the associated `_sysconfigdata*.py` file for Unix-like targets, or the Python DLL import libraries for the Windows target. This variable is only needed when the output binary must link to libpython explicitly (e.g. when targeting Windows and Android or embedding a Python interpreter), or when it is absolutely required to get the interpreter configuration from `_sysconfigdata*.py`.
* `PYO3_CROSS_PYTHON_VERSION`: Major and minor version (e.g. 3.9) of the target Python installation. This variable is only needed if PyO3 cannot determine the version to target from `abi3-py3*` features, or if `PYO3_CROSS_LIB_DIR` is not set, or if there are multiple versions of Python present in `PYO3_CROSS_LIB_DIR`.
* `PYO3_CROSS_PYTHON_IMPLEMENTATION`: Python implementation name ("CPython" or "PyPy") of the target Python installation. CPython is assumed by default when this variable is not set, unless `PYO3_CROSS_LIB_DIR` is set for a Unix-like target and PyO3 can get the interpreter configuration from `_sysconfigdata*.py`.
An example might look like the following (assuming your target's sysroot is at `/home/pyo3/cross/sysroot` and that your target is `armv7`):
Expand Down
154 changes: 128 additions & 26 deletions pyo3-build-config/src/impl_.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -690,6 +690,9 @@ pub struct CrossCompileConfig {
/// The version of the Python library to link against.
version: Option<PythonVersion>,

/// The target Python implementation hint (CPython or PyPy)
implementation: Option<PythonImplementation>,

/// The compile target triple (e.g. aarch64-unknown-linux-gnu)
target: Triple,
}
Expand All @@ -707,12 +710,14 @@ impl CrossCompileConfig {
if env_vars.any() || Self::is_cross_compiling_from_to(host, target) {
let lib_dir = env_vars.lib_dir_path()?;
let version = env_vars.parse_version()?;
let implementation = env_vars.parse_implementation()?;
let target = target.clone();

Ok(Some(CrossCompileConfig {
lib_dir,
target,
version,
implementation,
target,
}))
} else {
Ok(None)
Expand Down Expand Up @@ -752,17 +757,37 @@ impl CrossCompileConfig {
}
}

pub(crate) struct CrossCompileEnvVars {
/// PyO3-specific cross compile environment variable values
struct CrossCompileEnvVars {
/// `PYO3_CROSS`
pyo3_cross: Option<OsString>,
/// `PYO3_CROSS_LIB_DIR`
pyo3_cross_lib_dir: Option<OsString>,
/// `PYO3_CROSS_PYTHON_VERSION`
pyo3_cross_python_version: Option<OsString>,
/// `PYO3_CROSS_PYTHON_IMPLEMENTATION`
pyo3_cross_python_implementation: Option<OsString>,
}

impl CrossCompileEnvVars {
/// Grabs the PyO3 cross-compile variables from the environment.
///
/// Registers the build script to rerun if any of the variables changes.
fn from_env() -> Self {
CrossCompileEnvVars {
pyo3_cross: env_var("PYO3_CROSS"),
pyo3_cross_lib_dir: env_var("PYO3_CROSS_LIB_DIR"),
pyo3_cross_python_version: env_var("PYO3_CROSS_PYTHON_VERSION"),
pyo3_cross_python_implementation: env_var("PYO3_CROSS_PYTHON_IMPLEMENTATION"),
}
}

/// Checks if any of the variables is set.
fn any(&self) -> bool {
self.pyo3_cross.is_some()
|| self.pyo3_cross_lib_dir.is_some()
|| self.pyo3_cross_python_version.is_some()
|| self.pyo3_cross_python_implementation.is_some()
}

/// Parses `PYO3_CROSS_PYTHON_VERSION` environment variable value
Expand All @@ -784,6 +809,25 @@ impl CrossCompileEnvVars {
Ok(version)
}

/// Parses `PYO3_CROSS_PYTHON_IMPLEMENTATION` environment variable value
/// into `PythonImplementation`.
fn parse_implementation(&self) -> Result<Option<PythonImplementation>> {
let implementation = self
.pyo3_cross_python_implementation
.as_ref()
.map(|os_string| {
let utf8_str = os_string
.to_str()
.ok_or("PYO3_CROSS_PYTHON_IMPLEMENTATION is not valid a UTF-8 string")?;
utf8_str
.parse()
.context("failed to parse PYO3_CROSS_PYTHON_IMPLEMENTATION")
})
.transpose()?;

Ok(implementation)
}

/// Converts the stored `PYO3_CROSS_LIB_DIR` variable value (if any)
/// into a `PathBuf` instance.
///
Expand All @@ -802,14 +846,6 @@ impl CrossCompileEnvVars {
}
}

pub(crate) fn cross_compile_env_vars() -> CrossCompileEnvVars {
CrossCompileEnvVars {
pyo3_cross: env::var_os("PYO3_CROSS"),
pyo3_cross_lib_dir: env::var_os("PYO3_CROSS_LIB_DIR"),
pyo3_cross_python_version: env::var_os("PYO3_CROSS_PYTHON_VERSION"),
}
}

/// Detect whether we are cross compiling and return an assembled CrossCompileConfig if so.
///
/// This function relies on PyO3 cross-compiling environment variables:
Expand Down Expand Up @@ -879,7 +915,7 @@ pub fn cross_compiling_from_to(
host: &Triple,
target: &Triple,
) -> Result<Option<CrossCompileConfig>> {
let env_vars = cross_compile_env_vars();
let env_vars = CrossCompileEnvVars::from_env();
CrossCompileConfig::try_from_env_vars_host_target(env_vars, host, target)
}

Expand All @@ -889,7 +925,7 @@ pub fn cross_compiling_from_to(
/// This must be called from PyO3's build script, because it relies on environment
/// variables such as `CARGO_CFG_TARGET_OS` which aren't available at any other time.
pub fn cross_compiling_from_cargo_env() -> Result<Option<CrossCompileConfig>> {
let env_vars = cross_compile_env_vars();
let env_vars = CrossCompileEnvVars::from_env();
let host = Triple::host();
let target = target_triple_from_env();

Expand Down Expand Up @@ -1302,7 +1338,7 @@ fn cross_compile_from_sysconfigdata(
/// Generates "default" cross compilation information for the target.
///
/// This should work for most CPython extension modules when targeting
/// Windows, MacOS and Linux.
/// Windows, macOS and Linux.
///
/// Must be called from a PyO3 crate build script.
fn default_cross_compile(cross_compile_config: &CrossCompileConfig) -> Result<InterpreterConfig> {
Expand All @@ -1315,7 +1351,9 @@ fn default_cross_compile(cross_compile_config: &CrossCompileConfig) -> Result<In
)?;

let abi3 = is_abi3();
let implementation = PythonImplementation::CPython;
let implementation = cross_compile_config
.implementation
.unwrap_or(PythonImplementation::CPython);

let lib_name = if cross_compile_config.target.operating_system == OperatingSystem::Windows {
let mingw = cross_compile_config.target.environment == Environment::Gnu;
Expand Down Expand Up @@ -1808,12 +1846,20 @@ mod tests {

#[test]
fn windows_hardcoded_cross_compile() {
let cross_config = CrossCompileConfig {
lib_dir: Some("C:\\some\\path".into()),
version: Some(PythonVersion { major: 3, minor: 7 }),
target: triple!("i686-pc-windows-msvc"),
let env_vars = CrossCompileEnvVars {
pyo3_cross: None,
pyo3_cross_lib_dir: Some("C:\\some\\path".into()),
pyo3_cross_python_implementation: None,
pyo3_cross_python_version: Some("3.7".into()),
};

let host = triple!("x86_64-unknown-linux-gnu");
let target = triple!("i686-pc-windows-msvc");
let cross_config =
CrossCompileConfig::try_from_env_vars_host_target(env_vars, &host, &target)
.unwrap()
.unwrap();

assert_eq!(
default_cross_compile(&cross_config).unwrap(),
InterpreterConfig {
Expand All @@ -1834,12 +1880,20 @@ mod tests {

#[test]
fn mingw_hardcoded_cross_compile() {
let cross_config = CrossCompileConfig {
lib_dir: Some("/usr/lib/mingw".into()),
version: Some(PythonVersion { major: 3, minor: 8 }),
target: triple!("i686-pc-windows-gnu"),
let env_vars = CrossCompileEnvVars {
pyo3_cross: None,
pyo3_cross_lib_dir: Some("/usr/lib/mingw".into()),
pyo3_cross_python_implementation: None,
pyo3_cross_python_version: Some("3.8".into()),
};

let host = triple!("x86_64-unknown-linux-gnu");
let target = triple!("i686-pc-windows-gnu");
let cross_config =
CrossCompileConfig::try_from_env_vars_host_target(env_vars, &host, &target)
.unwrap()
.unwrap();

assert_eq!(
default_cross_compile(&cross_config).unwrap(),
InterpreterConfig {
Expand All @@ -1860,12 +1914,20 @@ mod tests {

#[test]
fn unix_hardcoded_cross_compile() {
let cross_config = CrossCompileConfig {
lib_dir: Some("/usr/arm64/lib".into()),
version: Some(PythonVersion { major: 3, minor: 9 }),
target: triple!("aarch64-unknown-linux-gnu"),
let env_vars = CrossCompileEnvVars {
pyo3_cross: None,
pyo3_cross_lib_dir: Some("/usr/arm64/lib".into()),
pyo3_cross_python_implementation: None,
pyo3_cross_python_version: Some("3.9".into()),
};

let host = triple!("x86_64-unknown-linux-gnu");
let target = triple!("aarch64-unknown-linux-gnu");
let cross_config =
CrossCompileConfig::try_from_env_vars_host_target(env_vars, &host, &target)
.unwrap()
.unwrap();

assert_eq!(
default_cross_compile(&cross_config).unwrap(),
InterpreterConfig {
Expand All @@ -1884,6 +1946,42 @@ mod tests {
);
}

#[test]
fn pypy_hardcoded_cross_compile() {
let env_vars = CrossCompileEnvVars {
pyo3_cross: None,
pyo3_cross_lib_dir: None,
pyo3_cross_python_implementation: Some("PyPy".into()),
pyo3_cross_python_version: Some("3.10".into()),
};

let triple = triple!("x86_64-unknown-linux-gnu");
let cross_config =
CrossCompileConfig::try_from_env_vars_host_target(env_vars, &triple, &triple)
.unwrap()
.unwrap();

assert_eq!(
default_cross_compile(&cross_config).unwrap(),
InterpreterConfig {
implementation: PythonImplementation::PyPy,
version: PythonVersion {
major: 3,
minor: 10
},
shared: true,
abi3: false,
lib_name: Some("pypy3.10-c".into()),
lib_dir: None,
executable: None,
pointer_width: None,
build_flags: BuildFlags::default(),
suppress_build_script_link_lines: false,
extra_build_script_lines: vec![],
}
);
}

#[test]
fn default_lib_name_windows() {
use PythonImplementation::*;
Expand Down Expand Up @@ -1975,6 +2073,7 @@ mod tests {
pyo3_cross: None,
pyo3_cross_lib_dir: None,
pyo3_cross_python_version: Some("3.9".into()),
pyo3_cross_python_implementation: None,
};

assert_eq!(
Expand All @@ -1986,6 +2085,7 @@ mod tests {
pyo3_cross: None,
pyo3_cross_lib_dir: None,
pyo3_cross_python_version: None,
pyo3_cross_python_implementation: None,
};

assert_eq!(env_vars.parse_version().unwrap(), None);
Expand All @@ -1994,6 +2094,7 @@ mod tests {
pyo3_cross: None,
pyo3_cross_lib_dir: None,
pyo3_cross_python_version: Some("100".into()),
pyo3_cross_python_implementation: None,
};

assert!(env_vars.parse_version().is_err());
Expand Down Expand Up @@ -2068,6 +2169,7 @@ mod tests {
let cross = CrossCompileConfig {
lib_dir: Some(lib_dir.into()),
version: Some(interpreter_config.version),
implementation: Some(interpreter_config.implementation),
target: triple!("x86_64-unknown-linux-gnu"),
};

Expand Down

0 comments on commit 8067536

Please sign in to comment.