From d20f741b4217a0225066243a9e1f116567f4ec6a Mon Sep 17 00:00:00 2001 From: konstin Date: Mon, 27 Dec 2021 18:23:07 +0900 Subject: [PATCH 1/8] Zig for manylinux compliance without docker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This allows compiling for a specified manylinux version by using `zig cc`s custom glibc targeting (https://andrewkelley.me/post/zig-cc-powerful-drop-in-replacement-gcc-clang.html). You just add `--zig`, and it will target the policy you want, defaulting to manylinux2010. Example from ubuntu 20.04: ``` $ cargo run -- build -m test-crates/pyo3-pure/Cargo.toml -i python --no-sdist 2> /dev/null 🔗 Found pyo3 bindings with abi3 support for Python ≥ 3.6 🐍 Not using a specific python interpreter (With abi3, an interpreter is only required on windows) 📖 Found type stub file at pyo3_pure.pyi 📦 Built wheel for abi3 Python ≥ 3.6 to /home/konsti/maturin/test-crates/pyo3-pure/target/wheels/pyo3_pure-2.1.2-cp36-abi3-manylinux_2_24_x86_64.whl $ cargo run -- build -m test-crates/pyo3-pure/Cargo.toml -i python --no-sdist --zig 2> /dev/null 🔗 Found pyo3 bindings with abi3 support for Python ≥ 3.6 🐍 Not using a specific python interpreter (With abi3, an interpreter is only required on windows) 📖 Found type stub file at pyo3_pure.pyi 📦 Built wheel for abi3 Python ≥ 3.6 to /home/konsti/maturin/test-crates/pyo3-pure/target/wheels/pyo3_pure-2.1.2-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.whl ``` Maturin has to be installed with `maturin[zig]` to get a compatible zig version. Still missing: * musl (code is there, but untested) * pyproject.toml integration (do we want/need that?) * Documentation in the user guide --- pyproject.toml | 5 +++ src/auditwheel/audit.rs | 3 +- src/auditwheel/policy.rs | 2 +- src/build_context.rs | 4 ++- src/build_options.rs | 34 ++++++++++++++---- src/compile.rs | 77 ++++++++++++++++++++++++++++++++++++++-- src/develop.rs | 3 +- src/module_writer.rs | 6 ++-- tests/common/mod.rs | 2 +- 9 files changed, 119 insertions(+), 17 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 620408640..e4e4c47c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,11 @@ classifiers = [ ] dependencies = ["toml~=0.10.2"] +[project.optional-dependencies] +zig = [ + "ziglang~=0.9.0", +] + [project.urls] "Source Code" = "https://github.com/PyO3/maturin" Issues = "https://github.com/PyO3/maturin/issues" diff --git a/src/auditwheel/audit.rs b/src/auditwheel/audit.rs index f29b5d7d2..cfa16ce9e 100644 --- a/src/auditwheel/audit.rs +++ b/src/auditwheel/audit.rs @@ -320,7 +320,8 @@ pub fn auditwheel_rs( policy.fixup_musl_libc_so_name(target.target_arch()); if let Some(highest_policy) = highest_policy { - if policy.priority < highest_policy.priority { + // Don't recommend manylinux1 because rust doesn't support it anymore + if policy.priority < highest_policy.priority && highest_policy.name != "manylinux_2_5" { println!( "📦 Wheel is eligible for a higher priority tag. \ You requested {} but this wheel is eligible for {}", diff --git a/src/auditwheel/policy.rs b/src/auditwheel/policy.rs index aa29c56b9..6aa8f8cc6 100644 --- a/src/auditwheel/policy.rs +++ b/src/auditwheel/policy.rs @@ -59,7 +59,7 @@ impl Display for Policy { f.write_str(&self.name) } else { f.write_fmt(format_args!( - "{}(aka {})", + "{} (aka {})", &self.name, self.aliases.join(",") )) diff --git a/src/build_context.rs b/src/build_context.rs index 74fb8c721..fe81c2304 100644 --- a/src/build_context.rs +++ b/src/build_context.rs @@ -158,8 +158,10 @@ pub struct BuildContext { pub release: bool, /// Strip the library for minimum file size pub strip: bool, - /// Whether to skip checking the linked libraries for manylinux/musllinux compliance + /// Skip checking the linked libraries for manylinux/musllinux compliance pub skip_auditwheel: bool, + /// When compiling for manylinux, use zig as linker to ensure glibc version compliance + pub zig: bool, /// Whether to use the the manylinux/musllinux or use the native linux tag (off) pub platform_tag: Option, /// Extra arguments that will be passed to cargo as `cargo rustc [...] [arg1] [arg2] --` diff --git a/src/build_options.rs b/src/build_options.rs index 6f0d2347c..1f2bf2c94 100644 --- a/src/build_options.rs +++ b/src/build_options.rs @@ -64,6 +64,13 @@ pub struct BuildOptions { /// Don't check for manylinux compliance #[structopt(long = "skip-auditwheel")] pub skip_auditwheel: bool, + /// For manylinux targets, use zig to ensure compliance for the chosen manylinux version + /// + /// Default to manylinux2010/manylinux_2_12 if you do not specify an `--compatibility` + /// + /// Make sure you installed zig with `pip install maturin[zig]` + #[structopt(long)] + pub zig: bool, /// The --target option for cargo #[structopt(long, name = "TRIPLE", env = "CARGO_BUILD_TARGET")] pub target: Option, @@ -96,6 +103,7 @@ impl Default for BuildOptions { manifest_path: PathBuf::from("Cargo.toml"), out: None, skip_auditwheel: false, + zig: false, target: None, cargo_extra_args: Vec::new(), rustc_extra_args: Vec::new(), @@ -271,14 +279,25 @@ impl BuildOptions { let strip = pyproject.map(|x| x.strip()).unwrap_or_default() || strip; let skip_auditwheel = pyproject.map(|x| x.skip_auditwheel()).unwrap_or_default() || self.skip_auditwheel; - let platform_tag = self.platform_tag.or_else(|| { - pyproject.and_then(|x| { - if x.compatibility().is_some() { - args_from_pyproject.push("compatibility"); - } - x.compatibility() + let platform_tag = self + .platform_tag + .or_else(|| { + pyproject.and_then(|x| { + if x.compatibility().is_some() { + args_from_pyproject.push("compatibility"); + } + x.compatibility() + }) }) - }); + .or( + // With zig we can compile to any glibc version that we want, so we pick the lowest + // one supported by the rust compiler + if self.zig { + Some(PlatformTag::manylinux2010()) + } else { + None + }, + ); if platform_tag == Some(PlatformTag::manylinux1()) { eprintln!("⚠️ Warning: manylinux1 is unsupported by the Rust compiler."); } @@ -302,6 +321,7 @@ impl BuildOptions { release, strip, skip_auditwheel, + zig: self.zig, platform_tag, cargo_extra_args, rustc_extra_args, diff --git a/src/compile.rs b/src/compile.rs index 1dbe5fd0b..1986afe21 100644 --- a/src/compile.rs +++ b/src/compile.rs @@ -1,13 +1,17 @@ use crate::build_context::BridgeModel; use crate::python_interpreter::InterpreterKind; -use crate::BuildContext; use crate::PythonInterpreter; +use crate::{BuildContext, PlatformTag}; use anyhow::{anyhow, bail, Context, Result}; use fat_macho::FatWriter; use fs_err::{self as fs, File}; use std::collections::HashMap; use std::env; -use std::io::{BufReader, Read}; +#[cfg(target_family = "unix")] +use std::fs::OpenOptions; +use std::io::{BufReader, Read, Write}; +#[cfg(target_family = "unix")] +use std::os::unix::fs::OpenOptionsExt; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use std::str; @@ -101,11 +105,70 @@ fn compile_universal2( Ok(result) } +/// We want to use `zig cc` as linker and c compiler. We want to call `python -m ziglang cc`, but +/// cargo only accepts a path to an executable as linker, so we add a wrapper script. We then also +/// use the wrapper script to pass arguments and substitute an unsupported argument. +/// +/// We create different files for different args because otherwise cargo might skip recompiling even +/// if the linker target changed +fn prepare_zig_linker(context: &BuildContext) -> Result { + let (zig_linker, target) = match context.platform_tag { + None | Some(PlatformTag::Linux) => ( + "./zigcc-gnu.sh".to_string(), + "native-native-gnu".to_string(), + ), + Some(PlatformTag::Musllinux { x, y }) => ( + format!("./zigcc-musl-{}-{}.sh", x, y), + format!("native-native-musl.{}.{}", x, y), + ), + Some(PlatformTag::Manylinux { x, y }) => ( + format!("./zigcc-gnu-{}-{}.sh", x, y), + format!("native-native-gnu.{}.{}", x, y), + ), + }; + + let zig_linker_dir = dirs::cache_dir() + // If the really is no cache dir, cwd will also do + .unwrap_or_else(|| PathBuf::from(".")) + .join(env!("CARGO_PKG_NAME")) + .join(env!("CARGO_PKG_VERSION")); + fs::create_dir_all(&zig_linker_dir)?; + let zig_linker = zig_linker_dir.join(zig_linker); + + #[cfg(target_family = "unix")] + let mut custom_linker_file = OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .mode(0o700) + .open(&zig_linker)?; + #[cfg(not(target_family = "unix"))] + let mut custom_linker_file = File::create(&zig_linker)?; + // https://github.com/ziglang/zig/issues/10050#issuecomment-956204098 + custom_linker_file.write_all( + format!( + r##"#!/bin/bash +python -m ziglang cc ${{@/-lgcc_s/-lunwind}} -target {} +"##, + target + ) + .as_bytes(), + )?; + drop(custom_linker_file); + Ok(zig_linker) +} + fn compile_target( context: &BuildContext, python_interpreter: Option<&PythonInterpreter>, bindings_crate: &BridgeModel, ) -> Result> { + let zig_linker = if context.zig && context.target.is_linux() { + Some(prepare_zig_linker(context).context("Failed to create zig linker wrapper")?) + } else { + None + }; + let mut shared_args = vec!["--manifest-path", context.manifest_path.to_str().unwrap()]; shared_args.extend(context.cargo_extra_args.iter().map(String::as_str)); @@ -207,6 +270,16 @@ fn compile_target( // but forwarding stderr is still useful in case there some non-json error .stderr(Stdio::inherit()); + if let Some(zig_linker) = zig_linker { + build_command.env("CC", &zig_linker); + let env_target = context + .target + .target_triple() + .to_uppercase() + .replace("-", "_"); + build_command.env(format!("CARGO_TARGET_{}_LINKER", env_target), &zig_linker); + } + if let BridgeModel::BindingsAbi3(_, _) = bindings_crate { let is_pypy = python_interpreter .map(|p| p.interpreter_kind == InterpreterKind::PyPy) diff --git a/src/develop.rs b/src/develop.rs index 0cde44cd1..acf8c905a 100644 --- a/src/develop.rs +++ b/src/develop.rs @@ -35,6 +35,7 @@ pub fn develop( manifest_path: manifest_file.to_path_buf(), out: Some(wheel_dir.path().to_path_buf()), skip_auditwheel: false, + zig: false, target: None, cargo_extra_args, rustc_extra_args, @@ -113,7 +114,7 @@ pub fn develop( // Y U NO accept windows path prefix, pip? // Anyways, here's shepmasters stack overflow solution // https://stackoverflow.com/a/50323079/3549270 -#[cfg(not(target_os = "windows"))] +#[cfg(target_family = "unix")] fn adjust_canonicalization(p: impl AsRef) -> String { p.as_ref().display().to_string() } diff --git a/src/module_writer.rs b/src/module_writer.rs index 652acfdd5..74393c495 100644 --- a/src/module_writer.rs +++ b/src/module_writer.rs @@ -12,11 +12,11 @@ use fs_err::File; use sha2::{Digest, Sha256}; use std::collections::HashMap; use std::ffi::OsStr; -#[cfg(not(target_os = "windows"))] +#[cfg(target_family = "unix")] use std::fs::OpenOptions; use std::io; use std::io::{Read, Write}; -#[cfg(not(target_os = "windows"))] +#[cfg(target_family = "unix")] use std::os::unix::fs::OpenOptionsExt; use std::path::{Path, PathBuf}; use std::process::{Command, Output}; @@ -164,7 +164,7 @@ impl ModuleWriter for PathWriter { // We only need to set the executable bit on unix let mut file = { - #[cfg(not(target_os = "windows"))] + #[cfg(target_family = "unix")] { OpenOptions::new() .create(true) diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 6a7967cdf..2a8e24479 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -15,7 +15,7 @@ pub mod other; // Y U NO accept windows path prefix, pip? // Anyways, here's shepmasters stack overflow solution // https://stackoverflow.com/a/50323079/3549270 -#[cfg(not(target_os = "windows"))] +#[cfg(target_family = "unix")] pub fn adjust_canonicalization(p: impl AsRef) -> String { p.as_ref().display().to_string() } From 2fe49d86fe12e68890c58684400b9c2f7a60e319 Mon Sep 17 00:00:00 2001 From: konstin Date: Mon, 27 Dec 2021 22:36:51 +0900 Subject: [PATCH 2/8] Add test for zig and small fixes --- src/build_options.rs | 2 +- src/compile.rs | 32 ++++++++++++++++---------------- tests/common/integration.rs | 16 +++++++++++++--- tests/run.rs | 7 +++++++ 4 files changed, 37 insertions(+), 20 deletions(-) diff --git a/src/build_options.rs b/src/build_options.rs index 1f2bf2c94..bb9e854b8 100644 --- a/src/build_options.rs +++ b/src/build_options.rs @@ -292,7 +292,7 @@ impl BuildOptions { .or( // With zig we can compile to any glibc version that we want, so we pick the lowest // one supported by the rust compiler - if self.zig { + if self.zig && !target.is_musl_target() { Some(PlatformTag::manylinux2010()) } else { None diff --git a/src/compile.rs b/src/compile.rs index 1986afe21..501661796 100644 --- a/src/compile.rs +++ b/src/compile.rs @@ -112,18 +112,26 @@ fn compile_universal2( /// We create different files for different args because otherwise cargo might skip recompiling even /// if the linker target changed fn prepare_zig_linker(context: &BuildContext) -> Result { - let (zig_linker, target) = match context.platform_tag { + let (zig_linker, cc_args) = match context.platform_tag { + // Not sure branch even has any use case, but it doesn't hurt to support it None | Some(PlatformTag::Linux) => ( "./zigcc-gnu.sh".to_string(), "native-native-gnu".to_string(), ), - Some(PlatformTag::Musllinux { x, y }) => ( - format!("./zigcc-musl-{}-{}.sh", x, y), - format!("native-native-musl.{}.{}", x, y), - ), + Some(PlatformTag::Musllinux { x, y }) => { + println!("⚠️ Warning: zig with musl is unstable"); + ( + format!("./zigcc-musl-{}-{}.sh", x, y), + format!("-target native-native-musl.{}.{}", x, y), + ) + } Some(PlatformTag::Manylinux { x, y }) => ( format!("./zigcc-gnu-{}-{}.sh", x, y), - format!("native-native-gnu.{}.{}", x, y), + // https://github.com/ziglang/zig/issues/10050#issuecomment-956204098 + format!( + "${{@/-lgcc_s/-lunwind}} -target native-native-gnu.{}.{}", + x, y + ), ), }; @@ -144,16 +152,8 @@ fn prepare_zig_linker(context: &BuildContext) -> Result { .open(&zig_linker)?; #[cfg(not(target_family = "unix"))] let mut custom_linker_file = File::create(&zig_linker)?; - // https://github.com/ziglang/zig/issues/10050#issuecomment-956204098 - custom_linker_file.write_all( - format!( - r##"#!/bin/bash -python -m ziglang cc ${{@/-lgcc_s/-lunwind}} -target {} -"##, - target - ) - .as_bytes(), - )?; + writeln!(&mut custom_linker_file, "#!/bin/bash")?; + writeln!(&mut custom_linker_file, "python -m ziglang cc {}", cc_args)?; drop(custom_linker_file); Ok(zig_linker) } diff --git a/tests/common/integration.rs b/tests/common/integration.rs index 1c9934522..58ca73b51 100644 --- a/tests/common/integration.rs +++ b/tests/common/integration.rs @@ -12,6 +12,7 @@ pub fn test_integration( package: impl AsRef, bindings: Option, unique_name: &str, + zig: bool, ) -> Result<()> { maybe_mock_cargo(); @@ -32,8 +33,6 @@ pub fn test_integration( &cargo_extra_args, "--out", &shed, - "--compatibility", - "linux", ]; if let Some(ref bindings) = bindings { @@ -41,8 +40,14 @@ pub fn test_integration( cli.push(bindings); } - let options: BuildOptions = BuildOptions::from_iter_safe(cli)?; + if zig { + cli.push("--zig") + } else { + cli.push("--compatibility"); + cli.push("linux"); + } + let options: BuildOptions = BuildOptions::from_iter_safe(cli)?; let build_context = options.into_build_context(false, cfg!(feature = "faster-tests"), false)?; let wheels = build_context.build_wheels()?; @@ -70,6 +75,11 @@ pub fn test_integration( // We can do this since we know that wheels are built and returned in the // order they are in the build context for ((filename, supported_version), python_interpreter) in wheels.iter().zip(interpreter) { + if zig && build_context.target.is_linux() && !build_context.target.is_musl_target() { + assert!(filename + .to_string_lossy() + .ends_with("manylinux_2_12_x86_64.manylinux2010_x86_64.whl")) + } let venv_name = if supported_version == "py3" { format!("{}-cffi", test_name) } else { diff --git a/tests/run.rs b/tests/run.rs index 442eb2d86..4b8ac3c19 100644 --- a/tests/run.rs +++ b/tests/run.rs @@ -100,6 +100,7 @@ fn integration_pyo3_pure() { "test-crates/pyo3-pure", None, "integration_pyo3_pure", + false, )); } @@ -109,6 +110,7 @@ fn integration_pyo3_mixed() { "test-crates/pyo3-mixed", None, "integration_pyo3_mixed", + false, )); } @@ -118,6 +120,7 @@ fn integration_pyo3_mixed_submodule() { "test-crates/pyo3-mixed-submodule", None, "integration_pyo3_mixed_submodule", + false, )); } @@ -127,6 +130,7 @@ fn integration_pyo3_mixed_py_subdir() { "test-crates/pyo3-mixed-py-subdir", None, "integration_pyo3_mixed_py_subdir", + true, )); } @@ -146,6 +150,7 @@ fn integration_cffi_pure() { "test-crates/cffi-pure", None, "integration_cffi_pure", + false, )); } @@ -155,6 +160,7 @@ fn integration_cffi_mixed() { "test-crates/cffi-mixed", None, "integration_cffi_mixed", + false, )); } @@ -164,6 +170,7 @@ fn integration_hello_world() { "test-crates/hello-world", None, "integration_hello_world", + false, )); } From 24a64b07f78c5c1d3e9f439e7843668c5629c93b Mon Sep 17 00:00:00 2001 From: konstin Date: Tue, 28 Dec 2021 09:45:55 +0900 Subject: [PATCH 3/8] Install zig in CI and clear CI cache --- .github/workflows/test.yml | 2 +- src/compile.rs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0f8e75b4f..cd7628954 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -31,7 +31,7 @@ jobs: with: python-version: "3.10.0" - name: Install cffi and virtualenv - run: pip install cffi virtualenv + run: pip install cffi virtualenv ziglang~=0.9.0 - uses: actions-rs/toolchain@v1 id: rustup with: diff --git a/src/compile.rs b/src/compile.rs index 501661796..c7b547d0d 100644 --- a/src/compile.rs +++ b/src/compile.rs @@ -278,6 +278,7 @@ fn compile_target( .to_uppercase() .replace("-", "_"); build_command.env(format!("CARGO_TARGET_{}_LINKER", env_target), &zig_linker); + build_command.env("TARGET_CC", &zig_linker); } if let BridgeModel::BindingsAbi3(_, _) = bindings_crate { From 69d079eea347d5347ad22166df5f683541aeec2f Mon Sep 17 00:00:00 2001 From: messense Date: Tue, 28 Dec 2021 18:50:48 +0800 Subject: [PATCH 4/8] Add support for cross compiling with zig --- src/build_options.rs | 2 +- src/compile.rs | 30 +++++++++++++++++++----------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/src/build_options.rs b/src/build_options.rs index bb9e854b8..349689ed7 100644 --- a/src/build_options.rs +++ b/src/build_options.rs @@ -293,7 +293,7 @@ impl BuildOptions { // With zig we can compile to any glibc version that we want, so we pick the lowest // one supported by the rust compiler if self.zig && !target.is_musl_target() { - Some(PlatformTag::manylinux2010()) + Some(target.get_default_manylinux_tag()) } else { None }, diff --git a/src/compile.rs b/src/compile.rs index c7b547d0d..7babc4e8b 100644 --- a/src/compile.rs +++ b/src/compile.rs @@ -1,7 +1,7 @@ use crate::build_context::BridgeModel; use crate::python_interpreter::InterpreterKind; -use crate::PythonInterpreter; -use crate::{BuildContext, PlatformTag}; +use crate::target::Arch; +use crate::{BuildContext, PlatformTag, PythonInterpreter}; use anyhow::{anyhow, bail, Context, Result}; use fat_macho::FatWriter; use fs_err::{self as fs, File}; @@ -112,25 +112,34 @@ fn compile_universal2( /// We create different files for different args because otherwise cargo might skip recompiling even /// if the linker target changed fn prepare_zig_linker(context: &BuildContext) -> Result { + let target = &context.target; + let arch = if target.cross_compiling() { + if matches!(target.target_arch(), Arch::Armv7L) { + "armv7".to_string() + } else { + target.target_arch().to_string() + } + } else { + "native".to_string() + }; let (zig_linker, cc_args) = match context.platform_tag { // Not sure branch even has any use case, but it doesn't hurt to support it - None | Some(PlatformTag::Linux) => ( - "./zigcc-gnu.sh".to_string(), - "native-native-gnu".to_string(), - ), + None | Some(PlatformTag::Linux) => { + ("./zigcc-gnu.sh".to_string(), format!("{}-linux-gnu", arch)) + } Some(PlatformTag::Musllinux { x, y }) => { println!("⚠️ Warning: zig with musl is unstable"); ( format!("./zigcc-musl-{}-{}.sh", x, y), - format!("-target native-native-musl.{}.{}", x, y), + format!("{}-linux-musl", arch), ) } Some(PlatformTag::Manylinux { x, y }) => ( format!("./zigcc-gnu-{}-{}.sh", x, y), // https://github.com/ziglang/zig/issues/10050#issuecomment-956204098 format!( - "${{@/-lgcc_s/-lunwind}} -target native-native-gnu.{}.{}", - x, y + "${{@/-lgcc_s/-lunwind}} -target {}-linux-gnu.{}.{}", + arch, x, y ), ), }; @@ -271,14 +280,13 @@ fn compile_target( .stderr(Stdio::inherit()); if let Some(zig_linker) = zig_linker { - build_command.env("CC", &zig_linker); let env_target = context .target .target_triple() .to_uppercase() .replace("-", "_"); + build_command.env(format!("{}_CC", env_target), &zig_linker); build_command.env(format!("CARGO_TARGET_{}_LINKER", env_target), &zig_linker); - build_command.env("TARGET_CC", &zig_linker); } if let BridgeModel::BindingsAbi3(_, _) = bindings_crate { From def3b59e2ada61fb75816cb3781c4646da51a4e3 Mon Sep 17 00:00:00 2001 From: messense Date: Tue, 28 Dec 2021 23:37:56 +0800 Subject: [PATCH 5/8] Add a zig cross compiling test to CI --- .github/workflows/test.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cd7628954..c9a06e234 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -106,6 +106,13 @@ jobs: pypy3 -m pip install --force-reinstall --no-index --find-links test-crates/pyo3-pure/target/wheels pyo3-pure pypy3 -m pip install pytest pypy3 -m pytest test-crates/pyo3-pure/test_pyo3_pure.py + - uses: actions/setup-python@v2 + with: + python-version: "3.10.0" + - name: test cross compiling with zig + run: | + rustup target add aarch64-unknown-linux-gnu + cargo run -- build --no-sdist -i python -m test-crates/pyo3-pure/Cargo.toml --target aarch64-unknown-linux-gnu --zig test-auditwheel: name: Test Auditwheel From 7709b8beb6b215808e52af0fbe8412d6fe573842 Mon Sep 17 00:00:00 2001 From: messense Date: Wed, 29 Dec 2021 14:07:02 +0800 Subject: [PATCH 6/8] Add C++ support for `--zig` Set `TARGET_CC` and `TARGET_CXX` to support both cc-rs and cmake-rs --- src/compile.rs | 67 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 46 insertions(+), 21 deletions(-) diff --git a/src/compile.rs b/src/compile.rs index 7babc4e8b..bbf72cc0e 100644 --- a/src/compile.rs +++ b/src/compile.rs @@ -111,7 +111,7 @@ fn compile_universal2( /// /// We create different files for different args because otherwise cargo might skip recompiling even /// if the linker target changed -fn prepare_zig_linker(context: &BuildContext) -> Result { +fn prepare_zig_linker(context: &BuildContext) -> Result<(PathBuf, PathBuf)> { let target = &context.target; let arch = if target.cross_compiling() { if matches!(target.target_arch(), Arch::Armv7L) { @@ -122,20 +122,24 @@ fn prepare_zig_linker(context: &BuildContext) -> Result { } else { "native".to_string() }; - let (zig_linker, cc_args) = match context.platform_tag { + let (zig_cc, zig_cxx, cc_args) = match context.platform_tag { // Not sure branch even has any use case, but it doesn't hurt to support it - None | Some(PlatformTag::Linux) => { - ("./zigcc-gnu.sh".to_string(), format!("{}-linux-gnu", arch)) - } + None | Some(PlatformTag::Linux) => ( + "./zigcc-gnu.sh".to_string(), + "./zigcxx-gnu.sh".to_string(), + format!("{}-linux-gnu", arch), + ), Some(PlatformTag::Musllinux { x, y }) => { println!("⚠️ Warning: zig with musl is unstable"); ( format!("./zigcc-musl-{}-{}.sh", x, y), + format!("./zigcxx-musl-{}-{}.sh", x, y), format!("{}-linux-musl", arch), ) } Some(PlatformTag::Manylinux { x, y }) => ( format!("./zigcc-gnu-{}-{}.sh", x, y), + format!("./zigcxx-gnu-{}-{}.sh", x, y), // https://github.com/ziglang/zig/issues/10050#issuecomment-956204098 format!( "${{@/-lgcc_s/-lunwind}} -target {}-linux-gnu.{}.{}", @@ -150,21 +154,36 @@ fn prepare_zig_linker(context: &BuildContext) -> Result { .join(env!("CARGO_PKG_NAME")) .join(env!("CARGO_PKG_VERSION")); fs::create_dir_all(&zig_linker_dir)?; - let zig_linker = zig_linker_dir.join(zig_linker); + let zig_cc = zig_linker_dir.join(zig_cc); + let zig_cxx = zig_linker_dir.join(zig_cxx); + + let mut zig_cc_file = create_linker_script(&zig_cc)?; + writeln!(&mut zig_cc_file, "#!/bin/bash")?; + writeln!(&mut zig_cc_file, "python -m ziglang cc {}", cc_args)?; + drop(zig_cc_file); + + let mut zig_cxx_file = create_linker_script(&zig_cxx)?; + writeln!(&mut zig_cxx_file, "#!/bin/bash")?; + writeln!(&mut zig_cxx_file, "python -m ziglang c++ {}", cc_args)?; + drop(zig_cxx_file); + + Ok((zig_cc, zig_cxx)) +} - #[cfg(target_family = "unix")] - let mut custom_linker_file = OpenOptions::new() +#[cfg(target_family = "unix")] +fn create_linker_script(path: &Path) -> Result { + let custom_linker_file = OpenOptions::new() .create(true) .write(true) .truncate(true) .mode(0o700) - .open(&zig_linker)?; - #[cfg(not(target_family = "unix"))] - let mut custom_linker_file = File::create(&zig_linker)?; - writeln!(&mut custom_linker_file, "#!/bin/bash")?; - writeln!(&mut custom_linker_file, "python -m ziglang cc {}", cc_args)?; - drop(custom_linker_file); - Ok(zig_linker) + .open(path)?; + Ok(custom_linker_file) +} + +#[cfg(not(target_family = "unix"))] +fn create_linker_script(path: &Path) -> Result { + File::create(path) } fn compile_target( @@ -172,10 +191,12 @@ fn compile_target( python_interpreter: Option<&PythonInterpreter>, bindings_crate: &BridgeModel, ) -> Result> { - let zig_linker = if context.zig && context.target.is_linux() { - Some(prepare_zig_linker(context).context("Failed to create zig linker wrapper")?) + let (zig_cc, zig_cxx) = if context.zig && context.target.is_linux() { + let (cc, cxx) = + prepare_zig_linker(context).context("Failed to create zig linker wrapper")?; + (Some(cc), Some(cxx)) } else { - None + (None, None) }; let mut shared_args = vec!["--manifest-path", context.manifest_path.to_str().unwrap()]; @@ -279,14 +300,18 @@ fn compile_target( // but forwarding stderr is still useful in case there some non-json error .stderr(Stdio::inherit()); - if let Some(zig_linker) = zig_linker { + // Also set TARGET_CC and TARGET_CXX for cc-rs and cmake-rs + if let Some(zig_cc) = zig_cc { let env_target = context .target .target_triple() .to_uppercase() .replace("-", "_"); - build_command.env(format!("{}_CC", env_target), &zig_linker); - build_command.env(format!("CARGO_TARGET_{}_LINKER", env_target), &zig_linker); + build_command.env("TARGET_CC", &zig_cc); + build_command.env(format!("CARGO_TARGET_{}_LINKER", env_target), &zig_cc); + } + if let Some(zig_cxx) = zig_cxx { + build_command.env("TARGET_CXX", &zig_cxx); } if let BridgeModel::BindingsAbi3(_, _) = bindings_crate { From 566abf51dc92cc04d721259f6810f1cb6c687604 Mon Sep 17 00:00:00 2001 From: messense Date: Wed, 29 Dec 2021 14:16:46 +0800 Subject: [PATCH 7/8] Fix compiling on Windows Disable Windows zig test for now --- .github/workflows/test.yml | 1 + src/compile.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c9a06e234..cf0a836f4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -110,6 +110,7 @@ jobs: with: python-version: "3.10.0" - name: test cross compiling with zig + if: matrix.os != 'windows-latest' run: | rustup target add aarch64-unknown-linux-gnu cargo run -- build --no-sdist -i python -m test-crates/pyo3-pure/Cargo.toml --target aarch64-unknown-linux-gnu --zig diff --git a/src/compile.rs b/src/compile.rs index bbf72cc0e..48d509d3b 100644 --- a/src/compile.rs +++ b/src/compile.rs @@ -183,7 +183,7 @@ fn create_linker_script(path: &Path) -> Result { #[cfg(not(target_family = "unix"))] fn create_linker_script(path: &Path) -> Result { - File::create(path) + Ok(File::create(path)?) } fn compile_target( From fa00117e5def1597f893757499da3956ddf5dec7 Mon Sep 17 00:00:00 2001 From: konstin Date: Fri, 31 Dec 2021 20:10:11 +0900 Subject: [PATCH 8/8] Fixup --- src/build_options.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build_options.rs b/src/build_options.rs index 349689ed7..a51f99d75 100644 --- a/src/build_options.rs +++ b/src/build_options.rs @@ -293,7 +293,7 @@ impl BuildOptions { // With zig we can compile to any glibc version that we want, so we pick the lowest // one supported by the rust compiler if self.zig && !target.is_musl_target() { - Some(target.get_default_manylinux_tag()) + Some(target.get_minimum_manylinux_tag()) } else { None },