Skip to content

Commit

Permalink
Add a subcommand wrapper for zig cc and zig c++
Browse files Browse the repository at this point in the history
  • Loading branch information
messense committed Jan 2, 2022
1 parent 33be362 commit d4c06ec
Show file tree
Hide file tree
Showing 8 changed files with 231 additions and 90 deletions.
1 change: 0 additions & 1 deletion .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,6 @@ jobs:
with:
python-version: "3.10.0"
- name: test cross compiling with zig
if: matrix.os != 'windows-latest'
run: |
rustup target add aarch64-unknown-linux-gnu
cargo run -- build --no-sdist -i python -m test-crates/pyo3-pure/Cargo.toml --target aarch64-unknown-linux-gnu --zig
Expand Down
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ cc = "1.0.72"
clap = { version = "3.0.0", features = ["derive", "env", "wrap_help"] }
clap_complete = "3.0.0"
clap_complete_fig = "3.0.0"
semver = "1.0.4"

[dev-dependencies]
indoc = "1.0.3"
Expand Down
91 changes: 3 additions & 88 deletions src/compile.rs
Original file line number Diff line number Diff line change
@@ -1,17 +1,13 @@
use crate::build_context::BridgeModel;
use crate::python_interpreter::InterpreterKind;
use crate::target::Arch;
use crate::{BuildContext, PlatformTag, PythonInterpreter};
use crate::zig::prepare_zig_linker;
use crate::{BuildContext, PythonInterpreter};
use anyhow::{anyhow, bail, Context, Result};
use fat_macho::FatWriter;
use fs_err::{self as fs, File};
use std::collections::HashMap;
use std::env;
#[cfg(target_family = "unix")]
use std::fs::OpenOptions;
use std::io::{BufReader, Read, Write};
#[cfg(target_family = "unix")]
use std::os::unix::fs::OpenOptionsExt;
use std::io::{BufReader, Read};
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::str;
Expand Down Expand Up @@ -105,87 +101,6 @@ fn compile_universal2(
Ok(result)
}

/// We want to use `zig cc` as linker and c compiler. We want to call `python -m ziglang cc`, but
/// cargo only accepts a path to an executable as linker, so we add a wrapper script. We then also
/// use the wrapper script to pass arguments and substitute an unsupported argument.
///
/// We create different files for different args because otherwise cargo might skip recompiling even
/// if the linker target changed
fn prepare_zig_linker(context: &BuildContext) -> Result<(PathBuf, PathBuf)> {
let target = &context.target;
let arch = if target.cross_compiling() {
if matches!(target.target_arch(), Arch::Armv7L) {
"armv7".to_string()
} else {
target.target_arch().to_string()
}
} else {
"native".to_string()
};
let (zig_cc, zig_cxx, cc_args) = match context.platform_tag {
// Not sure branch even has any use case, but it doesn't hurt to support it
None | Some(PlatformTag::Linux) => (
"./zigcc-gnu.sh".to_string(),
"./zigcxx-gnu.sh".to_string(),
format!("{}-linux-gnu", arch),
),
Some(PlatformTag::Musllinux { x, y }) => {
println!("⚠️ Warning: zig with musl is unstable");
(
format!("./zigcc-musl-{}-{}.sh", x, y),
format!("./zigcxx-musl-{}-{}.sh", x, y),
format!("{}-linux-musl", arch),
)
}
Some(PlatformTag::Manylinux { x, y }) => (
format!("./zigcc-gnu-{}-{}.sh", x, y),
format!("./zigcxx-gnu-{}-{}.sh", x, y),
// https://github.com/ziglang/zig/issues/10050#issuecomment-956204098
format!(
"${{@/-lgcc_s/-lunwind}} -target {}-linux-gnu.{}.{}",
arch, x, y
),
),
};

let zig_linker_dir = dirs::cache_dir()
// If the really is no cache dir, cwd will also do
.unwrap_or_else(|| PathBuf::from("."))
.join(env!("CARGO_PKG_NAME"))
.join(env!("CARGO_PKG_VERSION"));
fs::create_dir_all(&zig_linker_dir)?;
let zig_cc = zig_linker_dir.join(zig_cc);
let zig_cxx = zig_linker_dir.join(zig_cxx);

let mut zig_cc_file = create_linker_script(&zig_cc)?;
writeln!(&mut zig_cc_file, "#!/bin/bash")?;
writeln!(&mut zig_cc_file, "python -m ziglang cc {}", cc_args)?;
drop(zig_cc_file);

let mut zig_cxx_file = create_linker_script(&zig_cxx)?;
writeln!(&mut zig_cxx_file, "#!/bin/bash")?;
writeln!(&mut zig_cxx_file, "python -m ziglang c++ {}", cc_args)?;
drop(zig_cxx_file);

Ok((zig_cc, zig_cxx))
}

#[cfg(target_family = "unix")]
fn create_linker_script(path: &Path) -> Result<std::fs::File> {
let custom_linker_file = OpenOptions::new()
.create(true)
.write(true)
.truncate(true)
.mode(0o700)
.open(path)?;
Ok(custom_linker_file)
}

#[cfg(not(target_family = "unix"))]
fn create_linker_script(path: &Path) -> Result<File> {
Ok(File::create(path)?)
}

fn compile_target(
context: &BuildContext,
python_interpreter: Option<&PythonInterpreter>,
Expand Down
2 changes: 2 additions & 0 deletions src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ pub use crate::new_project::{init_project, new_project, GenerateProjectOptions};
pub use crate::pyproject_toml::PyProjectToml;
pub use crate::python_interpreter::PythonInterpreter;
pub use crate::target::Target;
pub use crate::zig::Zig;
pub use auditwheel::PlatformTag;
#[cfg(feature = "upload")]
pub use {
Expand All @@ -62,3 +63,4 @@ mod source_distribution;
mod target;
#[cfg(feature = "upload")]
mod upload;
mod zig;
8 changes: 7 additions & 1 deletion src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ use clap::{ArgEnum, IntoApp, Parser};
use clap_complete::Generator;
use maturin::{
develop, init_project, new_project, write_dist_info, BridgeModel, BuildOptions,
GenerateProjectOptions, PathWriter, PlatformTag, PythonInterpreter, Target,
GenerateProjectOptions, PathWriter, PlatformTag, PythonInterpreter, Target, Zig,
};
#[cfg(feature = "upload")]
use maturin::{upload_ui, PublishOpt};
Expand Down Expand Up @@ -163,6 +163,9 @@ enum Opt {
#[clap(name = "SHELL", parse(try_from_str))]
shell: Shell,
},
/// Zig linker wrapper
#[clap(subcommand)]
Zig(Zig),
}

/// Backend for the PEP 517 integration. Not for human consumption
Expand Down Expand Up @@ -449,6 +452,9 @@ fn run() -> Result<()> {
}
}
}
Opt::Zig(subcommand) => {
subcommand.execute()?;
}
}

Ok(())
Expand Down
214 changes: 214 additions & 0 deletions src/zig.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,214 @@
use crate::target::Arch;
use crate::{BuildContext, PlatformTag};
use anyhow::{bail, Context, Result};
use fs_err as fs;
use std::env;
#[cfg(target_family = "unix")]
use std::fs::OpenOptions;
use std::io::Write;
#[cfg(target_family = "unix")]
use std::os::unix::fs::OpenOptionsExt;
use std::path::{Path, PathBuf};
use std::process::{self, Command};
use std::str;

/// Zig linker wrapper
#[derive(Debug, clap::Parser)]
#[clap(name = "zig", setting = clap::AppSettings::Hidden)]
pub enum Zig {
/// `zig cc` wrapper
#[clap(name = "cc", setting = clap::AppSettings::TrailingVarArg)]
Cc {
/// `zig cc` arguments
#[clap(takes_value = true, multiple_values = true)]
args: Vec<String>,
},
/// `zig c++` wrapper
#[clap(name = "c++", setting = clap::AppSettings::TrailingVarArg)]
Cxx {
/// `zig c++` arguments
#[clap(takes_value = true, multiple_values = true)]
args: Vec<String>,
},
}

impl Zig {
/// Execute the underlying zig command
pub fn execute(&self) -> Result<()> {
let (cmd, cmd_args) = match self {
Zig::Cc { args } => ("cc", args),
Zig::Cxx { args } => ("c++", args),
};
// Replace libgcc_s with libunwind
let cmd_args: Vec<String> = cmd_args
.iter()
.map(|arg| {
let arg = if arg == "-lgcc_s" {
"-lunwind".to_string()
} else if arg.starts_with('@') && arg.ends_with("linker-arguments") {
// rustc passes arguments to linker via an @-file when arguments are too long
let content = fs::read(arg.trim_start_matches('@'))?;
let link_args = str::from_utf8(&content)?.replace("-lgcc_s", "-lunwind");
fs::write(arg.trim_start_matches('@'), link_args.as_bytes())?;
arg.to_string()
} else {
arg.to_string()
};
Ok(arg)
})
.collect::<Result<_>>()?;
let (zig, zig_args) = Self::find_zig()?;
let mut child = Command::new(zig)
.args(zig_args)
.arg(cmd)
.args(cmd_args)
.spawn()
.with_context(|| format!("Failed to run `zig {}`", cmd))?;
let status = child.wait().expect("Failed to wait on zig child process");
if !status.success() {
process::exit(status.code().unwrap_or(1));
}
Ok(())
}

fn find_zig() -> Result<(String, Vec<String>)> {
Self::find_zig_python()
.or_else(|_| Self::find_zig_bin())
.context("Failed to find zig")
}

/// Detect the plain zig binary
fn find_zig_bin() -> Result<(String, Vec<String>)> {
let output = Command::new("zig").arg("version").output()?;
let version_str =
str::from_utf8(&output.stdout).context("`zig version` didn't return utf8 output")?;
Self::validate_zig_version(version_str)?;
Ok(("zig".to_string(), Vec::new()))
}

/// Detect the Python ziglang package
fn find_zig_python() -> Result<(String, Vec<String>)> {
let output = Command::new("python3")
.args(&["-m", "ziglang", "version"])
.output()?;
let version_str = str::from_utf8(&output.stdout)
.context("`python3 -m ziglang version` didn't return utf8 output")?;
Self::validate_zig_version(version_str)?;
Ok((
"python3".to_string(),
vec!["-m".to_string(), "ziglang".to_string()],
))
}

fn validate_zig_version(version: &str) -> Result<()> {
let min_ver = semver::Version::new(0, 9, 0);
let version = semver::Version::parse(version.trim())?;
if version >= min_ver {
Ok(())
} else {
bail!(
"zig version {} is too old, need at least {}",
version,
min_ver
)
}
}
}

/// We want to use `zig cc` as linker and c compiler. We want to call `python -m ziglang cc`, but
/// cargo only accepts a path to an executable as linker, so we add a wrapper script. We then also
/// use the wrapper script to pass arguments and substitute an unsupported argument.
///
/// We create different files for different args because otherwise cargo might skip recompiling even
/// if the linker target changed
pub fn prepare_zig_linker(context: &BuildContext) -> Result<(PathBuf, PathBuf)> {
let target = &context.target;
let arch = if target.cross_compiling() {
if matches!(target.target_arch(), Arch::Armv7L) {
"armv7".to_string()
} else {
target.target_arch().to_string()
}
} else {
"native".to_string()
};
let file_ext = if cfg!(windows) { "bat" } else { "sh" };
let (zig_cc, zig_cxx, cc_args) = match context.platform_tag {
// Not sure branch even has any use case, but it doesn't hurt to support it
None | Some(PlatformTag::Linux) => (
format!("zigcc-gnu.{}", file_ext),
format!("zigcxx-gnu.{}", file_ext),
format!("-target {}-linux-gnu", arch),
),
Some(PlatformTag::Musllinux { x, y }) => {
println!("⚠️ Warning: zig with musl is unstable");
(
format!("zigcc-musl-{}-{}.{}", x, y, file_ext),
format!("zigcxx-musl-{}-{}.{}", x, y, file_ext),
format!("-target {}-linux-musl", arch),
)
}
Some(PlatformTag::Manylinux { x, y }) => (
format!("zigcc-gnu-{}-{}.{}", x, y, file_ext),
format!("zigcxx-gnu-{}-{}.{}", x, y, file_ext),
// https://github.com/ziglang/zig/issues/10050#issuecomment-956204098
format!("-target {}-linux-gnu.{}.{}", arch, x, y),
),
};

let zig_linker_dir = dirs::cache_dir()
// If the really is no cache dir, cwd will also do
.unwrap_or_else(|| env::current_dir().expect("Failed to get current dir"))
.join(env!("CARGO_PKG_NAME"))
.join(env!("CARGO_PKG_VERSION"));
fs::create_dir_all(&zig_linker_dir)?;

let zig_cc = zig_linker_dir.join(zig_cc);
let zig_cxx = zig_linker_dir.join(zig_cxx);
write_linker_script(&zig_cc, "cc", &cc_args)?;
write_linker_script(&zig_cxx, "c++", &cc_args)?;

Ok((zig_cc, zig_cxx))
}

#[cfg(target_family = "unix")]
fn write_linker_script(path: &Path, command: &str, args: &str) -> Result<()> {
let mut custom_linker_file = OpenOptions::new()
.create(true)
.write(true)
.truncate(true)
.mode(0o700)
.open(path)?;
let current_exe = if let Ok(maturin) = env::var("CARGO_BIN_EXE_maturin") {
PathBuf::from(maturin)
} else {
env::current_exe()?
};
writeln!(&mut custom_linker_file, "#!/bin/bash")?;
writeln!(
&mut custom_linker_file,
"{} zig {} -- {} $@",
current_exe.display(),
command,
args
)?;
Ok(())
}

#[cfg(not(target_family = "unix"))]
fn write_linker_script(path: &Path, command: &str, args: &str) -> Result<()> {
let mut custom_linker_file = fs::File::create(path)?;
let current_exe = if let Ok(maturin) = env::var("CARGO_BIN_EXE_maturin") {
PathBuf::from(maturin)
} else {
env::current_exe()?
};
writeln!(
&mut custom_linker_file,
"{} zig {} -- {} %*",
current_exe.display(),
command,
args
)?;
Ok(())
}
3 changes: 3 additions & 0 deletions tests/common/integration.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ pub fn test_integration(
) -> Result<()> {
maybe_mock_cargo();

// Pass CARGO_BIN_EXE_maturin for testing purpose
std::env::set_var("CARGO_BIN_EXE_maturin", env!("CARGO_BIN_EXE_maturin"));

let target = Target::from_target_triple(None)?;

let package_string = package.as_ref().join("Cargo.toml").display().to_string();
Expand Down

0 comments on commit d4c06ec

Please sign in to comment.