Skip to content

Commit

Permalink
Merge pull request #742 from messense/patchelf
Browse files Browse the repository at this point in the history
Implement auditwheel repair with patchelf
  • Loading branch information
messense authored Dec 19, 2021
2 parents c45e6b1 + 1af539a commit fb112e3
Show file tree
Hide file tree
Showing 9 changed files with 414 additions and 60 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ jobs:
python-version: "3.9"
- uses: actions/setup-python@v2
with:
python-version: "3.10"
python-version: "3.10.0"
- name: Install cffi and virtualenv
run: pip install cffi virtualenv
- uses: actions-rs/toolchain@v1
Expand Down
12 changes: 12 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ ignore = "0.4.18"
dialoguer = "0.9.0"
console = "0.15.0"
minijinja = "0.8.2"
lddtree = "0.1.4"

[dev-dependencies]
indoc = "1.0.3"
Expand Down
1 change: 1 addition & 0 deletions Changelog.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
* Fix undefined auditwheel policy panic in [#740](https://github.com/PyO3/maturin/pull/740)
* Fix sdist upload for packages where the pkgname contains multiple underscores in [#741](https://github.com/PyO3/maturin/pull/741)
* Add `Cargo.lock` to sdist when `--locked` or `--frozen` specified in [#749](https://github.com/PyO3/maturin/pull/749)
* Implement auditwheel repair with patchelf in [#742](https://github.com/PyO3/maturin/pull/742)

## [0.12.4] - 2021-12-06

Expand Down
100 changes: 75 additions & 25 deletions src/auditwheel/audit.rs
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,18 @@ pub enum AuditWheelError {
#[error(
"Your library is not {0} compliant because it links the following forbidden libraries: {1:?}",
)]
PlatformTagValidationError(Policy, Vec<String>),
/// The elf file isn't manylinux/musllinux compatible. Contains unsupported architecture
LinksForbiddenLibrariesError(Policy, Vec<String>),
/// The elf file isn't manylinux/musllinux compatible. Contains the list of offending
/// libraries.
#[error(
"Your library is not {0} compliant because of the presence of too-recent versioned symbols: {1:?}. Consider building in a manylinux docker container",
)]
VersionedSymbolTooNewError(Policy, Vec<String>),
/// The elf file isn't manylinux/musllinux compatible. Contains the list of offending
/// libraries with blacked-list symbols.
#[error("Your library is not {0} compliant because it depends on black-listed symbols: {1:?}")]
BlackListedSymbolsError(Policy, Vec<String>),
/// The elf file isn't manylinux/musllinux compaible. Contains unsupported architecture
#[error("Your library is not {0} compliant because it has unsupported architecture: {1}")]
UnsupportedArchitecture(Policy, String),
/// This platform tag isn't defined by auditwheel yet
Expand All @@ -43,15 +53,15 @@ pub enum AuditWheelError {
}

#[derive(Clone, Debug)]
struct VersionedLibrary {
pub struct VersionedLibrary {
/// library name
name: String,
pub name: String,
/// versions needed
versions: HashSet<String>,
}

/// Find required dynamic linked libraries with version information
fn find_versioned_libraries(elf: &Elf) -> Result<Vec<VersionedLibrary>, AuditWheelError> {
pub fn find_versioned_libraries(elf: &Elf) -> Vec<VersionedLibrary> {
let mut symbols = Vec::new();
if let Some(verneed) = &elf.verneed {
for need_file in verneed.iter() {
Expand All @@ -73,7 +83,7 @@ fn find_versioned_libraries(elf: &Elf) -> Result<Vec<VersionedLibrary>, AuditWhe
}
}
}
Ok(symbols)
symbols
}

/// Find incompliant symbols from symbol versions
Expand Down Expand Up @@ -107,6 +117,7 @@ fn policy_is_satisfied(
AuditWheelError::UnsupportedArchitecture(policy.clone(), arch.to_string())
})?;
let mut offending_libs = HashSet::new();
let mut offending_versioned_syms = HashSet::new();
let mut offending_blacklist_syms = HashMap::new();
let undef_symbols: HashSet<String> = elf
.dynsyms
Expand Down Expand Up @@ -173,26 +184,37 @@ fn policy_is_satisfied(
offending_symbols.join(", ")
)
};
offending_libs.insert(offender);
offending_versioned_syms.insert(offender);
}
}
}
// Checks if we can give a more helpful error message
let is_libpython = Regex::new(r"^libpython3\.\d+\.so\.\d+\.\d+$").unwrap();
let mut offenders: Vec<String> = offending_libs.into_iter().collect();
for (lib, syms) in offending_blacklist_syms {
offenders.push(format!(
"{} offending black-listed symbols: {}",
lib,
syms.join(", ")
// Check for black-listed symbols
if !offending_blacklist_syms.is_empty() {
let offenders = offending_blacklist_syms
.into_iter()
.map(|(lib, syms)| format!("{}: {}", lib, syms.join(", ")))
.collect();
return Err(AuditWheelError::BlackListedSymbolsError(
policy.clone(),
offenders,
));
}
// Check for too-recent versioned symbols
if !offending_versioned_syms.is_empty() {
return Err(AuditWheelError::VersionedSymbolTooNewError(
policy.clone(),
offending_versioned_syms.into_iter().collect(),
));
}
// Check for libpython and forbidden libraries
let is_libpython = Regex::new(r"^libpython3\.\d+\.so\.\d+\.\d+$").unwrap();
let offenders: Vec<String> = offending_libs.into_iter().collect();
match offenders.as_slice() {
[] => Ok(()),
[lib] if is_libpython.is_match(lib) => {
Err(AuditWheelError::LinksLibPythonError(lib.clone()))
}
offenders => Err(AuditWheelError::PlatformTagValidationError(
offenders => Err(AuditWheelError::LinksForbiddenLibrariesError(
policy.clone(),
offenders.to_vec(),
)),
Expand All @@ -218,8 +240,9 @@ fn get_default_platform_policies() -> Vec<Policy> {
/// An reimplementation of auditwheel, which checks elf files for
/// manylinux/musllinux compliance.
///
/// If `platform_tag`, is None, it returns the the highest matching manylinux/musllinux policy, or `linux`
/// if nothing else matches. It will error for bogus cases, e.g. if libpython is linked.
/// If `platform_tag`, is None, it returns the the highest matching manylinux/musllinux policy
/// and whether we need to repair with patchelf,, or `linux` if nothing else matches.
/// It will error for bogus cases, e.g. if libpython is linked.
///
/// If a specific manylinux/musllinux version is given, compliance is checked and a warning printed if
/// a higher version would be possible.
Expand All @@ -229,10 +252,11 @@ pub fn auditwheel_rs(
path: &Path,
target: &Target,
platform_tag: Option<PlatformTag>,
) -> Result<Policy, AuditWheelError> {
) -> Result<(Policy, bool), AuditWheelError> {
if !target.is_linux() || platform_tag == Some(PlatformTag::Linux) {
return Ok(Policy::default());
return Ok((Policy::default(), false));
}
let cross_compiling = target.cross_compiling();
let arch = target.target_arch().to_string();
let mut file = File::open(path).map_err(AuditWheelError::IoError)?;
let mut buffer = Vec::new();
Expand All @@ -241,7 +265,7 @@ pub fn auditwheel_rs(
let elf = Elf::parse(&buffer).map_err(AuditWheelError::GoblinError)?;
// This returns essentially the same as ldd
let deps: Vec<String> = elf.libraries.iter().map(ToString::to_string).collect();
let versioned_libraries = find_versioned_libraries(&elf)?;
let versioned_libraries = find_versioned_libraries(&elf);

// Find the highest possible policy, if any
let platform_policies = match platform_tag {
Expand All @@ -267,23 +291,36 @@ pub fn auditwheel_rs(
Some(PlatformTag::Linux) => unreachable!(),
};
let mut highest_policy = None;
let mut should_repair = false;
for policy in platform_policies.iter() {
let result = policy_is_satisfied(policy, &elf, &arch, &deps, &versioned_libraries);
match result {
Ok(_) => {
highest_policy = Some(policy.clone());
should_repair = false;
break;
}
Err(err @ AuditWheelError::LinksForbiddenLibrariesError(..)) => {
// TODO: support repair for cross compiled wheels
if !cross_compiling {
highest_policy = Some(policy.clone());
should_repair = true;
break;
} else {
return Err(err);
}
}
Err(AuditWheelError::VersionedSymbolTooNewError(..))
| Err(AuditWheelError::BlackListedSymbolsError(..))
// UnsupportedArchitecture happens when trying 2010 with aarch64
Err(AuditWheelError::PlatformTagValidationError(_, _))
| Err(AuditWheelError::UnsupportedArchitecture(..)) => continue,
// If there was an error parsing the symbols or libpython was linked,
// we error no matter what the requested policy was
Err(err) => return Err(err),
}
}

if let Some(platform_tag) = platform_tag {
let policy = if let Some(platform_tag) = platform_tag {
let tag = platform_tag.to_string();
let mut policy = Policy::from_name(&tag).ok_or(AuditWheelError::UndefinedPolicy(tag))?;
policy.fixup_musl_libc_so_name(target.target_arch());
Expand All @@ -299,7 +336,19 @@ pub fn auditwheel_rs(
}

match policy_is_satisfied(&policy, &elf, &arch, &deps, &versioned_libraries) {
Ok(_) => Ok(policy),
Ok(_) => {
should_repair = false;
Ok(policy)
}
Err(err @ AuditWheelError::LinksForbiddenLibrariesError(..)) => {
// TODO: support repair for cross compiled wheels
if !cross_compiling {
should_repair = true;
Ok(policy)
} else {
Err(err)
}
}
Err(err) => Err(err),
}
} else if let Some(policy) = highest_policy {
Expand All @@ -312,5 +361,6 @@ pub fn auditwheel_rs(

// Fallback to linux
Ok(Policy::default())
}
}?;
Ok((policy, should_repair))
}
5 changes: 4 additions & 1 deletion src/auditwheel/mod.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
mod audit;
mod musllinux;
pub mod patchelf;
mod platform_tag;
mod policy;
mod repair;

pub use self::audit::*;
pub use audit::*;
pub use platform_tag::PlatformTag;
pub use policy::{Policy, MANYLINUX_POLICIES, MUSLLINUX_POLICIES};
pub use repair::{get_external_libs, hash_file};
96 changes: 96 additions & 0 deletions src/auditwheel/patchelf.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
use anyhow::{bail, Context, Result};
use std::ffi::OsStr;
use std::path::Path;
use std::process::Command;

/// Replace a declared dependency on a dynamic library with another one (`DT_NEEDED`)
pub fn replace_needed<S: AsRef<OsStr>>(
file: impl AsRef<Path>,
old_lib: &str,
new_lib: &S,
) -> Result<()> {
let mut cmd = Command::new("patchelf");
cmd.arg("--replace-needed")
.arg(old_lib)
.arg(new_lib)
.arg(file.as_ref());
let output = cmd
.output()
.context("Failed to execute 'patchelf', did you install it?")?;
if !output.status.success() {
bail!(
"patchelf --replace-needed failed: {}",
String::from_utf8_lossy(&output.stderr)
);
}
Ok(())
}

/// Change `SONAME` of a dynamic library
pub fn set_soname<S: AsRef<OsStr>>(file: impl AsRef<Path>, soname: &S) -> Result<()> {
let mut cmd = Command::new("patchelf");
cmd.arg("--set-soname").arg(soname).arg(file.as_ref());
let output = cmd
.output()
.context("Failed to execute 'patchelf', did you install it?")?;
if !output.status.success() {
bail!(
"patchelf --set-soname failed: {}",
String::from_utf8_lossy(&output.stderr)
);
}
Ok(())
}

/// /// Remove a `RPATH` from executables and libraries
pub fn remove_rpath(file: impl AsRef<Path>) -> Result<()> {
let mut cmd = Command::new("patchelf");
cmd.arg("--remove-rpath").arg(file.as_ref());
let output = cmd
.output()
.context("Failed to execute 'patchelf', did you install it?")?;
if !output.status.success() {
bail!(
"patchelf --remove-rpath failed: {}",
String::from_utf8_lossy(&output.stderr)
);
}
Ok(())
}

/// Change the `RPATH` of executables and libraries
pub fn set_rpath<S: AsRef<OsStr>>(file: impl AsRef<Path>, rpath: &S) -> Result<()> {
remove_rpath(&file)?;
let mut cmd = Command::new("patchelf");
cmd.arg("--force-rpath")
.arg("--set-rpath")
.arg(rpath)
.arg(file.as_ref());
let output = cmd
.output()
.context("Failed to execute 'patchelf', did you install it?")?;
if !output.status.success() {
bail!(
"patchelf --set-rpath failed: {}",
String::from_utf8_lossy(&output.stderr)
);
}
Ok(())
}

/// Get the `RPATH` of executables and libraries
pub fn get_rpath(file: impl AsRef<Path>) -> Result<String> {
let mut cmd = Command::new("patchelf");
cmd.arg("--print-rpath").arg(file.as_ref());
let output = cmd
.output()
.context("Failed to execute 'patchelf', did you install it?")?;
if !output.status.success() {
bail!(
"patchelf --print-rpath failed: {}",
String::from_utf8_lossy(&output.stderr)
);
}
let rpath = String::from_utf8(output.stdout)?;
Ok(rpath.trim().to_string())
}
Loading

0 comments on commit fb112e3

Please sign in to comment.