Go for musl and default to debug

PyO3 · Sep 18, 2018 · 003165c · 003165c
1 parent 6f57592
commit 003165c
Show file tree

Hide file tree

Showing 13 changed files with 232 additions and 185 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -11,29 +11,33 @@ addons:
   apt:
     packages:
       - libdbus-1-dev
+      - musl-tools
 
 matrix:
   fast_finish: true
   include:
     # These create deployments; Apparently there is not 32 bit mac os
     - os: linux
       rust: stable
-      env: TARGET=x86_64-unknown-linux-gnu
+      env: TARGET=x86_64-unknown-linux-musl
     - os: osx
       rust: stable
       env: TARGET=x86_64-apple-darwin
     - os: linux
       rust: stable
-      env: TARGET=i686-unknown-linux-gnu
+      env: TARGET=i686-unknown-linux-musl
       addons:
         apt:
           packages:
             - libdbus-1-dev:i386
-            - libc6-dev-i386
-            - linux-libc-dev:i386
-            - libssl-dev:i386
             - libgmp-dev:i386
-            - gcc-multilib
+            - binutils:i386
+            # We actually only need musl-tools:i386 and gcc-multilib, but we need the others to get apt to install them
+            - cpp-4.8:i386
+            - musl:i386
+            - musl-tools:i386
+            - gcc-multilib:i386
+            - cpp:i386
 
     # Those are tested
     - os: linux
@@ -64,17 +68,24 @@ install:
     fi
 
 script:
-  - if [ "$TARGET" == "i686-unknown-linux-gnu" ]; then rustup target add $TARGET; fi
-  - cargo build --target $TARGET
-  - if [ "$TRAVIS_RUST_VERSION" == "nightly" ]; then cargo test --target $TARGET -- --nocapture; fi
+  - if [[ "$TARGET" == *musl ]]; then rustup target add $TARGET; fi
+  - if [ "$TRAVIS_RUST_VERSION" == "nightly" ]; then cargo test -- --nocapture; fi
 
 before_deploy:
-  - cargo build --release --target $TARGET
-  - cargo run --target $TARGET -- publish -b bin --cargo-extra-args="--target=$TARGET" --skip-auditwheel -u konstin
+  # We don't care for manylinux compliance for the downloads, so we can use the keyring.
+  # The stack protector story is a weird one; On 32 bit, you get errors such as the following without the option
+  # /usr/bin/ld: apps/openssl: hidden symbol `__stack_chk_fail_local' isn't defined
+  # I assume that this is a musl bug fixed in 2015 that didn't make into ubuntu 14.04, but that special
+  # case seems to be documented nowhere else:
+  # http://git.musl-libc.org/cgit/musl/commit/?id=55d061f031085f24d138664c897791aebe9a2fab
+  # We can't have a more recent musl on 14.04 (there's no ppa), so we have to disable that feature
+  - CFLAGS="-fno-stack-protector" cargo build --release --target $TARGET --features "password-storage musl"
   - cd target/$TARGET/release/
   # You can add more file to the archive by adding them to this line
   - tar czf ../../../${BINARY_NAME}-$TRAVIS_TAG-$TARGET.tar.gz ${BINARY_NAME}
   - cd ../../..
+  # We do care for manylinux compliance, so we use the musl feature to get static binaries for pypi
+  - CFLAGS="-fno-stack-protector" cargo run --release --target $TARGET --features musl -- publish -u konstin --release -b bin --target $TARGET --cargo-extra-args="--features=musl"
 
 deploy:
   - # Add zipped binary to the github release
@@ -99,5 +110,4 @@ branches:
     - /^v\d+\.\d+\.\d+.*$/
 
 notifications:
-  email:
-    on_success: never
+  email: false
diff --git a/Cargo.toml b/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 authors = ["konstin <[email protected]>"]
 name = "pyo3-pack"
-version = "0.3.3"
+version = "0.3.4"
 description = "Build and publish crates with pyo3 bindings as python packages"
 exclude = ["get-fourtytwo/**/*", "integration-test/**/*", "sysconfig/*"]
 readme = "Readme.md"
@@ -50,13 +50,16 @@ openssl = { version = "0.10", features = ["vendored"], optional = true }
 indoc = "0.2.8"
 
 [features]
-default = ["auditwheel", "upload", "password-storage"]
+default = ["auditwheel", "upload"]
 auditwheel = ["goblin"]
-# sdists can be created, but they are currently useless (#2)
-sdist = ["tar", "libflate"]
 upload = ["reqwest"]
 password-storage = ["upload", "keyring"]
-musl_wip = ["openssl"]
+
+# This will make rewquest use a statically linked version of openssl
+musl = ["openssl"]
+
+# sdists can be created, but they are currently useless (#2)
+sdist = ["tar", "libflate"]
 
 [workspace]
 members = [

diff --git a/Changelog.md b/Changelog.md
@@ -5,8 +5,18 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.3.4] - 2018-09-18
 
-## [Unreleased]
+### Added
+
+ * A `--target` option which behaves like cargo option of the same name
+
+### Changed
+
+ * Musl and auditwheel compliance: Using the new `musl` feature combined with the musl target, you can build completely static binaries. The `password-storage`, which enables keyring integration, is now disabled by default. The Pypi packages are now statically linked with musl so that they are audtiwheel compliant.
+ * Replaced `--debug` with `--release`. All builds are now debug by default
+
+## [0.3.3] - 2018-09-17
 
 ### Added
 
@@ -16,6 +26,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## Fixed
 
  * Usage with stable
+ * Wrong tags in WHEEL file on non-linux platforms
+ * Uploading on windows
 
 ## [0.3.1] - 2017-09-14
 
@@ -56,8 +68,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
  * Initial Release
 
-[Unreleased]: https://github.com/pyo3/pyo3-pack/compare/v0.3.1...HEAD
-[0.3.0]: https://github.com/pyo3/pyo3-pack/compare/v0.3.0...v0.3.1
+[Unreleased]: https://github.com/pyo3/pyo3-pack/compare/v0.3.3...HEAD
+[0.3.4]: https://github.com/pyo3/pyo3-pack/compare/v0.3.3...v0.3.4
+[0.3.3]: https://github.com/pyo3/pyo3-pack/compare/v0.3.1...v0.3.3
+[0.3.1]: https://github.com/pyo3/pyo3-pack/compare/v0.3.0...v0.3.1
 [0.3.0]: https://github.com/pyo3/pyo3-pack/compare/v0.2.0...v0.3.0
 [0.2.0]: https://github.com/pyo3/pyo3-pack/compare/v0.1.0...v0.2.0
 

diff --git a/Readme.md b/Readme.md
@@ -41,21 +41,55 @@ Pip allows adding so called console scripts, which are shell commands that execu
 get_42 = "get_fourtytwo:DummyClass.get_42"
 ```
 
+## pyo3 and rust-cpython
+
 For pyo3 and rust-cpython, pyo3-pack can only build packages for installed python versions, so you might want to use pyenv, deadsnakes or docker for building. If you don't set your own interpreters with `-i`, a heuristic is used to search for python installations. You can get a list all found versions with the `list-python` subcommand.
 
+
+## Cffi
+
+ Cffi wheels are compatible with all python versions, but they need to have `cffi` installed for the python used for building (`pip install cffi`).
+
+ Until [eqrion/cbdingen#203](https://github.com/eqrion/cbindgen/issues/203) is resolved, you also need to use a build script that writes c headers to a file called `target/header.h`
+
+```rust
+extern crate cbindgen;
+
+use std::env;
+use std::path::Path;
+
+fn main() {
+    let crate_dir = env::var("CARGO_MANIFEST_DIR").unwrap();
+
+    let mut config: cbindgen::Config = Default::default();
+    config.language = cbindgen::Language::C;
+    cbindgen::generate_with_config(&crate_dir, config)
+        .expect("Unable to generate bindings")
+        .write_to_file(Path::new("target").join("header.h"));
+}
+```
+
+## Manylinux and auditwheel
+
+For portability reasons, native python modules on linux must only dynamically link a set of very few libraries which are installed basically everywhere, hence the name manylinux. The pypa offers a special docker container and a tool called [auditwheel](https://github.com/pypa/auditwheel/) to ensure compliance with the [manylinux rules](https://www.python.org/dev/peps/pep-0513/#the-manylinux1-policy). pyo3-pack contains a reimplementation of the most important part of auditwheel that checks the generated library, so there's no need to use external tools. If you want to disable the manylinux compliance checks for some reason, use the `--skip-auditwheel` flag.
+
+pyo3-pack itself is manylinux compliant when compiled with the `musl` feature and a musl target, which is true for the version published on pypi. The binaries on the release pages have keyring integration (though the `password-storage` feature), which is not manylinux compliant.
+
 ### Build
 
 ```
 USAGE:
     pyo3-pack build [FLAGS] [OPTIONS]
 
 FLAGS:
-    -d, --debug              Do a debug build (don't pass --release to cargo)
     -h, --help               Prints help information
+        --release            Pass --release to cargo
         --skip-auditwheel    Don't check for manylinux compliance
     -V, --version            Prints version information
 
 OPTIONS:
+    -m, --manifest-path <PATH>                      The path to the Cargo.toml [default: Cargo.toml]
+        --target <TRIPLE>                           The --target option for cargo
     -b, --bindings-crate <bindings>
             The crate providing the python bindings. pyo3, rust-cpython and cffi are supported
 
@@ -65,7 +99,6 @@ OPTIONS:
     -i, --interpreter <interpreter>...
             The python versions to build wheels for, given as the names of the interpreters. Uses autodiscovery if not
             explicitly set.
-    -m, --manifest-path <manifest_path>             The path to the Cargo.toml [default: Cargo.toml]
     -o, --out <out>
             The directory to store the built wheels in. Defaults to a new "wheels" directory in the project's target
             directory
@@ -80,12 +113,14 @@ USAGE:
     pyo3-pack publish [FLAGS] [OPTIONS]
 
 FLAGS:
-    -d, --debug              Do a debug build (don't pass --release to cargo)
     -h, --help               Prints help information
+        --release            Pass --release to cargo
         --skip-auditwheel    Don't check for manylinux compliance
     -V, --version            Prints version information
 
 OPTIONS:
+    -m, --manifest-path <PATH>                      The path to the Cargo.toml [default: Cargo.toml]
+        --target <TRIPLE>                           The --target option for cargo
     -b, --bindings-crate <bindings>
             The crate providing the python bindings. pyo3, rust-cpython and cffi are supported
 
@@ -95,7 +130,6 @@ OPTIONS:
     -i, --interpreter <interpreter>...
             The python versions to build wheels for, given as the names of the interpreters. Uses autodiscovery if not
             explicitly set.
-    -m, --manifest-path <manifest_path>             The path to the Cargo.toml [default: Cargo.toml]
     -o, --out <out>
             The directory to store the built wheels in. Defaults to a new "wheels" directory in the project's target
             directory
@@ -119,7 +153,7 @@ USAGE:
 
 FLAGS:
     -h, --help       Prints help information
-        --release    Compile in release mode. This is useful e.g. for benchmarking
+        --release    Pass --release to cargo
     -V, --version    Prints version information
 
 OPTIONS:
@@ -134,36 +168,6 @@ OPTIONS:
             Extra arguments that will be passed to rustc as `cargo rustc [...] -- [arg1] [arg2]`
 ```
 
-
-## Cffi
-
- Cffi wheels are compatible with all python versions, but they need to have `cffi` installed to build (`pip install cffi`). Until [eqrion/cbdingen#203](https://github.com/eqrion/cbindgen/issues/203) is resolved, you also need to use a build script that writes c headers to a file called `target/header.h`
-
-```rust
-extern crate cbindgen;
-
-use std::env;
-use std::path::Path;
-
-fn main() {
-    let crate_dir = env::var("CARGO_MANIFEST_DIR").unwrap();
-
-    let mut config: cbindgen::Config = Default::default();
-    config.language = cbindgen::Language::C;
-    cbindgen::generate_with_config(&crate_dir, config)
-        .expect("Unable to generate bindings")
-        .write_to_file(Path::new("target").join("header.h"));
-}
-```
-
-## Manylinux and auditwheel
-
-For portability reasons, native python modules on linux must only dynamically link a set of very few libraries which are installed basically everywhere, hence the name manylinux. The pypa offers a special docker container and a tool called [auditwheel](https://github.com/pypa/auditwheel/) to ensure compliance with the [manylinux rules](https://www.python.org/dev/peps/pep-0513/#the-manylinux1-policy). pyo3-pack contains a reimplementation of the most important part of auditwheel that checks the generated library, so there's no need to use external tools. If you want to disable the manylinux compliance checks for some reason, use the `--skip-auditwheel` flag.
-
-To ship a completely static binary with musl, you can use `pyo3-pack build -b bin --cargo-extra-args="--target=x86_64-unknown-linux-musl"`.
-
-Note that the pyo3-pack pip package is not manylinux compliant (A compliant package, which you get with `--no-default-features --features auditwheel`, can neither upload nor use the keyring)
-
 ## Code
 
 The main part is the pyo3-pack library, which is completely documented and should be well integratable. The accompanying `main.rs` takes care username and password for the pypi upload and otherwise calls into the library.

diff --git a/appveyor.yml b/appveyor.yml
@@ -59,15 +59,16 @@ test_script:
       }
 
 before_deploy:
-  - cargo build --release
-  # I don't know how I can create a script deployment like in travis in appveyor, so let's hijack before_deploy
-  # Also we only need one version for win32 and one for win_amd64, so we only deploy to pypi for msvc
-  - cargo run -- publish -b bin -u konstin
+  - cargo build --release --features password-storage
   # Grab the binary and pack it into a zip archive
   - cd target\release\
   # You can add more file to the archive by adding them to this line
   - 7z a ../../%APPVEYOR_PROJECT_SLUG%-%APPVEYOR_REPO_TAG_NAME%-%TARGET%.zip %BINARY_NAME%
   - appveyor PushArtifact ../../%APPVEYOR_PROJECT_SLUG%-%APPVEYOR_REPO_TAG_NAME%-%TARGET%.zip
+  - cd ../..
+  # Publish pypi
+  - cargo run -- publish -b bin -u konstin --release
+
 
 deploy:
   # Add zipped binary to the github release

diff --git a/src/build_context.rs b/src/build_context.rs
@@ -4,13 +4,13 @@ use auditwheel_rs;
 use build_source_distribution;
 use compile;
 use failure::{Context, Error, ResultExt};
-use Metadata21;
-use module_writer::{write_bin, write_bindings_module, write_cffi_module};
 use module_writer::WheelWriter;
-use PythonInterpreter;
+use module_writer::{write_bin, write_bindings_module, write_cffi_module};
 use std::collections::HashMap;
 use std::fs;
 use std::path::PathBuf;
+use Metadata21;
+use PythonInterpreter;
 use Target;
 
 /// The way the rust code is bridged with python, i.e. either using extern c and cffi or
@@ -54,8 +54,8 @@ pub struct BuildContext {
     /// The directory to store the built wheels in. Defaults to a new "wheels"
     /// directory in the project's target directory
     pub out: PathBuf,
-    /// Do a debug build (don't pass --release to cargo)
-    pub debug: bool,
+    /// Pass --release to cargo
+    pub release: bool,
     /// Don't check for manylinux compliance
     pub skip_auditwheel: bool,
     /// Extra arguments that will be passed to cargo as `cargo rustc [...] [arg1] [arg2] --`
@@ -126,22 +126,22 @@ impl BuildContext {
         }
 
         #[cfg(feature = "sdist")]
-            {
-                let sdist_path = wheel_dir.join(format!(
-                    "{}-{}.tar.gz",
-                    &self.metadata21.get_distribution_encoded(),
-                    &self.metadata21.get_version_encoded()
-                ));
-
-                println!(
-                    "Building the source distribution to {}",
-                    sdist_path.display()
-                );
-                build_source_distribution(&self, &self.metadata21, &self.scripts, &sdist_path)
-                    .context("Failed to build the source distribution")?;
-
-                wheels.push((sdist_path, None));
-            }
+        {
+            let sdist_path = wheel_dir.join(format!(
+                "{}-{}.tar.gz",
+                &self.metadata21.get_distribution_encoded(),
+                &self.metadata21.get_version_encoded()
+            ));
+
+            println!(
+                "Building the source distribution to {}",
+                sdist_path.display()
+            );
+            build_source_distribution(&self, &self.metadata21, &self.scripts, &sdist_path)
+                .context("Failed to build the source distribution")?;
+
+            wheels.push((sdist_path, None));
+        }
 
         Ok(wheels)
     }
@@ -168,7 +168,7 @@ impl BuildContext {
 
         if !self.skip_auditwheel && target.is_linux() {
             #[cfg(feature = "auditwheel")]
-                auditwheel_rs(&artifact).context("Failed to ensure manylinux compliance")?;
+            auditwheel_rs(&artifact).context("Failed to ensure manylinux compliance")?;
         }
 
         Ok(artifact)
@@ -213,7 +213,7 @@ impl BuildContext {
 
         if !self.skip_auditwheel && self.target.is_linux() {
             #[cfg(feature = "auditwheel")]
-                auditwheel_rs(&artifact).context("Failed to ensure manylinux compliance")?;
+            auditwheel_rs(&artifact).context("Failed to ensure manylinux compliance")?;
         }
 
         let (tag, tags) = self.get_unversal_tags();