From d4d8ab5182a7f9d1b34cd921e5633eb062e8fb52 Mon Sep 17 00:00:00 2001 From: arthurprs Date: Wed, 5 Feb 2020 16:33:19 +0100 Subject: [PATCH] Support cloudfire optimized version of zlib as a backend --- .github/workflows/main.yml | 1 + Cargo.toml | 2 + README.md | 41 ++++++++++++------- examples/compress_file.rs | 28 +++++++++++++ src/ffi/c.rs | 80 ++++++++++++++++++++++++++++++++++---- src/ffi/mod.rs | 2 +- 6 files changed, 130 insertions(+), 24 deletions(-) create mode 100644 examples/compress_file.rs diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 56ef4c68c..d300b1d9c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -39,6 +39,7 @@ jobs: - run: cargo test --features zlib - run: cargo test --features miniz-sys - run: cargo test --features zlib --no-default-features + - run: cargo test --features cloudflare_zlib --no-default-features - run: cargo test --features miniz-sys --no-default-features - run: cargo test --features tokio diff --git a/Cargo.toml b/Cargo.toml index 31d54991c..964ac1f1a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ libc = "0.2.65" cfg-if = "0.1.6" miniz-sys = { path = "miniz-sys", version = "0.1.11", optional = true } libz-sys = { version = "1.0.25", optional = true } +cloudflare-zlib-sys = { version = "0.2.0", optional = true } tokio-io = { version = "0.1.11", optional = true } futures = { version = "0.1.25", optional = true } miniz_oxide = { version = "0.3.5", optional = true} @@ -44,6 +45,7 @@ futures = "0.1" [features] default = ["rust_backend"] zlib = ["libz-sys"] +cloudflare_zlib = ["cloudflare-zlib-sys"] rust_backend = ["miniz_oxide"] tokio = ["tokio-io", "futures"] diff --git a/README.md b/README.md index 59407fb11..81537d00d 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ A streaming compression/decompression library DEFLATE-based streams in Rust. This crate by default implemented as a wrapper around the `miniz_oxide` crate, a -port of `miniz.c` to Rust. This crate can also optionally use the zlib library +port of `miniz.c` to Rust. This crate can also optionally use other [backends](#Backends) like the zlib library or `miniz.c` itself. Supported formats: @@ -21,20 +21,6 @@ Supported formats: flate2 = "1.0" ``` -Using zlib instead of the Rust backend: - -```toml -[dependencies] -flate2 = { version = "1.0", features = ["zlib"], default-features = false } -``` - -Using `miniz.c`: - -```toml -[dependencies] -flate2 = { version = "1.0", features = ["miniz-sys"], default-features = false } -``` - ## Compression ```rust @@ -64,6 +50,31 @@ fn main() { } ``` +## Backends + +Using zlib instead of the (default) Rust backend: + +```toml +[dependencies] +flate2 = { version = "1.0", features = ["zlib"], default-features = false } +``` + +The cloudflare optimized version of zlib is also available. +While it's significantly faster it requires a x86-64 CPU with SSE 4.2 or ARM64 with NEON & CRC. +It does not support 32-bit CPUs at all. For more information check the [crate documentation](https://crates.io/crates/cloudflare-zlib-sys). + +```toml +[dependencies] +flate2 = { version = "1.0", features = ["cloudflare_zlib"], default-features = false } +``` + +Using `miniz.c`: + +```toml +[dependencies] +flate2 = { version = "1.0", features = ["miniz-sys"], default-features = false } +``` + # License This project is licensed under either of diff --git a/examples/compress_file.rs b/examples/compress_file.rs new file mode 100644 index 000000000..39ed8eed4 --- /dev/null +++ b/examples/compress_file.rs @@ -0,0 +1,28 @@ +extern crate flate2; + +use flate2::write::GzEncoder; +use flate2::Compression; +use std::env::args; +use std::fs::File; +use std::io::copy; +use std::io::BufReader; +use std::time::Instant; + +fn main() { + if args().len() != 3 { + eprintln!("Usage: ./compress_file `source` `target`"); + return; + } + let mut input = BufReader::new(File::open(args().nth(1).unwrap()).unwrap()); + let output = File::create(args().nth(2).unwrap()).unwrap(); + let mut encoder = GzEncoder::new(output, Compression::default()); + let start = Instant::now(); + copy(&mut input, &mut encoder).unwrap(); + let output = encoder.finish().unwrap(); + println!( + "Source len: {:?}", + input.get_ref().metadata().unwrap().len() + ); + println!("Target len: {:?}", output.metadata().unwrap().len()); + println!("Elapsed: {:?}", start.elapsed()); +} diff --git a/src/ffi/c.rs b/src/ffi/c.rs index 3530fd800..52a1b14cb 100644 --- a/src/ffi/c.rs +++ b/src/ffi/c.rs @@ -38,13 +38,13 @@ impl Default for StreamWrapper { reserved: 0, opaque: ptr::null_mut(), state: ptr::null_mut(), - #[cfg(feature = "zlib")] + #[cfg(any(feature = "zlib", feature = "cloudflare_zlib"))] zalloc, - #[cfg(feature = "zlib")] + #[cfg(any(feature = "zlib", feature = "cloudflare_zlib"))] zfree, - #[cfg(not(feature = "zlib"))] + #[cfg(not(any(feature = "zlib", feature = "cloudflare_zlib")))] zalloc: Some(zalloc), - #[cfg(not(feature = "zlib"))] + #[cfg(not(any(feature = "zlib", feature = "cloudflare_zlib")))] zfree: Some(zfree), }), } @@ -219,7 +219,7 @@ impl InflateBackend for Inflate { } } - #[cfg(feature = "zlib")] + #[cfg(any(feature = "zlib", feature = "cloudflare_zlib"))] fn reset(&mut self, zlib_header: bool) { let bits = if zlib_header { MZ_DEFAULT_WINDOW_BITS @@ -233,7 +233,7 @@ impl InflateBackend for Inflate { self.inner.total_in = 0; } - #[cfg(not(feature = "zlib"))] + #[cfg(not(any(feature = "zlib", feature = "cloudflare_zlib")))] fn reset(&mut self, zlib_header: bool) { *self = Self::make(zlib_header, MZ_DEFAULT_WINDOW_BITS as u8); } @@ -338,14 +338,14 @@ impl Backend for Deflate { pub use self::c_backend::*; /// Miniz specific -#[cfg(not(feature = "zlib"))] +#[cfg(not(any(feature = "zlib", feature = "cloudflare_zlib")))] mod c_backend { pub use miniz_sys::*; pub type AllocSize = libc::size_t; } /// Zlib specific -#[cfg(feature = "zlib")] +#[cfg(all(feature = "zlib", not(feature = "cloudflare_zlib")))] #[allow(bad_style)] mod c_backend { use libc::{c_char, c_int}; @@ -407,3 +407,67 @@ mod c_backend { ) } } + +/// Cloudflare optimized Zlib specific +#[cfg(feature = "cloudflare_zlib")] +#[allow(bad_style)] +mod c_backend { + use libc::{c_char, c_int}; + use std::mem; + + pub use cloudflare_zlib_sys::deflate as mz_deflate; + pub use cloudflare_zlib_sys::deflateEnd as mz_deflateEnd; + pub use cloudflare_zlib_sys::deflateReset as mz_deflateReset; + pub use cloudflare_zlib_sys::inflate as mz_inflate; + pub use cloudflare_zlib_sys::inflateEnd as mz_inflateEnd; + pub use cloudflare_zlib_sys::z_stream as mz_stream; + pub use cloudflare_zlib_sys::*; + + pub use cloudflare_zlib_sys::Z_BLOCK as MZ_BLOCK; + pub use cloudflare_zlib_sys::Z_BUF_ERROR as MZ_BUF_ERROR; + pub use cloudflare_zlib_sys::Z_DATA_ERROR as MZ_DATA_ERROR; + pub use cloudflare_zlib_sys::Z_DEFAULT_STRATEGY as MZ_DEFAULT_STRATEGY; + pub use cloudflare_zlib_sys::Z_DEFLATED as MZ_DEFLATED; + pub use cloudflare_zlib_sys::Z_FINISH as MZ_FINISH; + pub use cloudflare_zlib_sys::Z_FULL_FLUSH as MZ_FULL_FLUSH; + pub use cloudflare_zlib_sys::Z_NEED_DICT as MZ_NEED_DICT; + pub use cloudflare_zlib_sys::Z_NO_FLUSH as MZ_NO_FLUSH; + pub use cloudflare_zlib_sys::Z_OK as MZ_OK; + pub use cloudflare_zlib_sys::Z_PARTIAL_FLUSH as MZ_PARTIAL_FLUSH; + pub use cloudflare_zlib_sys::Z_STREAM_END as MZ_STREAM_END; + pub use cloudflare_zlib_sys::Z_STREAM_ERROR as MZ_STREAM_ERROR; + pub use cloudflare_zlib_sys::Z_SYNC_FLUSH as MZ_SYNC_FLUSH; + pub type AllocSize = cloudflare_zlib_sys::uInt; + + pub const MZ_DEFAULT_WINDOW_BITS: c_int = 15; + + const ZLIB_VERSION: &'static str = "1.2.8\0"; + + pub unsafe extern "C" fn mz_deflateInit2( + stream: *mut mz_stream, + level: c_int, + method: c_int, + window_bits: c_int, + mem_level: c_int, + strategy: c_int, + ) -> c_int { + cloudflare_zlib_sys::deflateInit2_( + stream, + level, + method, + window_bits, + mem_level, + strategy, + ZLIB_VERSION.as_ptr() as *const c_char, + mem::size_of::() as c_int, + ) + } + pub unsafe extern "C" fn mz_inflateInit2(stream: *mut mz_stream, window_bits: c_int) -> c_int { + cloudflare_zlib_sys::inflateInit2_( + stream, + window_bits, + ZLIB_VERSION.as_ptr() as *const c_char, + mem::size_of::() as c_int, + ) + } +} diff --git a/src/ffi/mod.rs b/src/ffi/mod.rs index 1fad61414..80376e567 100644 --- a/src/ffi/mod.rs +++ b/src/ffi/mod.rs @@ -41,7 +41,7 @@ cfg_if::cfg_if! { if #[cfg(target_arch = "wasm32")] { mod rust; pub use self::rust::*; - } else if #[cfg(any(feature = "miniz-sys", feature = "zlib"))] { + } else if #[cfg(any(feature = "miniz-sys", feature = "zlib", feature = "cloudflare_zlib"))] { mod c; pub use self::c::*; } else {