Support cloudfire optimized version of zlib as a backend (rust-lang#228)

* Support cloudfire optimized version of zlib as a backend

* Support cloudfire optimized version of zlib as a backend

.github/workflows/main.yml

@@ -39,6 +39,8 @@ jobs:
     - run: cargo test --features zlib
     - run: cargo test --features miniz-sys
     - run: cargo test --features zlib --no-default-features
+    - run: cargo test --features cloudflare_zlib --no-default-features
+      if: matrix.build != 'mingw'
     - run: cargo test --features miniz-sys --no-default-features
     - run: cargo test --features tokio
 

Cargo.toml

@@ -25,6 +25,7 @@ libc = "0.2.65"
 cfg-if = "0.1.6"
 miniz-sys = { path = "miniz-sys", version = "0.1.11", optional = true }
 libz-sys = { version = "1.0.25", optional = true }
+cloudflare-zlib-sys = { version = "0.2.0", optional = true }
 tokio-io = { version = "0.1.11", optional = true }
 futures = { version = "0.1.25", optional = true }
 miniz_oxide = { version = "0.3.5", optional = true}
@@ -44,6 +45,7 @@ futures = "0.1"
 [features]
 default = ["rust_backend"]
 zlib = ["libz-sys"]
+cloudflare_zlib = ["cloudflare-zlib-sys"]
 rust_backend = ["miniz_oxide"]
 tokio = ["tokio-io", "futures"]
 

README.md

@@ -6,7 +6,7 @@
 A streaming compression/decompression library DEFLATE-based streams in Rust.
 
 This crate by default implemented as a wrapper around the `miniz_oxide` crate, a
-port of `miniz.c` to Rust. This crate can also optionally use the zlib library
+port of `miniz.c` to Rust. This crate can also optionally use other [backends](#Backends) like the zlib library
 or `miniz.c` itself.
 
 Supported formats:
@@ -21,20 +21,6 @@ Supported formats:
 flate2 = "1.0"
 ```
 
-Using zlib instead of the Rust backend:
-
-```toml
-[dependencies]
-flate2 = { version = "1.0", features = ["zlib"], default-features = false }
-```
-
-Using `miniz.c`:
-
-```toml
-[dependencies]
-flate2 = { version = "1.0", features = ["miniz-sys"], default-features = false }
-```
-
 ## Compression
 
 ```rust
@@ -64,6 +50,32 @@ fn main() {
 }
 ```
 
+## Backends
+
+Using zlib instead of the (default) Rust backend:
+
+```toml
+[dependencies]
+flate2 = { version = "1.0", features = ["zlib"], default-features = false }
+```
+
+The cloudflare optimized version of zlib is also available.
+While it's significantly faster it requires a x86-64 CPU with SSE 4.2 or ARM64 with NEON & CRC.
+It does not support 32-bit CPUs at all and is incompatible with mingw.
+For more information check the [crate documentation](https://crates.io/crates/cloudflare-zlib-sys).
+
+```toml
+[dependencies]
+flate2 = { version = "1.0", features = ["cloudflare_zlib"], default-features = false }
+```
+
+Using `miniz.c`:
+
+```toml
+[dependencies]
+flate2 = { version = "1.0", features = ["miniz-sys"], default-features = false }
+```
+
 # License
 
 This project is licensed under either of

examples/compress_file.rs

@@ -0,0 +1,28 @@
+extern crate flate2;
+
+use flate2::write::GzEncoder;
+use flate2::Compression;
+use std::env::args;
+use std::fs::File;
+use std::io::copy;
+use std::io::BufReader;
+use std::time::Instant;
+
+fn main() {
+    if args().len() != 3 {
+        eprintln!("Usage: ./compress_file `source` `target`");
+        return;
+    }
+    let mut input = BufReader::new(File::open(args().nth(1).unwrap()).unwrap());
+    let output = File::create(args().nth(2).unwrap()).unwrap();
+    let mut encoder = GzEncoder::new(output, Compression::default());
+    let start = Instant::now();
+    copy(&mut input, &mut encoder).unwrap();
+    let output = encoder.finish().unwrap();
+    println!(
+        "Source len: {:?}",
+        input.get_ref().metadata().unwrap().len()
+    );
+    println!("Target len: {:?}", output.metadata().unwrap().len());
+    println!("Elapsed: {:?}", start.elapsed());
+}

src/ffi/c.rs

@@ -38,13 +38,13 @@ impl Default for StreamWrapper {
                 reserved: 0,
                 opaque: ptr::null_mut(),
                 state: ptr::null_mut(),
-                #[cfg(feature = "zlib")]
+                #[cfg(any(feature = "zlib", feature = "cloudflare_zlib"))]
                 zalloc,
-                #[cfg(feature = "zlib")]
+                #[cfg(any(feature = "zlib", feature = "cloudflare_zlib"))]
                 zfree,
-                #[cfg(not(feature = "zlib"))]
+                #[cfg(not(any(feature = "zlib", feature = "cloudflare_zlib")))]
                 zalloc: Some(zalloc),
-                #[cfg(not(feature = "zlib"))]
+                #[cfg(not(any(feature = "zlib", feature = "cloudflare_zlib")))]
                 zfree: Some(zfree),
             }),
         }
@@ -219,7 +219,7 @@ impl InflateBackend for Inflate {
         }
     }
 
-    #[cfg(feature = "zlib")]
+    #[cfg(any(feature = "zlib", feature = "cloudflare_zlib"))]
     fn reset(&mut self, zlib_header: bool) {
         let bits = if zlib_header {
             MZ_DEFAULT_WINDOW_BITS
@@ -233,7 +233,7 @@ impl InflateBackend for Inflate {
         self.inner.total_in = 0;
     }
 
-    #[cfg(not(feature = "zlib"))]
+    #[cfg(not(any(feature = "zlib", feature = "cloudflare_zlib")))]
     fn reset(&mut self, zlib_header: bool) {
         *self = Self::make(zlib_header, MZ_DEFAULT_WINDOW_BITS as u8);
     }
@@ -338,14 +338,14 @@ impl Backend for Deflate {
 pub use self::c_backend::*;
 
 /// Miniz specific
-#[cfg(not(feature = "zlib"))]
+#[cfg(not(any(feature = "zlib", feature = "cloudflare_zlib")))]
 mod c_backend {
     pub use miniz_sys::*;
     pub type AllocSize = libc::size_t;
 }
 
 /// Zlib specific
-#[cfg(feature = "zlib")]
+#[cfg(all(feature = "zlib", not(feature = "cloudflare_zlib")))]
 #[allow(bad_style)]
 mod c_backend {
     use libc::{c_char, c_int};
@@ -407,3 +407,67 @@ mod c_backend {
         )
     }
 }
+
+/// Cloudflare optimized Zlib specific
+#[cfg(feature = "cloudflare_zlib")]
+#[allow(bad_style)]
+mod c_backend {
+    use libc::{c_char, c_int};
+    use std::mem;
+
+    pub use cloudflare_zlib_sys::deflate as mz_deflate;
+    pub use cloudflare_zlib_sys::deflateEnd as mz_deflateEnd;
+    pub use cloudflare_zlib_sys::deflateReset as mz_deflateReset;
+    pub use cloudflare_zlib_sys::inflate as mz_inflate;
+    pub use cloudflare_zlib_sys::inflateEnd as mz_inflateEnd;
+    pub use cloudflare_zlib_sys::z_stream as mz_stream;
+    pub use cloudflare_zlib_sys::*;
+
+    pub use cloudflare_zlib_sys::Z_BLOCK as MZ_BLOCK;
+    pub use cloudflare_zlib_sys::Z_BUF_ERROR as MZ_BUF_ERROR;
+    pub use cloudflare_zlib_sys::Z_DATA_ERROR as MZ_DATA_ERROR;
+    pub use cloudflare_zlib_sys::Z_DEFAULT_STRATEGY as MZ_DEFAULT_STRATEGY;
+    pub use cloudflare_zlib_sys::Z_DEFLATED as MZ_DEFLATED;
+    pub use cloudflare_zlib_sys::Z_FINISH as MZ_FINISH;
+    pub use cloudflare_zlib_sys::Z_FULL_FLUSH as MZ_FULL_FLUSH;
+    pub use cloudflare_zlib_sys::Z_NEED_DICT as MZ_NEED_DICT;
+    pub use cloudflare_zlib_sys::Z_NO_FLUSH as MZ_NO_FLUSH;
+    pub use cloudflare_zlib_sys::Z_OK as MZ_OK;
+    pub use cloudflare_zlib_sys::Z_PARTIAL_FLUSH as MZ_PARTIAL_FLUSH;
+    pub use cloudflare_zlib_sys::Z_STREAM_END as MZ_STREAM_END;
+    pub use cloudflare_zlib_sys::Z_STREAM_ERROR as MZ_STREAM_ERROR;
+    pub use cloudflare_zlib_sys::Z_SYNC_FLUSH as MZ_SYNC_FLUSH;
+    pub type AllocSize = cloudflare_zlib_sys::uInt;
+
+    pub const MZ_DEFAULT_WINDOW_BITS: c_int = 15;
+
+    const ZLIB_VERSION: &'static str = "1.2.8\0";
+
+    pub unsafe extern "C" fn mz_deflateInit2(
+        stream: *mut mz_stream,
+        level: c_int,
+        method: c_int,
+        window_bits: c_int,
+        mem_level: c_int,
+        strategy: c_int,
+    ) -> c_int {
+        cloudflare_zlib_sys::deflateInit2_(
+            stream,
+            level,
+            method,
+            window_bits,
+            mem_level,
+            strategy,
+            ZLIB_VERSION.as_ptr() as *const c_char,
+            mem::size_of::<mz_stream>() as c_int,
+        )
+    }
+    pub unsafe extern "C" fn mz_inflateInit2(stream: *mut mz_stream, window_bits: c_int) -> c_int {
+        cloudflare_zlib_sys::inflateInit2_(
+            stream,
+            window_bits,
+            ZLIB_VERSION.as_ptr() as *const c_char,
+            mem::size_of::<mz_stream>() as c_int,
+        )
+    }
+}

src/ffi/mod.rs

@@ -41,7 +41,7 @@ cfg_if::cfg_if! {
     if #[cfg(target_arch = "wasm32")] {
         mod rust;
         pub use self::rust::*;
-    } else if #[cfg(any(feature = "miniz-sys", feature = "zlib"))] {
+    } else if #[cfg(any(feature = "miniz-sys", feature = "zlib", feature = "cloudflare_zlib"))] {
         mod c;
         pub use self::c::*;
     } else {