diff --git a/Cargo.toml b/Cargo.toml index 05ea9a9f3..0bab578b4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ streams. members = ['systest'] [dependencies] -libc = "0.2" +libc = { version = "0.2", optional = true } miniz-sys = { path = "miniz-sys", version = "0.1.11", optional = true } libz-sys = { version = "1.0", optional = true } tokio-io = { version = "0.1.11", optional = true } @@ -37,8 +37,8 @@ tokio-threadpool = "0.1.10" futures = "0.1" [features] -default = ["miniz-sys"] -zlib = ["libz-sys"] +default = ["miniz-sys", "libc"] +zlib = ["libz-sys", "libc"] rust_backend = ["miniz_oxide"] tokio = ["tokio-io", "futures"] diff --git a/src/ffi.rs b/src/ffi.rs index 718fc147b..c346e96b3 100644 --- a/src/ffi.rs +++ b/src/ffi.rs @@ -145,10 +145,6 @@ mod imp { StreamWrapper::None } } - - /// Dummy - #[allow(non_camel_case_types)] - pub struct mz_stream {} } #[cfg(all( diff --git a/src/lib.rs b/src/lib.rs index 33836ed99..35e195b86 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -82,7 +82,10 @@ extern crate crc32fast; #[cfg(feature = "tokio")] extern crate futures; -#[cfg(not(all(target_arch = "wasm32", not(target_os = "emscripten"))))] +#[cfg(not(any( + all(not(feature = "zlib"), feature = "rust_backend"), + all(target_arch = "wasm32", not(target_os = "emscripten")) +)))] extern crate libc; #[cfg(test)] extern crate quickcheck; @@ -91,22 +94,6 @@ extern crate rand; #[cfg(feature = "tokio")] extern crate tokio_io; -// These must currently agree with here -- -// https://github.com/Frommi/miniz_oxide/blob/e6c214efd253491ac072c2c9adba87ef5b4cd5cb/src/lib.rs#L14-L19 -// -// Eventually we'll want to actually move these into `libc` itself for wasm, or -// otherwise not use the capi crate for miniz_oxide but rather use the -// underlying types. -#[cfg(all(target_arch = "wasm32", not(target_os = "emscripten")))] -mod libc { - #![allow(non_camel_case_types)] - pub type c_ulong = u64; - pub type off_t = i64; - pub type c_int = i32; - pub type c_uint = u32; - pub type size_t = usize; -} - pub use crc::{Crc, CrcReader, CrcWriter}; pub use gz::GzBuilder; pub use gz::GzHeader; diff --git a/src/mem.rs b/src/mem.rs index bb030a8c0..8c3c038a8 100644 --- a/src/mem.rs +++ b/src/mem.rs @@ -9,12 +9,11 @@ use std::io; use std::marker; use std::slice; -use libc::c_int; #[cfg(not(any( all(not(feature = "zlib"), feature = "rust_backend"), all(target_arch = "wasm32", not(target_os = "emscripten")) )))] -use libc::c_uint; +use libc::c_int; use ffi; use Compression; @@ -65,6 +64,10 @@ unsafe impl Send for Stream {} unsafe impl Sync for Stream {} trait Direction { + #[cfg(not(any( + all(not(feature = "zlib"), feature = "rust_backend"), + all(target_arch = "wasm32", not(target_os = "emscripten")) + )))] unsafe fn destroy(stream: *mut ffi::mz_stream) -> c_int; } @@ -237,80 +240,6 @@ impl Compress { Compress::make(level, zlib_header, window_bits) } - #[cfg(not(any( - all(not(feature = "zlib"), feature = "rust_backend"), - all(target_arch = "wasm32", not(target_os = "emscripten")) - )))] - fn make(level: Compression, zlib_header: bool, window_bits: u8) -> Compress { - assert!( - window_bits > 8 && window_bits < 16, - "window_bits must be within 9 ..= 15" - ); - unsafe { - let mut state = ffi::StreamWrapper::default(); - let ret = ffi::mz_deflateInit2( - &mut *state, - level.0 as c_int, - ffi::MZ_DEFLATED, - if zlib_header { - window_bits as c_int - } else { - -(window_bits as c_int) - }, - 9, - ffi::MZ_DEFAULT_STRATEGY, - ); - assert_eq!(ret, 0); - Compress { - inner: Stream { - stream_wrapper: state, - total_in: 0, - total_out: 0, - _marker: marker::PhantomData, - }, - } - } - } - - #[cfg(any( - all(not(feature = "zlib"), feature = "rust_backend"), - all(target_arch = "wasm32", not(target_os = "emscripten")) - ))] - fn make(level: Compression, zlib_header: bool, window_bits: u8) -> Compress { - use ffi::deflate::core::CompressorOxide; - use ffi::DataFormat; - use std::convert::TryInto; - assert!( - window_bits > 8 && window_bits < 16, - "window_bits must be within 9 ..= 15" - ); - - // Check in case the integer value changes at some point. - debug_assert!(level.level() <= 10); - - let state = { - let format = if zlib_header { - DataFormat::Zlib - } else { - DataFormat::Raw - }; - - let mut inner: Box = Box::default(); - inner.set_format_and_level(format, level.level().try_into().unwrap_or(1)); - - ffi::StreamWrapper::Deflate(inner) - }; - - Compress { - inner: Stream { - stream_wrapper: state, - total_in: 0, - total_out: 0, - _marker: marker::PhantomData, - }, - } - } - /// Returns the total number of input bytes which have been processed by /// this compression object. pub fn total_in(&self) -> u64 { @@ -350,23 +279,6 @@ impl Compress { self.inner.total_out = 0; } - #[cfg(not(any( - all(not(feature = "zlib"), feature = "rust_backend"), - all(target_arch = "wasm32", not(target_os = "emscripten")) - )))] - fn reset_inner(&mut self) { - let rc = unsafe { ffi::mz_deflateReset(&mut *self.inner.stream_wrapper) }; - assert_eq!(rc, ffi::MZ_OK); - } - - #[cfg(any( - all(not(feature = "zlib"), feature = "rust_backend"), - all(target_arch = "wasm32", not(target_os = "emscripten")) - ))] - fn reset_inner(&mut self) { - self.inner.stream_wrapper.compressor().reset(); - } - /// Dynamically updates the compression level. /// /// This can be used to switch between compression levels for different @@ -406,70 +318,6 @@ impl Compress { self.compress_inner(input, output, flush) } - #[cfg(any( - all(not(feature = "zlib"), feature = "rust_backend"), - all(target_arch = "wasm32", not(target_os = "emscripten")) - ))] - fn compress_inner( - &mut self, - input: &[u8], - output: &mut [u8], - flush: FlushCompress, - ) -> Result { - use ffi::{MZError, MZFlush, MZStatus}; - - let state = self.inner.stream_wrapper.compressor(); - let flush = MZFlush::new(flush as i32).unwrap(); - - let res = ffi::deflate::stream::deflate(state, input, output, flush); - self.inner.total_in += res.bytes_consumed as u64; - self.inner.total_out += res.bytes_written as u64; - - match res.status { - Ok(status) => match status { - MZStatus::Ok => Ok(Status::Ok), - MZStatus::StreamEnd => Ok(Status::StreamEnd), - MZStatus::NeedDict => Err(CompressError(())), - }, - Err(status) => match status { - MZError::Buf => Ok(Status::BufError), - _ => Err(CompressError(())), - }, - } - } - - #[cfg(not(any( - all(not(feature = "zlib"), feature = "rust_backend"), - all(target_arch = "wasm32", not(target_os = "emscripten")) - )))] - fn compress_inner( - &mut self, - input: &[u8], - output: &mut [u8], - flush: FlushCompress, - ) -> Result { - let raw = &mut *self.inner.stream_wrapper; - raw.next_in = input.as_ptr() as *mut _; - raw.avail_in = cmp::min(input.len(), c_uint::max_value() as usize) as c_uint; - raw.next_out = output.as_mut_ptr(); - raw.avail_out = cmp::min(output.len(), c_uint::max_value() as usize) as c_uint; - - let rc = unsafe { ffi::mz_deflate(raw, flush as c_int) }; - - // Unfortunately the total counters provided by zlib might be only - // 32 bits wide and overflow while processing large amounts of data. - self.inner.total_in += (raw.next_in as usize - input.as_ptr() as usize) as u64; - self.inner.total_out += (raw.next_out as usize - output.as_ptr() as usize) as u64; - - match rc { - ffi::MZ_OK => Ok(Status::Ok), - ffi::MZ_BUF_ERROR => Ok(Status::BufError), - ffi::MZ_STREAM_END => Ok(Status::StreamEnd), - ffi::MZ_STREAM_ERROR => Err(CompressError(())), - c => panic!("unknown return code: {}", c), - } - } - /// Compresses the input data into the extra space of the output, consuming /// only as much input as needed and writing as much output as possible. /// @@ -529,66 +377,6 @@ impl Decompress { Decompress::make(zlib_header, window_bits) } - #[cfg(not(any( - all(not(feature = "zlib"), feature = "rust_backend"), - all(target_arch = "wasm32", not(target_os = "emscripten")) - )))] - fn make(zlib_header: bool, window_bits: u8) -> Decompress { - assert!( - window_bits > 8 && window_bits < 16, - "window_bits must be within 9 ..= 15" - ); - unsafe { - let mut state = ffi::StreamWrapper::default(); - let ret = ffi::mz_inflateInit2( - &mut *state, - if zlib_header { - window_bits as c_int - } else { - -(window_bits as c_int) - }, - ); - assert_eq!(ret, 0); - Decompress { - inner: Stream { - stream_wrapper: state, - total_in: 0, - total_out: 0, - _marker: marker::PhantomData, - }, - } - } - } - - #[cfg(any( - all(not(feature = "zlib"), feature = "rust_backend"), - all(target_arch = "wasm32", not(target_os = "emscripten")) - ))] - fn make(zlib_header: bool, window_bits: u8) -> Decompress { - use ffi::{DataFormat, InflateState, StreamWrapper}; - assert!( - window_bits > 8 && window_bits < 16, - "window_bits must be within 9 ..= 15" - ); - - let format = if zlib_header { - DataFormat::Zlib - } else { - DataFormat::Raw - }; - - let state = StreamWrapper::Inflate(InflateState::new_boxed(format)); - - Decompress { - inner: Stream { - stream_wrapper: state, - total_in: 0, - total_out: 0, - _marker: marker::PhantomData, - }, - } - } - /// Returns the total number of input bytes which have been processed by /// this decompression object. pub fn total_in(&self) -> u64 { @@ -632,75 +420,6 @@ impl Decompress { self.decompress_inner(input, output, flush) } - #[cfg(any( - all(not(feature = "zlib"), feature = "rust_backend"), - all(target_arch = "wasm32", not(target_os = "emscripten")) - ))] - fn decompress_inner( - &mut self, - input: &[u8], - output: &mut [u8], - flush: FlushDecompress, - ) -> Result { - use ffi::{MZError, MZFlush, MZStatus}; - - let state = self.inner.stream_wrapper.decompressor(); - let flush = MZFlush::new(flush as i32).unwrap(); - - let res = ffi::inflate::stream::inflate(state, input, output, flush); - self.inner.total_in += res.bytes_consumed as u64; - self.inner.total_out += res.bytes_written as u64; - - match res.status { - Ok(status) => match status { - MZStatus::Ok => Ok(Status::Ok), - MZStatus::StreamEnd => Ok(Status::StreamEnd), - MZStatus::NeedDict => Err(DecompressError(DecompressErrorInner { - needs_dictionary: Some(state.decompressor().adler32().unwrap_or(0)), - })), - }, - Err(status) => match status { - MZError::Buf => Ok(Status::BufError), - _ => Err(DecompressError(Default::default())), - }, - } - } - - #[cfg(not(any( - all(not(feature = "zlib"), feature = "rust_backend"), - all(target_arch = "wasm32", not(target_os = "emscripten")) - )))] - fn decompress_inner( - &mut self, - input: &[u8], - output: &mut [u8], - flush: FlushDecompress, - ) -> Result { - let raw = &mut *self.inner.stream_wrapper; - raw.next_in = input.as_ptr() as *mut u8; - raw.avail_in = cmp::min(input.len(), c_uint::max_value() as usize) as c_uint; - raw.next_out = output.as_mut_ptr(); - raw.avail_out = cmp::min(output.len(), c_uint::max_value() as usize) as c_uint; - - let rc = unsafe { ffi::mz_inflate(raw, flush as c_int) }; - - // Unfortunately the total counters provided by zlib might be only - // 32 bits wide and overflow while processing large amounts of data. - self.inner.total_in += (raw.next_in as usize - input.as_ptr() as usize) as u64; - self.inner.total_out += (raw.next_out as usize - output.as_ptr() as usize) as u64; - - match rc { - ffi::MZ_DATA_ERROR | ffi::MZ_STREAM_ERROR => Err(DecompressError(Default::default())), - ffi::MZ_OK => Ok(Status::Ok), - ffi::MZ_BUF_ERROR => Ok(Status::BufError), - ffi::MZ_STREAM_END => Ok(Status::StreamEnd), - ffi::MZ_NEED_DICT => Err(DecompressError(DecompressErrorInner { - needs_dictionary: Some(raw.adler as u32), - })), - c => panic!("unknown return code: {}", c), - } - } - /// Decompresses the input data into the extra space in the output vector /// specified by `output`. /// @@ -827,19 +546,135 @@ impl fmt::Display for CompressError { all(not(feature = "zlib"), feature = "rust_backend"), all(target_arch = "wasm32", not(target_os = "emscripten")) ))] -impl Direction for DirCompress { - unsafe fn destroy(_stream: *mut ffi::mz_stream) -> c_int { - 0 +mod rust_backend { + use super::*; + impl Direction for DirCompress {} + impl Direction for DirDecompress {} + + impl Compress { + pub(super) fn reset_inner(&mut self) { + self.inner.stream_wrapper.compressor().reset(); + } + + pub(super) fn make(level: Compression, zlib_header: bool, window_bits: u8) -> Compress { + use ffi::deflate::core::CompressorOxide; + use ffi::DataFormat; + use std::convert::TryInto; + assert!( + window_bits > 8 && window_bits < 16, + "window_bits must be within 9 ..= 15" + ); + + // Check in case the integer value changes at some point. + debug_assert!(level.level() <= 10); + + let state = { + let format = if zlib_header { + DataFormat::Zlib + } else { + DataFormat::Raw + }; + + let mut inner: Box = Box::default(); + inner.set_format_and_level(format, level.level().try_into().unwrap_or(1)); + + ffi::StreamWrapper::Deflate(inner) + }; + + Compress { + inner: Stream { + stream_wrapper: state, + total_in: 0, + total_out: 0, + _marker: marker::PhantomData, + }, + } + } + + pub(super) fn compress_inner( + &mut self, + input: &[u8], + output: &mut [u8], + flush: FlushCompress, + ) -> Result { + use ffi::{MZError, MZFlush, MZStatus}; + + let state = self.inner.stream_wrapper.compressor(); + let flush = MZFlush::new(flush as i32).unwrap(); + + let res = ffi::deflate::stream::deflate(state, input, output, flush); + self.inner.total_in += res.bytes_consumed as u64; + self.inner.total_out += res.bytes_written as u64; + + match res.status { + Ok(status) => match status { + MZStatus::Ok => Ok(Status::Ok), + MZStatus::StreamEnd => Ok(Status::StreamEnd), + MZStatus::NeedDict => Err(CompressError(())), + }, + Err(status) => match status { + MZError::Buf => Ok(Status::BufError), + _ => Err(CompressError(())), + }, + } + } } -} -#[cfg(any( - all(not(feature = "zlib"), feature = "rust_backend"), - all(target_arch = "wasm32", not(target_os = "emscripten")) -))] -impl Direction for DirDecompress { - unsafe fn destroy(_stream: *mut ffi::mz_stream) -> c_int { - 0 + impl Decompress { + pub(super) fn make(zlib_header: bool, window_bits: u8) -> Decompress { + use ffi::{DataFormat, InflateState, StreamWrapper}; + assert!( + window_bits > 8 && window_bits < 16, + "window_bits must be within 9 ..= 15" + ); + + let format = if zlib_header { + DataFormat::Zlib + } else { + DataFormat::Raw + }; + + let state = StreamWrapper::Inflate(InflateState::new_boxed(format)); + + Decompress { + inner: Stream { + stream_wrapper: state, + total_in: 0, + total_out: 0, + _marker: marker::PhantomData, + }, + } + } + + pub(super) fn decompress_inner( + &mut self, + input: &[u8], + output: &mut [u8], + flush: FlushDecompress, + ) -> Result { + use ffi::{MZError, MZFlush, MZStatus}; + + let state = self.inner.stream_wrapper.decompressor(); + let flush = MZFlush::new(flush as i32).unwrap(); + + let res = ffi::inflate::stream::inflate(state, input, output, flush); + self.inner.total_in += res.bytes_consumed as u64; + self.inner.total_out += res.bytes_written as u64; + + match res.status { + Ok(status) => match status { + MZStatus::Ok => Ok(Status::Ok), + MZStatus::StreamEnd => Ok(Status::StreamEnd), + MZStatus::NeedDict => Err(DecompressError(DecompressErrorInner { + needs_dictionary: Some(state.decompressor().adler32().unwrap_or(0)), + })), + }, + Err(status) => match status { + MZError::Buf => Ok(Status::BufError), + _ => Err(DecompressError(Default::default())), + }, + } + } } } @@ -847,34 +682,159 @@ impl Direction for DirDecompress { all(not(feature = "zlib"), feature = "rust_backend"), all(target_arch = "wasm32", not(target_os = "emscripten")) )))] -impl Direction for DirCompress { - unsafe fn destroy(stream: *mut ffi::mz_stream) -> c_int { - ffi::mz_deflateEnd(stream) +mod c_backend { + use super::*; + use libc::c_int; + use libc::c_uint; + + impl Compress { + pub(super) fn make(level: Compression, zlib_header: bool, window_bits: u8) -> Compress { + assert!( + window_bits > 8 && window_bits < 16, + "window_bits must be within 9 ..= 15" + ); + unsafe { + let mut state = ffi::StreamWrapper::default(); + let ret = ffi::mz_deflateInit2( + &mut *state, + level.0 as c_int, + ffi::MZ_DEFLATED, + if zlib_header { + window_bits as c_int + } else { + -(window_bits as c_int) + }, + 9, + ffi::MZ_DEFAULT_STRATEGY, + ); + assert_eq!(ret, 0); + Compress { + inner: Stream { + stream_wrapper: state, + total_in: 0, + total_out: 0, + _marker: marker::PhantomData, + }, + } + } + } + + pub(super) fn reset_inner(&mut self) { + let rc = unsafe { ffi::mz_deflateReset(&mut *self.inner.stream_wrapper) }; + assert_eq!(rc, ffi::MZ_OK); + } + + pub(super) fn compress_inner( + &mut self, + input: &[u8], + output: &mut [u8], + flush: FlushCompress, + ) -> Result { + let raw = &mut *self.inner.stream_wrapper; + raw.next_in = input.as_ptr() as *mut _; + raw.avail_in = cmp::min(input.len(), c_uint::max_value() as usize) as c_uint; + raw.next_out = output.as_mut_ptr(); + raw.avail_out = cmp::min(output.len(), c_uint::max_value() as usize) as c_uint; + + let rc = unsafe { ffi::mz_deflate(raw, flush as c_int) }; + + // Unfortunately the total counters provided by zlib might be only + // 32 bits wide and overflow while processing large amounts of data. + self.inner.total_in += (raw.next_in as usize - input.as_ptr() as usize) as u64; + self.inner.total_out += (raw.next_out as usize - output.as_ptr() as usize) as u64; + + match rc { + ffi::MZ_OK => Ok(Status::Ok), + ffi::MZ_BUF_ERROR => Ok(Status::BufError), + ffi::MZ_STREAM_END => Ok(Status::StreamEnd), + ffi::MZ_STREAM_ERROR => Err(CompressError(())), + c => panic!("unknown return code: {}", c), + } + } } -} -#[cfg(not(any( - all(not(feature = "zlib"), feature = "rust_backend"), - all(target_arch = "wasm32", not(target_os = "emscripten")) -)))] -impl Direction for DirDecompress { - unsafe fn destroy(stream: *mut ffi::mz_stream) -> c_int { - ffi::mz_inflateEnd(stream) + impl Decompress { + pub(super) fn make(zlib_header: bool, window_bits: u8) -> Decompress { + assert!( + window_bits > 8 && window_bits < 16, + "window_bits must be within 9 ..= 15" + ); + unsafe { + let mut state = ffi::StreamWrapper::default(); + let ret = ffi::mz_inflateInit2( + &mut *state, + if zlib_header { + window_bits as c_int + } else { + -(window_bits as c_int) + }, + ); + assert_eq!(ret, 0); + Decompress { + inner: Stream { + stream_wrapper: state, + total_in: 0, + total_out: 0, + _marker: marker::PhantomData, + }, + } + } + } + + pub(super) fn decompress_inner( + &mut self, + input: &[u8], + output: &mut [u8], + flush: FlushDecompress, + ) -> Result { + let raw = &mut *self.inner.stream_wrapper; + raw.next_in = input.as_ptr() as *mut u8; + raw.avail_in = cmp::min(input.len(), c_uint::max_value() as usize) as c_uint; + raw.next_out = output.as_mut_ptr(); + raw.avail_out = cmp::min(output.len(), c_uint::max_value() as usize) as c_uint; + + let rc = unsafe { ffi::mz_inflate(raw, flush as c_int) }; + + // Unfortunately the total counters provided by zlib might be only + // 32 bits wide and overflow while processing large amounts of data. + self.inner.total_in += (raw.next_in as usize - input.as_ptr() as usize) as u64; + self.inner.total_out += (raw.next_out as usize - output.as_ptr() as usize) as u64; + + match rc { + ffi::MZ_DATA_ERROR | ffi::MZ_STREAM_ERROR => { + Err(DecompressError(Default::default())) + } + ffi::MZ_OK => Ok(Status::Ok), + ffi::MZ_BUF_ERROR => Ok(Status::BufError), + ffi::MZ_STREAM_END => Ok(Status::StreamEnd), + ffi::MZ_NEED_DICT => Err(DecompressError(DecompressErrorInner { + needs_dictionary: Some(raw.adler as u32), + })), + c => panic!("unknown return code: {}", c), + } + } } -} -#[cfg(not(any( - all(not(feature = "zlib"), feature = "rust_backend"), - all(target_arch = "wasm32", not(target_os = "emscripten")) -)))] -impl Drop for Stream { - fn drop(&mut self) { - unsafe { - let _ = D::destroy(&mut *self.stream_wrapper); + impl Direction for DirDecompress { + unsafe fn destroy(stream: *mut ffi::mz_stream) -> c_int { + ffi::mz_inflateEnd(stream) } } -} + impl Direction for DirCompress { + unsafe fn destroy(stream: *mut ffi::mz_stream) -> c_int { + ffi::mz_deflateEnd(stream) + } + } + + impl Drop for Stream { + fn drop(&mut self) { + unsafe { + let _ = D::destroy(&mut *self.stream_wrapper); + } + } + } +} #[cfg(test)] mod tests { use std::io::Write;