diff --git a/Cargo.toml b/Cargo.toml index fbc77abd..fdd48135 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,6 +51,7 @@ all_components = [ ] gzip = ["flate2"] +zstd = ["dep:zstd"] [[bench]] name = "rotation" @@ -60,6 +61,7 @@ harness = false arc-swap = "1.6" chrono = { version = "0.4.23", optional = true, features = ["clock"], default-features = false } flate2 = { version = "1.0", optional = true } +zstd = { version = "0.13", optional = true } fnv = "1.0" humantime = { version = "2.1", optional = true } log = { version = "0.4.20", features = ["std"] } diff --git a/README.md b/README.md index c35baf4d..3b2cb368 100644 --- a/README.md +++ b/README.md @@ -69,15 +69,16 @@ fn main() { ## Compression If you are using the file rotation in your configuration there is a known -substantial performance issue with the `gzip` feature. When rolling files -it will zip log archives automatically. This is a problem when the log archives -are large as the zip happens in the main thread and will halt the process while -the zip is completed. +substantial performance issue with either the `gzip` or `zstd` +features. When rolling files it will zip log archives automatically. This is +a problem when the log archives are large as the zip process occurs in +the main thread and will halt the process until the zip process +completes. The methods to mitigate this are as follows. 1. Use the `background_rotation` feature which spawns an os thread to do the compression. -2. Do not enable the `gzip` feature. +2. Do not enable the `gzip` nor the `zstd` features. 3. Ensure the archives are small enough that the compression time is acceptable. For more information see the PR that added [`background_rotation`](https://github.com/estk/log4rs/pull/117). diff --git a/benches/rotation.rs b/benches/rotation.rs index 4167331f..d647cfd1 100644 --- a/benches/rotation.rs +++ b/benches/rotation.rs @@ -60,10 +60,15 @@ fn mk_config(file_size: u64, file_count: u32) -> log4rs::config::Config { let log_path = LOGDIR.path(); let log_pattern = log_path.join("log.log"); - #[cfg(feature = "gzip")] - let roll_pattern = format!("{}/{}", log_path.to_string_lossy(), "log.{}.gz"); - #[cfg(not(feature = "gzip"))] - let roll_pattern = format!("{}/{}", log_path.to_string_lossy(), "log.{}"); + let roll_pattern = { + if cfg!(feature = "gzip") { + format!("{}/{}", log_path.to_string_lossy(), "log.{}.gz") + } else if cfg!(feature = "zstd") { + format!("{}/{}", log_path.to_string_lossy(), "log.{}.zst") + } else { + format!("{}/{}", log_path.to_string_lossy(), "log.{}") + } + }; use log::LevelFilter; use log4rs::{ diff --git a/docs/Configuration.md b/docs/Configuration.md index a7dd94a3..715b8d55 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -253,6 +253,9 @@ double curly brace `{}`. For example `archive/foo.{}.log`. Each instance of `{}` will be replaced with the index number of the configuration file. Note that if the file extension of the pattern is `.gz` and the `gzip` Cargo feature is enabled, the archive files will be gzip-compressed. +If the file extension of the pattern is `.zst` and the `zstd` Cargo +feature is enabled, the archive files will be compressed using the +[Zstandard](https://facebook.github.io/zstd/) compression algorithm. > Note: This pattern field is only used for archived files. The `path` field > of the higher level `rolling_file` will be used for the active log file. diff --git a/src/append/rolling_file/policy/compound/roll/fixed_window.rs b/src/append/rolling_file/policy/compound/roll/fixed_window.rs index 340ddd05..40fc3222 100644 --- a/src/append/rolling_file/policy/compound/roll/fixed_window.rs +++ b/src/append/rolling_file/policy/compound/roll/fixed_window.rs @@ -31,6 +31,8 @@ enum Compression { None, #[cfg(feature = "gzip")] Gzip, + #[cfg(feature = "zstd")] + Zstd, } impl Compression { @@ -54,6 +56,19 @@ impl Compression { fs::remove_file(src) } + #[cfg(feature = "zstd")] + Compression::Zstd => { + use std::fs::File; + let mut i = File::open(src)?; + let mut o = { + let target = File::create(dst)?; + zstd::Encoder::new(target, zstd::DEFAULT_COMPRESSION_LEVEL)? + }; + io::copy(&mut i, &mut o)?; + drop(o.finish()?); + drop(i); + fs::remove_file(src) + } } } } @@ -275,6 +290,12 @@ impl FixedWindowRollerBuilder { Some(e) if e == "gz" => { bail!("gzip compression requires the `gzip` feature"); } + #[cfg(feature = "zstd")] + Some(e) if e == "zst" => Compression::Zstd, + #[cfg(not(feature = "zstd"))] + Some(e) if e == "zst" => { + bail!("zstd compression requires the `zstd` feature"); + } _ => Compression::None, }; @@ -560,6 +581,44 @@ mod test { assert_eq!(contents, actual); } + #[test] + #[cfg_attr(feature = "zstd", ignore)] + fn unsupported_zstd() { + let dir = tempfile::tempdir().unwrap(); + + let pattern = dir.path().join("{}.zst"); + let roller = FixedWindowRoller::builder().build(pattern.to_str().unwrap(), 2); + assert!(roller.is_err()); + assert!(roller + .unwrap_err() + .to_string() + .contains("zstd compression requires the `zstd` feature")); + } + + #[test] + #[cfg(feature = "zstd")] + fn supported_zstd() { + let dir = tempfile::tempdir().unwrap(); + + let pattern = dir.path().join("{}.zst"); + let roller = FixedWindowRoller::builder() + .build(pattern.to_str().unwrap(), 2) + .unwrap(); + + let contents = (0..10000).map(|i| i as u8).collect::>(); + + let file = dir.path().join("foo.log"); + File::create(&file).unwrap().write_all(&contents).unwrap(); + + roller.roll(&file).unwrap(); + wait_for_roller(&roller); + + let compressed_data = fs::read(dir.path().join("0.zst")).unwrap(); + let actual = zstd::decode_all(compressed_data.as_slice()).unwrap(); + + assert_eq!(contents, actual); + } + #[test] fn roll_with_env_var() { std::env::set_var("LOG_DIR", "test_log_dir");