Various build system improvements

Cargo.toml

@@ -41,4 +41,8 @@ dynamic_linking = ["rdkafka-sys/dynamic_linking"]
 # Use external library for lz4. Use internal if not enabled.
 external_lz4 = ["rdkafka-sys/external_lz4"]
 
+# Enable support for zstd compression.
+zstd = ["rdkafka-sys/zstd"]
+zstd-pkg-config = ["rdkafka-sys/zstd-pkg-config"]
+
 cmake_build = ["rdkafka-sys/cmake_build"]

Dockerfile

@@ -2,7 +2,6 @@ FROM ubuntu:16.04
 
 RUN apt-get update && apt-get install -y build-essential \
     curl \
-    llvm-3.9-dev libclang-3.9-dev clang-3.9 \
     openssl libssl-dev \
     pkg-config \
     python \

changelog.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## Unreleased
+
+* Bump librdkafka to v1.2.1.
+* Stop automatically generating librdkafka bindings. Platform-independent
+  bindings are now checked in to the repository.
+* Remove build-time dependency on bindgen, clang, and libclang.
+* Move zstd compression support behind the `zstd` feature flag.
+* Ensure all features are honored in the CMake build system.
+
+
 <a name="0.21.0"></a>
 ## 0.21.0 (2019-04-24)
 

rdkafka-sys/Cargo.toml

@@ -11,12 +11,13 @@ keywords = ["kafka", "rdkafka"]
 categories = ["external-ffi-bindings"]
 
 [dependencies]
+libc = "0.2.65"
 libz-sys = "1.0"
-zstd-sys = { version = "1.3", features = [] }
+zstd-sys = { version = "1.3", features = [], optional = true }
 openssl-sys = { version = "~ 0.9.0", optional = true }
+lz4-sys = { version = "1.8.3", optional = true }
 
 [build-dependencies]
-bindgen = "0.51.1"
 num_cpus = "0.2.0"
 pkg-config = "0.3.9"
 cmake = { version = "^0.1", optional = true }
@@ -36,6 +37,10 @@ sasl = ["ssl"]
 dynamic_linking = []
 
 # Use external library for lz4. Use internal if not enabled.
-external_lz4 = []
+external_lz4 = ["lz4-sys"]
+
+# Enable support for zstd compression.
+zstd = ["zstd-sys"]
+zstd-pkg-config = ["zstd-sys/pkg-config"]
 
 cmake_build = ["cmake"]

rdkafka-sys/README.md

@@ -8,8 +8,8 @@ To regenerate the bindings:
 
 ``` bash
 git submodule update --init
-cargo install bindgen --vers 0.30.0
-bindgen --builtins --no-doc-comments librdkafka/src/rdkafka.h -o src/bindings/{platform}.rs
+cargo install bindgen
+./update-bindings.sh
 ```
 
 ## Version

rdkafka-sys/build.rs

@@ -4,17 +4,9 @@ extern crate num_cpus;
 extern crate pkg_config;
 
 use std::env;
-use std::io::Write;
 use std::path::{Path, PathBuf};
 use std::process::{self, Command};
 
-macro_rules! println_stderr(
-    ($($arg:tt)*) => { {
-        let r = writeln!(&mut ::std::io::stderr(), $($arg)*);
-        r.expect("failed printing to stderr");
-    } }
-);
-
 fn run_command_or_fail<P>(dir: &str, cmd: P, args: &[&str])
 where
     P: AsRef<Path>,
@@ -32,7 +24,7 @@ where
     } else {
         PathBuf::from(cmd)
     };
-    println_stderr!(
+    eprintln!(
         "Running command: \"{} {}\" in dir: {}",
         cmd.display(),
         args.join(" "),
@@ -54,59 +46,36 @@ fn main() {
         .expect("Crate version is not valid");
 
     if env::var("CARGO_FEATURE_DYNAMIC_LINKING").is_ok() {
-        println_stderr!("librdkafka will be linked dynamically");
+        eprintln!("librdkafka will be linked dynamically");
         let pkg_probe = pkg_config::Config::new()
             .cargo_metadata(true)
             .atleast_version(librdkafka_version)
             .probe("rdkafka");
 
         match pkg_probe {
             Ok(library) => {
-                println_stderr!("librdkafka found on the system:");
-                println_stderr!("  Name: {:?}", library.libs);
-                println_stderr!("  Path: {:?}", library.link_paths);
-                println_stderr!("  Version: {}", library.version);
+                eprintln!("librdkafka found on the system:");
+                eprintln!("  Name: {:?}", library.libs);
+                eprintln!("  Path: {:?}", library.link_paths);
+                eprintln!("  Version: {}", library.version);
             }
             Err(_) => {
-                println_stderr!(
+                eprintln!(
                     "librdkafka {} cannot be found on the system",
                     librdkafka_version
                 );
-                println_stderr!("Dynamic linking failed. Exiting.");
+                eprintln!("Dynamic linking failed. Exiting.");
                 process::exit(1);
             }
         }
     } else {
         if !Path::new("librdkafka/LICENSE").exists() {
-            println_stderr!("Setting up submodules");
+            eprintln!("Setting up submodules");
             run_command_or_fail("../", "git", &["submodule", "update", "--init"]);
         }
-        println_stderr!("Building and linking librdkafka statically");
+        eprintln!("Building and linking librdkafka statically");
         build_librdkafka();
     }
-
-    let bindings = bindgen::Builder::default()
-        .header("librdkafka/src/rdkafka.h")
-        .generate_comments(false)
-        .emit_builtins()
-        // TODO: using rustified_enum is somewhat dangerous, especially when
-        // also using shared libraries.
-        // For details: https://github.com/rust-lang/rust-bindgen/issues/758
-        .rustified_enum("rd_kafka_vtype_t")
-        .rustified_enum("rd_kafka_type_t")
-        .rustified_enum("rd_kafka_conf_res_t")
-        .rustified_enum("rd_kafka_resp_err_t")
-        .rustified_enum("rd_kafka_timestamp_type_t")
-        .rustified_enum("rd_kafka_admin_op_t")
-        .rustified_enum("rd_kafka_ResourceType_t")
-        .rustified_enum("rd_kafka_ConfigSource_t")
-        .generate()
-        .expect("failed to generate bindings");
-
-    let out_path = PathBuf::from(env::var("OUT_DIR").unwrap());
-    bindings
-        .write_to_file(out_path.join("bindings.rs"))
-        .expect("failed to write bindings");
 }
 
 #[cfg(not(feature = "cmake_build"))]
@@ -126,8 +95,16 @@ fn build_librdkafka() {
         configure_flags.push("--disable-ssl");
     }
 
+    if env::var("CARGO_FEATURE_ZSTD").is_ok() {
+        configure_flags.push("--enable-zstd");
+        println!("cargo:rustc-link-lib=static=zstd");
+    } else {
+        configure_flags.push("--disable-zstd");
+    }
+
     if env::var("CARGO_FEATURE_EXTERNAL_LZ4").is_ok() {
         configure_flags.push("--enable-lz4");
+        println!("cargo:rustc-link-lib=static=lz4");
     } else {
         configure_flags.push("--disable-lz4");
     }
@@ -144,24 +121,14 @@ fn build_librdkafka() {
             .expect("Can't find current dir")
             .display()
     );
-    println!("cargo:rustc-link-lib=static=zstd");
     println!("cargo:rustc-link-lib=static=rdkafka");
 }
 
-#[cfg(not(target_os = "freebsd"))]
-fn make_librdkafka() {
-    run_command_or_fail(
-        "librdkafka",
-        "make",
-        &["-j", &num_cpus::get().to_string(), "libs"],
-    );
-}
-
-#[cfg(target_os = "freebsd")]
+#[cfg(not(feature = "cmake_build"))]
 fn make_librdkafka() {
     run_command_or_fail(
         "librdkafka",
-        "gmake",
+        if cfg!(target_os = "freebsd") { "gmake" } else { "make" },
         &["-j", &num_cpus::get().to_string(), "libs"],
     );
 }
@@ -183,6 +150,22 @@ fn build_librdkafka() {
     } else {
         config.define("WITH_SASL", "0");
     }
+    if env::var("CARGO_FEATURE_ZSTD").is_ok() {
+        config.define("WITH_ZSTD", "1");
+        config.register_dep("zstd");
+        println!("cargo:rustc-link-lib=static=zstd");
+    } else {
+        config.define("WITH_ZSTD", "0");
+    }
+    if env::var("CARGO_FEATURE_EXTERNAL_LZ4").is_ok() {
+        config.define("ENABLE_LZ4_EXT", "1");
+        println!("cargo:rustc-link-lib=static=lz4");
+    } else {
+        config.define("ENABLE_LZ4_EXT", "0");
+    }
+    if let Ok(system_name) = env::var("CMAKE_SYSTEM_NAME") {
+        config.define("CMAKE_SYSTEM_NAME", system_name);
+    }
     let dst = config.build();
     println!("cargo:rustc-link-search=native={}/build/src", dst.display());
     println!("cargo:rustc-link-lib=static=rdkafka");