Merge branch 'master' into dougsmith/server-telemetry

.github/semantic.yml

@@ -199,7 +199,6 @@ scopes:
   - clickhouse sink # Anything `clickhouse` sink related
   - console sink # Anything `console` sink related
   - databend sink # Anything `databend` sink related
-  - datadog_archives sink # Anything `datadog_archives` sink related
   - datadog_events sink # Anything `datadog_events` sink related
   - datadog_logs sink # Anything `datadog_logs` sink related
   - datadog_metrics sink # Anything `datadog_metrics` sink related

Cargo.toml

@@ -141,11 +141,7 @@ opentelemetry-proto = { path = "lib/opentelemetry-proto", optional = true }
 tracing-limit = { path = "lib/tracing-limit" }
 vector-api-client = { path = "lib/vector-api-client", optional = true }
 vector-buffers = { path = "lib/vector-buffers", default-features = false }
-vector-config = { path = "lib/vector-config" }
-vector-config-common = { path = "lib/vector-config-common" }
-vector-config-macros = { path = "lib/vector-config-macros" }
 vector-lib = { path = "lib/vector-lib", default-features = false, features = ["vrl"] }
-vector-stream = { path = "lib/vector-stream" }
 vector-vrl-functions = { path = "lib/vector-vrl/functions" }
 loki-logproto = { path = "lib/loki-logproto", optional = true }
 
@@ -203,7 +199,7 @@ opendal = {version = "0.38", default-features = false, features = ["native-tls",
 tower = { version = "0.4.13", default-features = false, features = ["buffer", "limit", "retry", "timeout", "util", "balance", "discover"] }
 tower-http = { version = "0.4.4", default-features = false, features = ["decompression-gzip", "trace"]}
 # Serde
-serde = { version = "1.0.189", default-features = false, features = ["derive"] }
+serde = { version = "1.0.190", default-features = false, features = ["derive"] }
 serde-toml-merge = { version = "0.3.3", default-features = false }
 serde_bytes = { version = "0.11.12", default-features = false, features = ["std"], optional = true }
 serde_json = { version = "1.0.107", default-features = false, features = ["raw_value"] }
@@ -267,7 +263,7 @@ encoding_rs = { version = "0.8.33", default-features = false, features = ["serde
 enum_dispatch = { version = "0.3.12", default-features = false }
 exitcode = { version = "1.1.2", default-features = false }
 flate2 = { version = "1.0.28", default-features = false, features = ["default"] }
-futures-util = { version = "0.3.28", default-features = false }
+futures-util = { version = "0.3.29", default-features = false }
 glob = { version = "0.3.1", default-features = false }
 governor = { version = "0.6.0", default-features = false, features = ["dashmap", "jitter", "std"], optional = true }
 grok = { version = "2.0.0", default-features = false, optional = true }
@@ -324,7 +320,7 @@ syslog = { version = "6.1.0", default-features = false, optional = true }
 tikv-jemallocator = { version = "0.5.4", default-features = false, optional = true }
 tokio-postgres = { version = "0.7.10", default-features = false, features = ["runtime", "with-chrono-0_4"], optional = true }
 tokio-tungstenite = {version = "0.20.1", default-features = false, features = ["connect"], optional = true}
-toml = { version = "0.8.4", default-features = false, features = ["parse", "display"] }
+toml = { version = "0.8.5", default-features = false, features = ["parse", "display"] }
 tonic = { version = "0.10", optional = true, default-features = false, features = ["transport", "codegen", "prost", "tls", "tls-roots", "gzip"] }
 trust-dns-proto = { version = "0.23.2", default-features = false, features = ["dnssec"], optional = true }
 typetag = { version = "0.2.13", default-features = false }

lib/vector-api-client/Cargo.toml

@@ -9,7 +9,7 @@ license = "MPL-2.0"
 [dependencies]
 
 # Serde
-serde = { version = "1.0.189", default-features = false, features = ["derive"] }
+serde = { version = "1.0.190", default-features = false, features = ["derive"] }
 serde_json = { version = "1.0.107", default-features = false, features = ["raw_value"] }
 
 # Error handling

lib/vector-buffers/Cargo.toml

@@ -21,7 +21,7 @@ metrics = "0.21.1"
 num-traits = { version = "0.2.17", default-features = false }
 pin-project.workspace = true
 rkyv = { version = "0.7.40", default-features = false, features = ["size_32", "std", "strict", "validation"] }
-serde = { version = "1.0.189", default-features = false, features = ["derive"] }
+serde = { version = "1.0.190", default-features = false, features = ["derive"] }
 snafu = { version = "0.7.5", default-features = false, features = ["std"] }
 tokio-util = { version = "0.7.0", default-features = false }
 tokio = { version = "1.33.0", default-features = false, features = ["rt", "macros", "rt-multi-thread", "sync", "fs", "io-util", "time"] }

lib/vector-common/Cargo.toml

@@ -56,7 +56,7 @@ paste = "1.0.14"
 pin-project.workspace = true
 ryu = { version = "1", default-features = false }
 serde_json = { version = "1.0.107", default-features = false, features = ["std", "raw_value"] }
-serde = { version = "1.0.189", optional = true, features = ["derive"] }
+serde = { version = "1.0.190", optional = true, features = ["derive"] }
 smallvec = { version = "1", default-features = false }
 snafu = { version = "0.7", optional = true }
 stream-cancel = { version = "0.8.1", default-features = false }

lib/vector-config-common/src/lib.rs

@@ -1,3 +1,5 @@
+#![deny(warnings)]
+
 // TODO: `darling` is currently strict about accepting only matching literal types for scalar fields i.e. a `f64` field
 // can only be parsed from a string or float literal, but not an integer literal... and float literals have to be in the
 // form of `1000.0`, not `1000`.
@@ -10,10 +12,35 @@
 // `#[configurable(validation(length(min = 1)))]` to indicate the string cannot be empty, when
 // something like `#[configurable(validation(not_empty)]` is a bit more self-evident, and shorter to boot
 
-#![deny(warnings)]
+use std::sync::OnceLock;
+
 pub mod attributes;
 pub mod constants;
 pub mod human_friendly;
 pub mod num;
 pub mod schema;
 pub mod validation;
+
+/// Generate the package name to reach `vector_config` in the output of the macros. This should be
+/// `vector_lib::configurable` in all packages that can import `vector_lib` to allow for a single
+/// import interface, but it needs to explicitly name `vector_config` in all packages that
+/// themselves import `vector_lib`.
+pub fn configurable_package_name_hack() -> proc_macro2::TokenStream {
+    // `TokenStream2` does not implement `Sync`, so we can't use it directly in `OnceLock`. As such,
+    // this hack needs to recreate the package name token stream each time. We can also not return a
+    // string type, which could be stored in the `OnceLock`, as one of the options is a multi-token
+    // value and the string will always be parsed as a single token.
+    static RUNNING_IN_LIB: OnceLock<bool> = OnceLock::new();
+    // We can't use `env!("CARGO_PKG_NAME")` to be able to create a `const`, as that is evaluated
+    // once when this macro package is built rather than when they are evaluated. This has to be
+    // evaluated in the context of the package in which the macro is being expanded.
+    let running_in_lib = *RUNNING_IN_LIB.get_or_init(|| {
+        let package = std::env::var("CARGO_PKG_NAME").expect("Must be built by cargo");
+        package.starts_with("vector-") || package == "file-source" || package == "codecs"
+    });
+    if running_in_lib {
+        syn::parse_quote! { ::vector_config }
+    } else {
+        syn::parse_quote! { ::vector_lib::configurable }
+    }
+}