From 874c0c079538ce39c8259e1ca477fab49d3acddd Mon Sep 17 00:00:00 2001 From: Alexander Rodin Date: Mon, 16 Mar 2020 15:12:43 +0300 Subject: [PATCH] enhancement(lua transform): Add `version` configuration option (#2056) * enhancement(lua transform): Add "version" configuration option Signed-off-by: Alexander Rodin * Don't expose internal typetag names `lua_v1` and `lua_v1` to users Signed-off-by: Alexander Rodin * Add `version` configuration option to the documentation Signed-off-by: Alexander Rodin * Use independent implementations of `rlua::UserData` for `Event` Signed-off-by: Alexander Rodin * Fix benchmarks Signed-off-by: Alexander Rodin * Benchmark both version 1 and version 2 Signed-off-by: Alexander Rodin * Don't show version 2 in the docs Signed-off-by: Alexander Rodin * Add behavior tests for `version` configuration option Signed-off-by: Alexander Rodin * Update CODEOWNERS to match the new directory structure Signed-off-by: Alexander Rodin --- .github/CODEOWNERS | 2 +- .meta/transforms/lua.toml | 8 + benches/lua.rs | 60 ++- config/vector.spec.toml | 8 + src/transforms/lua/mod.rs | 78 ++++ src/transforms/{lua.rs => lua/v1/mod.rs} | 68 ++-- src/transforms/lua/v2/mod.rs | 485 +++++++++++++++++++++++ tests/behavior/transforms/lua.toml | 58 +++ website/docs/reference/transforms/lua.md | 26 ++ 9 files changed, 755 insertions(+), 38 deletions(-) create mode 100644 src/transforms/lua/mod.rs rename src/transforms/{lua.rs => lua/v1/mod.rs} (84%) create mode 100644 src/transforms/lua/v2/mod.rs create mode 100644 tests/behavior/transforms/lua.toml diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index aceb017fbaec4..5c2f4853ffccc 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -64,7 +64,7 @@ /src/transforms/grok_parser.rs @lukesteensen /src/transforms/json_parser.rs @LucioFranco /src/transforms/log_to_metric.rs @lukesteensen -/src/transforms/lua.rs @a-rodin +/src/transforms/lua/ @a-rodin /src/transforms/merge.rs @MOZGIII /src/transforms/regex_parser.rs @lukesteensen /src/transforms/remove_fields.rs @LucioFranco diff --git a/.meta/transforms/lua.toml b/.meta/transforms/lua.toml index dfeec6bd3473e..fa3ea492b04ea 100644 --- a/.meta/transforms/lua.toml +++ b/.meta/transforms/lua.toml @@ -10,6 +10,14 @@ requirements = {} <%= render("_partials/_component_options.toml", type: "transform", name: "lua") %> +[transforms.lua.options.version] +type = "string" +common = true +required = false +description = "transform API version" +default = "1" +enum = { 1 = "transform API version 1" } + [transforms.lua.options.source] type = "string" common = true diff --git a/benches/lua.rs b/benches/lua.rs index 0d642eb4018a1..8d2a93a4cea7f 100644 --- a/benches/lua.rs +++ b/benches/lua.rs @@ -12,10 +12,12 @@ fn add_fields(c: &mut Criterion) { let key = "the key"; let value = "this is the value"; - let key_atom = key.into(); - let value_bytes = value.into(); - let key_atom2 = key.into(); - let value_bytes2 = value.into(); + let key_atom_native = key.into(); + let value_bytes_native = value.into(); + let key_atom_v1 = key.into(); + let value_bytes_v1 = value.into(); + let key_atom_v2 = key.into(); + let value_bytes_v2 = value.into(); c.bench( "lua_add_fields", @@ -30,22 +32,37 @@ fn add_fields(c: &mut Criterion) { for _ in 0..num_events { let event = Event::new_empty_log(); let event = transform.transform(event).unwrap(); - assert_eq!(event.as_log()[&key_atom], value_bytes); + assert_eq!(event.as_log()[&key_atom_native], value_bytes_native); } }, ) }) - .with_function("lua", move |b| { + .with_function("v1", move |b| { b.iter_with_setup( || { let source = format!("event['{}'] = '{}'", key, value); - transforms::lua::Lua::new(&source, vec![]).unwrap() + transforms::lua::v1::Lua::new(&source, vec![]).unwrap() }, |mut transform| { for _ in 0..num_events { let event = Event::new_empty_log(); let event = transform.transform(event).unwrap(); - assert_eq!(event.as_log()[&key_atom2], value_bytes2); + assert_eq!(event.as_log()[&key_atom_v1], value_bytes_v1); + } + }, + ) + }) + .with_function("v2", move |b| { + b.iter_with_setup( + || { + let source = format!("event['{}'] = '{}'", key, value); + transforms::lua::v2::Lua::new(&source, vec![]).unwrap() + }, + |mut transform| { + for _ in 0..num_events { + let event = Event::new_empty_log(); + let event = transform.transform(event).unwrap(); + assert_eq!(event.as_log()[&key_atom_v2], value_bytes_v2); } }, ) @@ -83,7 +100,30 @@ fn field_filter(c: &mut Criterion) { }, ) }) - .with_function("lua", move |b| { + .with_function("v1", move |b| { + b.iter_with_setup( + || { + let source = r#" + if event["the_field"] ~= "0" then + event = nil + end + "#; + transforms::lua::v1::Lua::new(&source, vec![]).unwrap() + }, + |mut transform| { + let num = (0..num_events) + .map(|i| { + let mut event = Event::new_empty_log(); + event.as_mut_log().insert("the_field", (i % 10).to_string()); + event + }) + .filter_map(|r| transform.transform(r)) + .count(); + assert_eq!(num, num_events / 10); + }, + ) + }) + .with_function("v2", move |b| { b.iter_with_setup( || { let source = r#" @@ -91,7 +131,7 @@ fn field_filter(c: &mut Criterion) { event = nil end "#; - transforms::lua::Lua::new(&source, vec![]).unwrap() + transforms::lua::v2::Lua::new(&source, vec![]).unwrap() }, |mut transform| { let num = (0..num_events) diff --git a/config/vector.spec.toml b/config/vector.spec.toml index 767777ec4a4b0..db27044a538c7 100644 --- a/config/vector.spec.toml +++ b/config/vector.spec.toml @@ -1869,6 +1869,14 @@ end # * type: [string] search_dirs = ["/etc/vector/lua"] + # transform API version + # + # * optional + # * default: "1" + # * type: string + # * must be: "1" (if supplied) + version = "1" + # Accepts and outputs `log` events allowing you to merge partial log events into a single event. [transforms.merge] # The component type. This is a required field that tells Vector which diff --git a/src/transforms/lua/mod.rs b/src/transforms/lua/mod.rs new file mode 100644 index 0000000000000..da15366529348 --- /dev/null +++ b/src/transforms/lua/mod.rs @@ -0,0 +1,78 @@ +pub mod v1; +pub mod v2; + +use crate::{ + topology::config::{DataType, TransformConfig, TransformContext, TransformDescription}, + transforms::Transform, +}; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Debug)] +enum V1 { + #[serde(rename = "1")] + V1, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] +pub struct LuaConfigV1 { + version: Option, + #[serde(flatten)] + config: v1::LuaConfig, +} + +#[derive(Serialize, Deserialize, Debug)] +enum V2 { + #[serde(rename = "2")] + V2, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] +pub struct LuaConfigV2 { + version: V2, + #[serde(flatten)] + config: v2::LuaConfig, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(untagged)] +pub enum LuaConfig { + V1(LuaConfigV1), + V2(LuaConfigV2), +} + +inventory::submit! { + TransformDescription::new_without_default::("lua") +} + +#[typetag::serde(name = "lua")] +impl TransformConfig for LuaConfig { + fn build(&self, cx: TransformContext) -> crate::Result> { + match self { + LuaConfig::V1(v1) => v1.config.build(cx), + LuaConfig::V2(v2) => v2.config.build(cx), + } + } + + fn input_type(&self) -> DataType { + match self { + LuaConfig::V1(v1) => v1.config.input_type(), + LuaConfig::V2(v2) => v2.config.input_type(), + } + } + + fn output_type(&self) -> DataType { + match self { + LuaConfig::V1(v1) => v1.config.output_type(), + LuaConfig::V2(v2) => v2.config.output_type(), + } + } + + fn transform_type(&self) -> &'static str { + match self { + LuaConfig::V1(v1) => v1.config.transform_type(), + LuaConfig::V2(v2) => v2.config.transform_type(), + } + } +} diff --git a/src/transforms/lua.rs b/src/transforms/lua/v1/mod.rs similarity index 84% rename from src/transforms/lua.rs rename to src/transforms/lua/v1/mod.rs index 8d2acaba51d15..de20067b2287b 100644 --- a/src/transforms/lua.rs +++ b/src/transforms/lua/v1/mod.rs @@ -1,7 +1,7 @@ -use super::Transform; use crate::{ event::{Event, Value}, - topology::config::{DataType, TransformConfig, TransformContext, TransformDescription}, + topology::config::{DataType, TransformContext}, + transforms::Transform, }; use serde::{Deserialize, Serialize}; use snafu::{ResultExt, Snafu}; @@ -20,28 +20,29 @@ pub struct LuaConfig { search_dirs: Vec, } -inventory::submit! { - TransformDescription::new_without_default::("lua") -} - -#[typetag::serde(name = "lua")] -impl TransformConfig for LuaConfig { - fn build(&self, _cx: TransformContext) -> crate::Result> { +// Implementation of methods from `TransformConfig` +// Note that they are implemented as struct methods instead of trait implementation methods +// because `TransformConfig` trait requires specification of a unique `typetag::serde` name. +// Specifying some name (for example, "lua_v*") results in this name being listed among +// possible configuration options for `transforms` section, but such internal name should not +// be exposed to users. +impl LuaConfig { + pub fn build(&self, _cx: TransformContext) -> crate::Result> { Lua::new(&self.source, self.search_dirs.clone()).map(|l| { let b: Box = Box::new(l); b }) } - fn input_type(&self) -> DataType { + pub fn input_type(&self) -> DataType { DataType::Log } - fn output_type(&self) -> DataType { + pub fn output_type(&self) -> DataType { DataType::Log } - fn transform_type(&self) -> &'static str { + pub fn transform_type(&self) -> &'static str { "lua" } } @@ -59,6 +60,13 @@ pub struct Lua { invocations_after_gc: usize, } +// This wrapping structure is added in order to make it possible to have independent implementations +// of `rlua::UserData` trait for event in version 1 and version 2 of the transform. +#[derive(Clone)] +struct LuaEvent { + inner: Event, +} + impl Lua { pub fn new(source: &str, search_dirs: Vec) -> crate::Result { let lua = rlua::Lua::new(); @@ -95,11 +103,13 @@ impl Lua { let result = self.lua.context(|ctx| { let globals = ctx.globals(); - globals.set("event", event)?; + globals.set("event", LuaEvent { inner: event })?; let func = ctx.named_registry_value::<_, rlua::Function<'_>>("vector_func")?; func.call(())?; - globals.get::<_, Option>("event") + globals + .get::<_, Option>("event") + .map(|option| option.map(|lua_event| lua_event.inner)) }); self.invocations_after_gc += 1; if self.invocations_after_gc % GC_INTERVAL == 0 { @@ -122,26 +132,26 @@ impl Transform for Lua { } } -impl rlua::UserData for Event { +impl rlua::UserData for LuaEvent { fn add_methods<'lua, M: rlua::UserDataMethods<'lua, Self>>(methods: &mut M) { methods.add_meta_method_mut( rlua::MetaMethod::NewIndex, |_ctx, this, (key, value): (String, Option>)| { match value { Some(rlua::Value::String(string)) => { - this.as_mut_log().insert(key, string.as_bytes()); + this.inner.as_mut_log().insert(key, string.as_bytes()); } Some(rlua::Value::Integer(integer)) => { - this.as_mut_log().insert(key, Value::Integer(integer)); + this.inner.as_mut_log().insert(key, Value::Integer(integer)); } Some(rlua::Value::Number(number)) => { - this.as_mut_log().insert(key, Value::Float(number)); + this.inner.as_mut_log().insert(key, Value::Float(number)); } Some(rlua::Value::Boolean(boolean)) => { - this.as_mut_log().insert(key, Value::Boolean(boolean)); + this.inner.as_mut_log().insert(key, Value::Boolean(boolean)); } Some(rlua::Value::Nil) | None => { - this.as_mut_log().remove(&key.into()); + this.inner.as_mut_log().remove(&key.into()); } _ => { info!( @@ -150,7 +160,7 @@ impl rlua::UserData for Event { field = key.as_str(), rate_limit_secs = 30 ); - this.as_mut_log().remove(&key.into()); + this.inner.as_mut_log().remove(&key.into()); } } @@ -159,7 +169,7 @@ impl rlua::UserData for Event { ); methods.add_meta_method(rlua::MetaMethod::Index, |ctx, this, key: String| { - if let Some(value) = this.as_log().get(&key.into()) { + if let Some(value) = this.inner.as_log().get(&key.into()) { let string = ctx.create_string(&value.as_bytes())?; Ok(Some(string)) } else { @@ -167,21 +177,25 @@ impl rlua::UserData for Event { } }); - methods.add_meta_function(rlua::MetaMethod::Pairs, |ctx, event: Event| { + methods.add_meta_function(rlua::MetaMethod::Pairs, |ctx, event: LuaEvent| { let state = ctx.create_table()?; { - let keys = - ctx.create_table_from(event.as_log().keys().map(|k| (k.to_string(), true)))?; + let keys = ctx.create_table_from( + event.inner.as_log().keys().map(|k| (k.to_string(), true)), + )?; state.set("event", event)?; state.set("keys", keys)?; } let function = ctx.create_function(|ctx, (state, prev): (rlua::Table, Option)| { - let event: Event = state.get("event")?; + let event: LuaEvent = state.get("event")?; let keys: rlua::Table = state.get("keys")?; let next: rlua::Function = ctx.globals().get("next")?; let key: Option = next.call((keys, prev))?; - match key.clone().and_then(|k| event.as_log().get(&k.into())) { + match key + .clone() + .and_then(|k| event.inner.as_log().get(&k.into())) + { Some(value) => Ok((key, Some(ctx.create_string(&value.as_bytes())?))), None => Ok((None, None)), } diff --git a/src/transforms/lua/v2/mod.rs b/src/transforms/lua/v2/mod.rs new file mode 100644 index 0000000000000..de20067b2287b --- /dev/null +++ b/src/transforms/lua/v2/mod.rs @@ -0,0 +1,485 @@ +use crate::{ + event::{Event, Value}, + topology::config::{DataType, TransformContext}, + transforms::Transform, +}; +use serde::{Deserialize, Serialize}; +use snafu::{ResultExt, Snafu}; + +#[derive(Debug, Snafu)] +enum BuildError { + #[snafu(display("Lua error: {}", source))] + InvalidLua { source: rlua::Error }, +} + +#[derive(Deserialize, Serialize, Debug)] +#[serde(deny_unknown_fields)] +pub struct LuaConfig { + source: String, + #[serde(default)] + search_dirs: Vec, +} + +// Implementation of methods from `TransformConfig` +// Note that they are implemented as struct methods instead of trait implementation methods +// because `TransformConfig` trait requires specification of a unique `typetag::serde` name. +// Specifying some name (for example, "lua_v*") results in this name being listed among +// possible configuration options for `transforms` section, but such internal name should not +// be exposed to users. +impl LuaConfig { + pub fn build(&self, _cx: TransformContext) -> crate::Result> { + Lua::new(&self.source, self.search_dirs.clone()).map(|l| { + let b: Box = Box::new(l); + b + }) + } + + pub fn input_type(&self) -> DataType { + DataType::Log + } + + pub fn output_type(&self) -> DataType { + DataType::Log + } + + pub fn transform_type(&self) -> &'static str { + "lua" + } +} + +// Lua's garbage collector sometimes seems to be not executed automatically on high event rates, +// which leads to leak-like RAM consumption pattern. This constant sets the number of invocations of +// the Lua transform after which GC would be called, thus ensuring that the RAM usage is not too high. +// +// This constant is larger than 1 because calling GC is an expensive operation, so doing it +// after each transform would have significant footprint on the performance. +const GC_INTERVAL: usize = 16; + +pub struct Lua { + lua: rlua::Lua, + invocations_after_gc: usize, +} + +// This wrapping structure is added in order to make it possible to have independent implementations +// of `rlua::UserData` trait for event in version 1 and version 2 of the transform. +#[derive(Clone)] +struct LuaEvent { + inner: Event, +} + +impl Lua { + pub fn new(source: &str, search_dirs: Vec) -> crate::Result { + let lua = rlua::Lua::new(); + + let additional_paths = search_dirs + .into_iter() + .map(|d| format!("{}/?.lua", d)) + .collect::>() + .join(";"); + + lua.context(|ctx| { + if !additional_paths.is_empty() { + let package = ctx.globals().get::<_, rlua::Table<'_>>("package")?; + let current_paths = package + .get::<_, String>("path") + .unwrap_or_else(|_| ";".to_string()); + let paths = format!("{};{}", additional_paths, current_paths); + package.set("path", paths)?; + } + + let func = ctx.load(&source).into_function()?; + ctx.set_named_registry_value("vector_func", func)?; + Ok(()) + }) + .context(InvalidLua)?; + + Ok(Self { + lua, + invocations_after_gc: 0, + }) + } + + fn process(&mut self, event: Event) -> Result, rlua::Error> { + let result = self.lua.context(|ctx| { + let globals = ctx.globals(); + + globals.set("event", LuaEvent { inner: event })?; + + let func = ctx.named_registry_value::<_, rlua::Function<'_>>("vector_func")?; + func.call(())?; + globals + .get::<_, Option>("event") + .map(|option| option.map(|lua_event| lua_event.inner)) + }); + self.invocations_after_gc += 1; + if self.invocations_after_gc % GC_INTERVAL == 0 { + self.lua.gc_collect()?; + self.invocations_after_gc = 0; + } + result + } +} + +impl Transform for Lua { + fn transform(&mut self, event: Event) -> Option { + match self.process(event) { + Ok(event) => event, + Err(err) => { + error!(message = "Error in lua script; discarding event.", error = %format_error(&err), rate_limit_secs = 30); + None + } + } + } +} + +impl rlua::UserData for LuaEvent { + fn add_methods<'lua, M: rlua::UserDataMethods<'lua, Self>>(methods: &mut M) { + methods.add_meta_method_mut( + rlua::MetaMethod::NewIndex, + |_ctx, this, (key, value): (String, Option>)| { + match value { + Some(rlua::Value::String(string)) => { + this.inner.as_mut_log().insert(key, string.as_bytes()); + } + Some(rlua::Value::Integer(integer)) => { + this.inner.as_mut_log().insert(key, Value::Integer(integer)); + } + Some(rlua::Value::Number(number)) => { + this.inner.as_mut_log().insert(key, Value::Float(number)); + } + Some(rlua::Value::Boolean(boolean)) => { + this.inner.as_mut_log().insert(key, Value::Boolean(boolean)); + } + Some(rlua::Value::Nil) | None => { + this.inner.as_mut_log().remove(&key.into()); + } + _ => { + info!( + message = + "Could not set field to Lua value of invalid type, dropping field", + field = key.as_str(), + rate_limit_secs = 30 + ); + this.inner.as_mut_log().remove(&key.into()); + } + } + + Ok(()) + }, + ); + + methods.add_meta_method(rlua::MetaMethod::Index, |ctx, this, key: String| { + if let Some(value) = this.inner.as_log().get(&key.into()) { + let string = ctx.create_string(&value.as_bytes())?; + Ok(Some(string)) + } else { + Ok(None) + } + }); + + methods.add_meta_function(rlua::MetaMethod::Pairs, |ctx, event: LuaEvent| { + let state = ctx.create_table()?; + { + let keys = ctx.create_table_from( + event.inner.as_log().keys().map(|k| (k.to_string(), true)), + )?; + state.set("event", event)?; + state.set("keys", keys)?; + } + let function = + ctx.create_function(|ctx, (state, prev): (rlua::Table, Option)| { + let event: LuaEvent = state.get("event")?; + let keys: rlua::Table = state.get("keys")?; + let next: rlua::Function = ctx.globals().get("next")?; + let key: Option = next.call((keys, prev))?; + match key + .clone() + .and_then(|k| event.inner.as_log().get(&k.into())) + { + Some(value) => Ok((key, Some(ctx.create_string(&value.as_bytes())?))), + None => Ok((None, None)), + } + })?; + Ok((function, state)) + }); + } +} + +fn format_error(error: &rlua::Error) -> String { + match error { + rlua::Error::CallbackError { traceback, cause } => format_error(&cause) + "\n" + traceback, + err => err.to_string(), + } +} + +#[cfg(test)] +mod tests { + use super::{format_error, Lua}; + use crate::{ + event::{Event, Value}, + transforms::Transform, + }; + + #[test] + fn lua_add_field() { + let mut transform = Lua::new( + r#" + event["hello"] = "goodbye" + "#, + vec![], + ) + .unwrap(); + + let event = Event::from("program me"); + + let event = transform.transform(event).unwrap(); + + assert_eq!(event.as_log()[&"hello".into()], "goodbye".into()); + } + + #[test] + fn lua_read_field() { + let mut transform = Lua::new( + r#" + _, _, name = string.find(event["message"], "Hello, my name is (%a+).") + event["name"] = name + "#, + vec![], + ) + .unwrap(); + + let event = Event::from("Hello, my name is Bob."); + + let event = transform.transform(event).unwrap(); + + assert_eq!(event.as_log()[&"name".into()], "Bob".into()); + } + + #[test] + fn lua_remove_field() { + let mut transform = Lua::new( + r#" + event["name"] = nil + "#, + vec![], + ) + .unwrap(); + + let mut event = Event::new_empty_log(); + event.as_mut_log().insert("name", "Bob"); + let event = transform.transform(event).unwrap(); + + assert!(event.as_log().get(&"name".into()).is_none()); + } + + #[test] + fn lua_drop_event() { + let mut transform = Lua::new( + r#" + event = nil + "#, + vec![], + ) + .unwrap(); + + let mut event = Event::new_empty_log(); + event.as_mut_log().insert("name", "Bob"); + let event = transform.transform(event); + + assert!(event.is_none()); + } + + #[test] + fn lua_read_empty_field() { + let mut transform = Lua::new( + r#" + if event["non-existant"] == nil then + event["result"] = "empty" + else + event["result"] = "found" + end + "#, + vec![], + ) + .unwrap(); + + let event = Event::new_empty_log(); + let event = transform.transform(event).unwrap(); + + assert_eq!(event.as_log()[&"result".into()], "empty".into()); + } + + #[test] + fn lua_integer_value() { + let mut transform = Lua::new( + r#" + event["number"] = 3 + "#, + vec![], + ) + .unwrap(); + + let event = transform.transform(Event::new_empty_log()).unwrap(); + assert_eq!(event.as_log()[&"number".into()], Value::Integer(3)); + } + + #[test] + fn lua_numeric_value() { + let mut transform = Lua::new( + r#" + event["number"] = 3.14159 + "#, + vec![], + ) + .unwrap(); + + let event = transform.transform(Event::new_empty_log()).unwrap(); + assert_eq!(event.as_log()[&"number".into()], Value::Float(3.14159)); + } + + #[test] + fn lua_boolean_value() { + let mut transform = Lua::new( + r#" + event["bool"] = true + "#, + vec![], + ) + .unwrap(); + + let event = transform.transform(Event::new_empty_log()).unwrap(); + assert_eq!(event.as_log()[&"bool".into()], Value::Boolean(true)); + } + + #[test] + fn lua_non_coercible_value() { + let mut transform = Lua::new( + r#" + event["junk"] = {"asdf"} + "#, + vec![], + ) + .unwrap(); + + let event = transform.transform(Event::new_empty_log()).unwrap(); + assert_eq!(event.as_log().get(&"junk".into()), None); + } + + #[test] + fn lua_non_string_key_write() { + let mut transform = Lua::new( + r#" + event[false] = "hello" + "#, + vec![], + ) + .unwrap(); + + let err = transform.process(Event::new_empty_log()).unwrap_err(); + let err = format_error(&err); + assert!(err.contains("error converting Lua boolean to String"), err); + } + + #[test] + fn lua_non_string_key_read() { + let mut transform = Lua::new( + r#" + print(event[false]) + "#, + vec![], + ) + .unwrap(); + + let err = transform.process(Event::new_empty_log()).unwrap_err(); + let err = format_error(&err); + assert!(err.contains("error converting Lua boolean to String"), err); + } + + #[test] + fn lua_script_error() { + let mut transform = Lua::new( + r#" + error("this is an error") + "#, + vec![], + ) + .unwrap(); + + let err = transform.process(Event::new_empty_log()).unwrap_err(); + let err = format_error(&err); + assert!(err.contains("this is an error"), err); + } + + #[test] + fn lua_syntax_error() { + let err = Lua::new( + r#" + 1234 = sadf <>&*!#@ + "#, + vec![], + ) + .map(|_| ()) + .unwrap_err() + .to_string(); + + assert!(err.contains("syntax error:"), err); + } + + #[test] + fn lua_load_file() { + use std::fs::File; + use std::io::Write; + + let dir = tempfile::tempdir().unwrap(); + + let mut file = File::create(dir.path().join("script2.lua")).unwrap(); + write!( + &mut file, + r#" + local M = {{}} + + local function modify(event2) + event2["new field"] = "new value" + end + M.modify = modify + + return M + "# + ) + .unwrap(); + + let source = r#" + local script2 = require("script2") + script2.modify(event) + "#; + + let mut transform = + Lua::new(source, vec![dir.path().to_string_lossy().into_owned()]).unwrap(); + let event = Event::new_empty_log(); + let event = transform.transform(event).unwrap(); + + assert_eq!(event.as_log()[&"new field".into()], "new value".into()); + } + + #[test] + fn lua_pairs() { + let mut transform = Lua::new( + r#" + for k,v in pairs(event) do + event[k] = k .. v + end + "#, + vec![], + ) + .unwrap(); + + let mut event = Event::new_empty_log(); + event.as_mut_log().insert("name", "Bob"); + event.as_mut_log().insert("friend", "Alice"); + + let event = transform.transform(event).unwrap(); + + assert_eq!(event.as_log()[&"name".into()], "nameBob".into()); + assert_eq!(event.as_log()[&"friend".into()], "friendAlice".into()); + } +} diff --git a/tests/behavior/transforms/lua.toml b/tests/behavior/transforms/lua.toml new file mode 100644 index 0000000000000..b86d308db2905 --- /dev/null +++ b/tests/behavior/transforms/lua.toml @@ -0,0 +1,58 @@ +[transforms.lua_unversioned] + inputs = [] + type = "lua" + source = """ + event["a"], event["b"] = nil, event["a"] + """ +[[tests]] + name = "lua_unversioned" + [tests.input] + insert_at = "lua_unversioned" + type = "log" + [tests.input.log_fields] + a = "example value" + [[tests.outputs]] + extract_from = "lua_unversioned" + [[tests.outputs.conditions]] + "a.exists" = false + "b.equals" = "example value" + +[transforms.lua_v1] + inputs = [] + type = "lua" + version = "1" + source = """ + event["a"], event["b"] = nil, event["a"] + """ +[[tests]] + name = "lua_v1" + [tests.input] + insert_at = "lua_v1" + type = "log" + [tests.input.log_fields] + a = "example value" + [[tests.outputs]] + extract_from = "lua_v1" + [[tests.outputs.conditions]] + "a.exists" = false + "b.equals" = "example value" + +[transforms.lua_v2] + inputs = [] + type = "lua" + version = "2" + source = """ + event["a"], event["b"] = nil, event["a"] + """ +[[tests]] + name = "lua_v2" + [tests.input] + insert_at = "lua_v2" + type = "log" + [tests.input.log_fields] + a = "example value" + [[tests.outputs]] + extract_from = "lua_v2" + [[tests.outputs.conditions]] + "a.exists" = false + "b.equals" = "example value" diff --git a/website/docs/reference/transforms/lua.md b/website/docs/reference/transforms/lua.md index d08d93d658171..bb42073c771fe 100644 --- a/website/docs/reference/transforms/lua.md +++ b/website/docs/reference/transforms/lua.md @@ -46,6 +46,7 @@ import CodeHeader from '@site/src/components/CodeHeader'; end """ # required search_dirs = ["/etc/vector/lua"] # optional, no default + version = "1" # optional, default ``` ## Options @@ -104,6 +105,31 @@ The inline Lua source to evaluate. See [Global Variables](#global-variables) for more info. + + + + + +### version + +transform API version + + + +