From 622cddc3992aeb6a751b4f1640d467e24c154d88 Mon Sep 17 00:00:00 2001 From: Attila Szakacs Date: Mon, 25 Nov 2024 14:23:15 +0100 Subject: [PATCH 1/3] filterx: add $ to logmessage variables in vars() So we can restore them better. Signed-off-by: Attila Szakacs --- lib/filterx/func-vars.c | 29 +++++++++++++++---- .../functional_tests/filterx/test_filterx.py | 4 +-- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/lib/filterx/func-vars.c b/lib/filterx/func-vars.c index f71f3681a1..a1ef72a8bc 100644 --- a/lib/filterx/func-vars.c +++ b/lib/filterx/func-vars.c @@ -26,13 +26,25 @@ #include "object-json.h" #include "object-string.h" +#include "scratch-buffers.h" + static gboolean _add_to_dict(FilterXVariable *variable, gpointer user_data) { - FilterXObject *vars = (FilterXObject *) user_data; + FilterXObject *vars = ((gpointer *)(user_data))[0]; + GString *name_buf = ((gpointer *)(user_data))[1]; gssize name_len; const gchar *name_str = filterx_variable_get_name(variable, &name_len); + + if (!filterx_variable_is_floating(variable)) + { + g_string_assign(name_buf, "$"); + g_string_append_len(name_buf, name_str, name_len); + name_str = name_buf->str; + name_len = name_buf->len; + } + FilterXObject *name = filterx_string_new(name_str, name_len); FilterXObject *value = filterx_variable_get_value(variable); @@ -58,9 +70,16 @@ filterx_simple_function_vars(FilterXExpr *s, GPtrArray *args) FilterXEvalContext *context = filterx_eval_get_context(); FilterXObject *vars = filterx_json_object_new_empty(); - if (filterx_scope_foreach_variable(context->scope, _add_to_dict, vars)) - return vars; + ScratchBuffersMarker marker; + GString *name_buf = scratch_buffers_alloc_and_mark(&marker); + + gpointer user_data[] = { vars, name_buf }; + if (!filterx_scope_foreach_variable(context->scope, _add_to_dict, user_data)) + { + filterx_object_unref(vars); + vars = NULL; + } - filterx_object_unref(vars); - return NULL; + scratch_buffers_reclaim_marked(marker); + return vars; } diff --git a/tests/light/functional_tests/filterx/test_filterx.py b/tests/light/functional_tests/filterx/test_filterx.py index c7d9cebd00..b53b49830e 100644 --- a/tests/light/functional_tests/filterx/test_filterx.py +++ b/tests/light/functional_tests/filterx/test_filterx.py @@ -1530,7 +1530,7 @@ def test_vars(config, syslog_ng): assert file_true.get_stats()["processed"] == 1 assert "processed" not in file_false.get_stats() - assert file_true.read_log() == '{"logmsg_variable":"foo","pipeline_level_variable":"baz","log":{"body":"foobar","attributes":{"attribute":42}},"js_array":[1,2,3,[4,5,6]]}\n' + assert file_true.read_log() == '{"$logmsg_variable":"foo","pipeline_level_variable":"baz","log":{"body":"foobar","attributes":{"attribute":42}},"js_array":[1,2,3,[4,5,6]]}\n' def test_macro_caching(config, syslog_ng): @@ -2222,7 +2222,7 @@ def test_done(config, syslog_ng): assert file_true.get_stats()["processed"] == 1 assert "processed" not in file_false.get_stats() - assert file_true.read_log() == '{"MESSAGE":"foo","var_wont_change":true}\n' + assert file_true.read_log() == '{"$MESSAGE":"foo","var_wont_change":true}\n' def test_parse_xml(config, syslog_ng): From 8289cc9efbdcc07451e7c1fd7a355de4016970a7 Mon Sep 17 00:00:00 2001 From: Attila Szakacs Date: Mon, 25 Nov 2024 16:21:44 +0100 Subject: [PATCH 2/3] filterx: add load_vars() function Signed-off-by: Attila Szakacs --- lib/filterx/filterx-globals.c | 1 + lib/filterx/func-vars.c | 107 ++++++++++++++++++ lib/filterx/func-vars.h | 1 + .../functional_tests/filterx/test_filterx.py | 33 ++++++ 4 files changed, 142 insertions(+) diff --git a/lib/filterx/filterx-globals.c b/lib/filterx/filterx-globals.c index e53eecac21..f64879c791 100644 --- a/lib/filterx/filterx-globals.c +++ b/lib/filterx/filterx-globals.c @@ -103,6 +103,7 @@ _simple_init(void) filterx_simple_function_dedup_metrics_labels)); g_assert(filterx_builtin_simple_function_register("len", filterx_simple_function_len)); g_assert(filterx_builtin_simple_function_register("vars", filterx_simple_function_vars)); + g_assert(filterx_builtin_simple_function_register("load_vars", filterx_simple_function_load_vars)); g_assert(filterx_builtin_simple_function_register("lower", filterx_simple_function_lower)); g_assert(filterx_builtin_simple_function_register("upper", filterx_simple_function_upper)); g_assert(filterx_builtin_simple_function_register("has_sdata", diff --git a/lib/filterx/func-vars.c b/lib/filterx/func-vars.c index a1ef72a8bc..4251a36140 100644 --- a/lib/filterx/func-vars.c +++ b/lib/filterx/func-vars.c @@ -25,8 +25,13 @@ #include "filterx-eval.h" #include "object-json.h" #include "object-string.h" +#include "object-primitive.h" +#include "object-message-value.h" +#include "object-dict-interface.h" +#include "filterx-object-istype.h" #include "scratch-buffers.h" +#include "str-utils.h" static gboolean _add_to_dict(FilterXVariable *variable, gpointer user_data) @@ -83,3 +88,105 @@ filterx_simple_function_vars(FilterXExpr *s, GPtrArray *args) scratch_buffers_reclaim_marked(marker); return vars; } + +static gboolean +_load_from_dict(FilterXObject *key, FilterXObject *value, gpointer user_data) +{ + FilterXExpr *s = ((gpointer *)(user_data))[0]; + FilterXScope *scope = ((gpointer *)(user_data))[1]; + + if (!filterx_object_is_type(key, &FILTERX_TYPE_NAME(string))) + { + filterx_eval_push_error("Variable name must be a string", s, key); + return FALSE; + } + + gsize key_len; + const gchar *key_str = filterx_string_get_value_ref(key, &key_len); + APPEND_ZERO(key_str, key_str, key_len); + + if (key_len == 0) + { + filterx_eval_push_error("Variable name must not be empty", s, key); + return FALSE; + } + + gboolean is_floating = key_str[0] != '$'; + FilterXVariableHandle handle = filterx_map_varname_to_handle(key_str, is_floating ? FX_VAR_FLOATING : FX_VAR_MESSAGE); + + FilterXVariable *variable = NULL; + if (is_floating) + variable = filterx_scope_register_declared_variable(scope, handle, NULL); + else + variable = filterx_scope_register_variable(scope, handle, NULL); + + FilterXObject *cloned_value = filterx_object_clone(value); + filterx_variable_set_value(variable, cloned_value); + filterx_object_unref(cloned_value); + + if (!variable) + { + filterx_eval_push_error("Failed to register variable", NULL, key); + return FALSE; + } + + if (debug_flag) + { + LogMessageValueType type; + + GString *repr = scratch_buffers_alloc(); + if (!filterx_object_repr(value, repr)) + filterx_object_marshal_append(value, repr, &type); + + msg_trace("FILTERX LOADV", + filterx_expr_format_location_tag(s), + evt_tag_str("key", key_str), + evt_tag_str("value", repr->str), + evt_tag_str("type", value->type->name), + evt_tag_str("variable_type", is_floating ? "declared" : "message")); + } + + return TRUE; +} + +FilterXObject * +filterx_simple_function_load_vars(FilterXExpr *s, GPtrArray *args) +{ + if (!args || args->len != 1) + { + filterx_simple_function_argument_error(s, "Incorrect number of arguments", FALSE); + return NULL; + } + + FilterXObject *vars = g_ptr_array_index(args, 0); + FilterXObject *vars_unwrapped = filterx_ref_unwrap_ro(vars); + FilterXObject *vars_unmarshalled = NULL; + + if (!filterx_object_is_type(vars_unwrapped, &FILTERX_TYPE_NAME(dict))) + { + if (!filterx_object_is_type(vars_unwrapped, &FILTERX_TYPE_NAME(message_value))) + { + filterx_simple_function_argument_error(s, g_strdup_printf("Argument must be dict typed, got %s instead", + vars_unwrapped->type->name), TRUE); + return NULL; + } + + vars_unmarshalled = filterx_object_unmarshal(vars_unwrapped); + vars_unwrapped = NULL; + + if (!filterx_object_is_type(vars_unmarshalled, &FILTERX_TYPE_NAME(dict))) + { + filterx_simple_function_argument_error(s, g_strdup_printf("Argument must be dict typed, got %s instead", + vars_unmarshalled->type->name), TRUE); + filterx_object_unref(vars_unmarshalled); + return NULL; + } + } + + FilterXScope *scope = filterx_eval_get_scope(); + gpointer user_data[] = { s, scope }; + gboolean success = filterx_dict_iter(vars_unwrapped ? : vars_unmarshalled, _load_from_dict, user_data); + + filterx_object_unref(vars_unmarshalled); + return success ? filterx_boolean_new(TRUE) : NULL; +} diff --git a/lib/filterx/func-vars.h b/lib/filterx/func-vars.h index cc32ce7720..88d0161efe 100644 --- a/lib/filterx/func-vars.h +++ b/lib/filterx/func-vars.h @@ -27,5 +27,6 @@ #include "filterx/expr-function.h" FilterXObject *filterx_simple_function_vars(FilterXExpr *s, GPtrArray *args); +FilterXObject *filterx_simple_function_load_vars(FilterXExpr *s, GPtrArray *args); #endif diff --git a/tests/light/functional_tests/filterx/test_filterx.py b/tests/light/functional_tests/filterx/test_filterx.py index b53b49830e..9908cd9fbd 100644 --- a/tests/light/functional_tests/filterx/test_filterx.py +++ b/tests/light/functional_tests/filterx/test_filterx.py @@ -1533,6 +1533,39 @@ def test_vars(config, syslog_ng): assert file_true.read_log() == '{"$logmsg_variable":"foo","pipeline_level_variable":"baz","log":{"body":"foobar","attributes":{"attribute":42}},"js_array":[1,2,3,[4,5,6]]}\n' +def test_load_vars(config, syslog_ng): + (file_true, file_false) = create_config( + config, + filterx_expr_1=r""" + my_vars = { + "$" + "logmsg_variable": "foo", # "$" + "...": workaround for templating + "pipeline_level_variable": "baz", + "log": { + "body": "foobar", + "attributes": { + "attribute": 42 + } + }, + "js_array": [1, 2, 3, [4, 5, 6]] + }; + load_vars(my_vars); + """, + filterx_expr_2=r""" + $MSG = { + "$" + "logmsg_variable": $logmsg_variable, # "$" + "...": workaround for templating + "pipeline_level_variable": pipeline_level_variable, + "log": log, + "js_array": js_array + }; + """, + ) + syslog_ng.start(config) + + assert file_true.get_stats()["processed"] == 1 + assert "processed" not in file_false.get_stats() + assert file_true.read_log() == '{"$logmsg_variable":"foo","pipeline_level_variable":"baz","log":{"body":"foobar","attributes":{"attribute":42}},"js_array":[1,2,3,[4,5,6]]}\n' + + def test_macro_caching(config, syslog_ng): (file_true, file_false) = create_config( config, From 18918e48153313d540f72e3591fc2a437d311a45 Mon Sep 17 00:00:00 2001 From: Attila Szakacs Date: Mon, 25 Nov 2024 16:24:55 +0100 Subject: [PATCH 3/3] news: add entries for #393 Signed-off-by: Attila Szakacs --- news/fx-feature-393-1.md | 1 + news/fx-feature-393-2.md | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 news/fx-feature-393-1.md create mode 100644 news/fx-feature-393-2.md diff --git a/news/fx-feature-393-1.md b/news/fx-feature-393-1.md new file mode 100644 index 0000000000..f1a75064b9 --- /dev/null +++ b/news/fx-feature-393-1.md @@ -0,0 +1 @@ +`vars()`: `$` is now prepended for the names of message variables. diff --git a/news/fx-feature-393-2.md b/news/fx-feature-393-2.md new file mode 100644 index 0000000000..932b99b76a --- /dev/null +++ b/news/fx-feature-393-2.md @@ -0,0 +1,5 @@ +`load_vars()`: Added new function to load variables from a dict. + +Inverse of `vars()`. + +Note: FilterX level variables are loaded and `declare`d.