Merge pull request #393 from alltilla/filterx-load-vars

filterx: `load_vars()`
axoflow · Nov 26, 2024 · 6dcefd6 · 6dcefd6
2 parents e52373d + 18918e4
commit 6dcefd6
Show file tree

Hide file tree

Showing 6 changed files with 174 additions and 7 deletions.
diff --git a/lib/filterx/filterx-globals.c b/lib/filterx/filterx-globals.c
@@ -103,6 +103,7 @@ _simple_init(void)
                                                     filterx_simple_function_dedup_metrics_labels));
   g_assert(filterx_builtin_simple_function_register("len", filterx_simple_function_len));
   g_assert(filterx_builtin_simple_function_register("vars", filterx_simple_function_vars));
+  g_assert(filterx_builtin_simple_function_register("load_vars", filterx_simple_function_load_vars));
   g_assert(filterx_builtin_simple_function_register("lower", filterx_simple_function_lower));
   g_assert(filterx_builtin_simple_function_register("upper", filterx_simple_function_upper));
   g_assert(filterx_builtin_simple_function_register("has_sdata",

diff --git a/lib/filterx/func-vars.c b/lib/filterx/func-vars.c
@@ -25,14 +25,31 @@
 #include "filterx-eval.h"
 #include "object-json.h"
 #include "object-string.h"
+#include "object-primitive.h"
+#include "object-message-value.h"
+#include "object-dict-interface.h"
+#include "filterx-object-istype.h"
+
+#include "scratch-buffers.h"
+#include "str-utils.h"
 
 static gboolean
 _add_to_dict(FilterXVariable *variable, gpointer user_data)
 {
-  FilterXObject *vars = (FilterXObject *) user_data;
+  FilterXObject *vars = ((gpointer *)(user_data))[0];
+  GString *name_buf = ((gpointer *)(user_data))[1];
 
   gssize name_len;
   const gchar *name_str = filterx_variable_get_name(variable, &name_len);
+
+  if (!filterx_variable_is_floating(variable))
+    {
+      g_string_assign(name_buf, "$");
+      g_string_append_len(name_buf, name_str, name_len);
+      name_str = name_buf->str;
+      name_len = name_buf->len;
+    }
+
   FilterXObject *name = filterx_string_new(name_str, name_len);
 
   FilterXObject *value = filterx_variable_get_value(variable);
@@ -58,9 +75,118 @@ filterx_simple_function_vars(FilterXExpr *s, GPtrArray *args)
   FilterXEvalContext *context = filterx_eval_get_context();
   FilterXObject *vars = filterx_json_object_new_empty();
 
-  if (filterx_scope_foreach_variable(context->scope, _add_to_dict, vars))
-    return vars;
+  ScratchBuffersMarker marker;
+  GString *name_buf = scratch_buffers_alloc_and_mark(&marker);
+
+  gpointer user_data[] = { vars, name_buf };
+  if (!filterx_scope_foreach_variable(context->scope, _add_to_dict, user_data))
+    {
+      filterx_object_unref(vars);
+      vars = NULL;
+    }
+
+  scratch_buffers_reclaim_marked(marker);
+  return vars;
+}
+
+static gboolean
+_load_from_dict(FilterXObject *key, FilterXObject *value, gpointer user_data)
+{
+  FilterXExpr *s = ((gpointer *)(user_data))[0];
+  FilterXScope *scope = ((gpointer *)(user_data))[1];
+
+  if (!filterx_object_is_type(key, &FILTERX_TYPE_NAME(string)))
+    {
+      filterx_eval_push_error("Variable name must be a string", s, key);
+      return FALSE;
+    }
+
+  gsize key_len;
+  const gchar *key_str = filterx_string_get_value_ref(key, &key_len);
+  APPEND_ZERO(key_str, key_str, key_len);
+
+  if (key_len == 0)
+    {
+      filterx_eval_push_error("Variable name must not be empty", s, key);
+      return FALSE;
+    }
+
+  gboolean is_floating = key_str[0] != '$';
+  FilterXVariableHandle handle = filterx_map_varname_to_handle(key_str, is_floating ? FX_VAR_FLOATING : FX_VAR_MESSAGE);
+
+  FilterXVariable *variable = NULL;
+  if (is_floating)
+    variable = filterx_scope_register_declared_variable(scope, handle, NULL);
+  else
+    variable = filterx_scope_register_variable(scope, handle, NULL);
+
+  FilterXObject *cloned_value = filterx_object_clone(value);
+  filterx_variable_set_value(variable, cloned_value);
+  filterx_object_unref(cloned_value);
+
+  if (!variable)
+    {
+      filterx_eval_push_error("Failed to register variable", NULL, key);
+      return FALSE;
+    }
+
+  if (debug_flag)
+    {
+      LogMessageValueType type;
+
+      GString *repr = scratch_buffers_alloc();
+      if (!filterx_object_repr(value, repr))
+        filterx_object_marshal_append(value, repr, &type);
+
+      msg_trace("FILTERX LOADV",
+                filterx_expr_format_location_tag(s),
+                evt_tag_str("key", key_str),
+                evt_tag_str("value", repr->str),
+                evt_tag_str("type", value->type->name),
+                evt_tag_str("variable_type", is_floating ? "declared" : "message"));
+    }
+
+  return TRUE;
+}
+
+FilterXObject *
+filterx_simple_function_load_vars(FilterXExpr *s, GPtrArray *args)
+{
+  if (!args || args->len != 1)
+    {
+      filterx_simple_function_argument_error(s, "Incorrect number of arguments", FALSE);
+      return NULL;
+    }
+
+  FilterXObject *vars = g_ptr_array_index(args, 0);
+  FilterXObject *vars_unwrapped = filterx_ref_unwrap_ro(vars);
+  FilterXObject *vars_unmarshalled = NULL;
+
+  if (!filterx_object_is_type(vars_unwrapped, &FILTERX_TYPE_NAME(dict)))
+    {
+      if (!filterx_object_is_type(vars_unwrapped, &FILTERX_TYPE_NAME(message_value)))
+        {
+          filterx_simple_function_argument_error(s, g_strdup_printf("Argument must be dict typed, got %s instead",
+                                                                    vars_unwrapped->type->name), TRUE);
+          return NULL;
+        }
+
+      vars_unmarshalled = filterx_object_unmarshal(vars_unwrapped);
+      vars_unwrapped = NULL;
+
+      if (!filterx_object_is_type(vars_unmarshalled, &FILTERX_TYPE_NAME(dict)))
+        {
+          filterx_simple_function_argument_error(s, g_strdup_printf("Argument must be dict typed, got %s instead",
+                                                                    vars_unmarshalled->type->name), TRUE);
+          filterx_object_unref(vars_unmarshalled);
+          return NULL;
+        }
+    }
+
+  FilterXScope *scope = filterx_eval_get_scope();
+  gpointer user_data[] = { s, scope };
+  gboolean success = filterx_dict_iter(vars_unwrapped ? : vars_unmarshalled, _load_from_dict, user_data);
 
-  filterx_object_unref(vars);
-  return NULL;
+  filterx_object_unref(vars_unmarshalled);
+  return success ? filterx_boolean_new(TRUE) : NULL;
 }
diff --git a/lib/filterx/func-vars.h b/lib/filterx/func-vars.h
@@ -27,5 +27,6 @@
 #include "filterx/expr-function.h"
 
 FilterXObject *filterx_simple_function_vars(FilterXExpr *s, GPtrArray *args);
+FilterXObject *filterx_simple_function_load_vars(FilterXExpr *s, GPtrArray *args);
 
 #endif
diff --git a/news/fx-feature-393-1.md b/news/fx-feature-393-1.md
@@ -0,0 +1 @@
+`vars()`: `$` is now prepended for the names of message variables.
diff --git a/news/fx-feature-393-2.md b/news/fx-feature-393-2.md
@@ -0,0 +1,5 @@
+`load_vars()`: Added new function to load variables from a dict.
+
+Inverse of `vars()`.
+
+Note: FilterX level variables are loaded and `declare`d.
diff --git a/tests/light/functional_tests/filterx/test_filterx.py b/tests/light/functional_tests/filterx/test_filterx.py
@@ -1530,7 +1530,40 @@ def test_vars(config, syslog_ng):
 
     assert file_true.get_stats()["processed"] == 1
     assert "processed" not in file_false.get_stats()
-    assert file_true.read_log() == '{"logmsg_variable":"foo","pipeline_level_variable":"baz","log":{"body":"foobar","attributes":{"attribute":42}},"js_array":[1,2,3,[4,5,6]]}\n'
+    assert file_true.read_log() == '{"$logmsg_variable":"foo","pipeline_level_variable":"baz","log":{"body":"foobar","attributes":{"attribute":42}},"js_array":[1,2,3,[4,5,6]]}\n'
+
+
+def test_load_vars(config, syslog_ng):
+    (file_true, file_false) = create_config(
+        config,
+        filterx_expr_1=r"""
+            my_vars = {
+                "$" + "logmsg_variable": "foo",  # "$" + "...": workaround for templating
+                "pipeline_level_variable": "baz",
+                "log": {
+                    "body": "foobar",
+                    "attributes": {
+                        "attribute": 42
+                    }
+                },
+                "js_array": [1, 2, 3, [4, 5, 6]]
+            };
+            load_vars(my_vars);
+        """,
+        filterx_expr_2=r"""
+            $MSG = {
+                "$" + "logmsg_variable": $logmsg_variable,  # "$" + "...": workaround for templating
+                "pipeline_level_variable": pipeline_level_variable,
+                "log": log,
+                "js_array": js_array
+            };
+        """,
+    )
+    syslog_ng.start(config)
+
+    assert file_true.get_stats()["processed"] == 1
+    assert "processed" not in file_false.get_stats()
+    assert file_true.read_log() == '{"$logmsg_variable":"foo","pipeline_level_variable":"baz","log":{"body":"foobar","attributes":{"attribute":42}},"js_array":[1,2,3,[4,5,6]]}\n'
 
 
 def test_macro_caching(config, syslog_ng):
@@ -2222,7 +2255,7 @@ def test_done(config, syslog_ng):
 
     assert file_true.get_stats()["processed"] == 1
     assert "processed" not in file_false.get_stats()
-    assert file_true.read_log() == '{"MESSAGE":"foo","var_wont_change":true}\n'
+    assert file_true.read_log() == '{"$MESSAGE":"foo","var_wont_change":true}\n'
 
 
 def test_parse_xml(config, syslog_ng):
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		`vars()`: `$` is now prepended for the names of message variables.