From 71343bd91ee6e851b430c69ac27753ba0e41104c Mon Sep 17 00:00:00 2001 From: Hugo Hromic Date: Mon, 21 Aug 2023 21:27:56 +0100 Subject: [PATCH] feat(route transform): Add option to enable/disable unmatched output (#18309) * feat(route transform): Add option to enable/disable unmatched output This commit adds a new boolean option `reroute_unmatched` to the `route` transform. It is inspired on the `reroute_dropped` option in the `remap` transform, allowing the user to control if they want or not the `._unmatched` output to be created and used. For backwards compatibility, this new option defaults to `true`. Users that are not interested in processing unmatched events can use this option to avoid the following warning on Vector startup, and also to remove the unnecessary `_unmatched` output in `vector top`: ``` WARN vector::config::loading: Transform "route._unmatched" has no consumers ``` Signed-off-by: Hugo Hromic * Update existing `can_serialize_remap` test * Add test for the new `reroute_unmatched` option --------- Signed-off-by: Hugo Hromic --- src/transforms/route.rs | 77 ++++++++++++++++--- .../components/transforms/base/route.cue | 45 +++++++---- 2 files changed, 99 insertions(+), 23 deletions(-) diff --git a/src/transforms/route.rs b/src/transforms/route.rs index e410277914a8f..721d92a9789ac 100644 --- a/src/transforms/route.rs +++ b/src/transforms/route.rs @@ -19,6 +19,7 @@ pub(crate) const UNMATCHED_ROUTE: &str = "_unmatched"; #[derive(Clone)] pub struct Route { conditions: Vec<(String, Condition)>, + reroute_unmatched: bool, } impl Route { @@ -28,7 +29,10 @@ impl Route { let condition = condition.build(&context.enrichment_tables)?; conditions.push((output_name.clone(), condition)); } - Ok(Self { conditions }) + Ok(Self { + conditions, + reroute_unmatched: config.reroute_unmatched, + }) } } @@ -47,7 +51,7 @@ impl SyncTransform for Route { check_failed += 1; } } - if check_failed == self.conditions.len() { + if self.reroute_unmatched && check_failed == self.conditions.len() { output.push(Some(UNMATCHED_ROUTE), event); } } @@ -61,11 +65,24 @@ impl SyncTransform for Route { #[derive(Clone, Debug)] #[serde(deny_unknown_fields)] pub struct RouteConfig { + /// Reroutes unmatched events to a named output instead of silently discarding them. + /// + /// Normally, if an event doesn't match any defined route, it is sent to the `._unmatched` + /// output for further processing. In some cases, you may want to simply discard unmatched events and not + /// process them any further. + /// + /// In these cases, `reroute_unmatched` can be set to `false` to disable the `._unmatched` + /// output and instead silently discard any unmatched events. + #[serde(default = "crate::serde::default_true")] + #[configurable(metadata(docs::human_name = "Reroute Unmatched Events"))] + reroute_unmatched: bool, + /// A table of route identifiers to logical conditions representing the filter of the route. /// /// Each route can then be referenced as an input by other components with the name - /// `.`. If an event doesn’t match any route, it is sent to the - /// `._unmatched` output. + /// `.`. If an event doesn’t match any route, and if `reroute_unmatched` + /// is set to `true` (the default), it is sent to the `._unmatched` output. + /// Otherwise, the unmatched event is instead silently discarded. /// /// Both `_unmatched`, as well as `_default`, are reserved output names and thus cannot be used /// as a route name. @@ -76,6 +93,7 @@ pub struct RouteConfig { impl GenerateConfig for RouteConfig { fn generate_config() -> toml::Value { toml::Value::try_from(Self { + reroute_unmatched: true, route: IndexMap::new(), }) .unwrap() @@ -118,10 +136,12 @@ impl TransformConfig for RouteConfig { .with_port(output_name) }) .collect(); - result.push( - TransformOutput::new(DataType::all(), clone_input_definitions(input_definitions)) - .with_port(UNMATCHED_ROUTE), - ); + if self.reroute_unmatched { + result.push( + TransformOutput::new(DataType::all(), clone_input_definitions(input_definitions)) + .with_port(UNMATCHED_ROUTE), + ); + } result } @@ -162,7 +182,7 @@ mod test { assert_eq!( serde_json::to_string(&config).unwrap(), - r#"{"route":{"first":{"type":"vrl","source":".message == \"hello world\"","runtime":"ast"}}}"# + r#"{"reroute_unmatched":true,"route":{"first":{"type":"vrl","source":".message == \"hello world\"","runtime":"ast"}}}"# ); } @@ -293,6 +313,45 @@ mod test { } } + #[test] + fn route_no_unmatched_output() { + let output_names = vec!["first", "second", "third", UNMATCHED_ROUTE]; + let event = Event::try_from(serde_json::json!({"message": "NOPE"})).unwrap(); + let config = toml::from_str::( + r#" + reroute_unmatched = false + + route.first.type = "vrl" + route.first.source = '.message == "hello world"' + + route.second.type = "vrl" + route.second.source = '.second == "second"' + + route.third.type = "vrl" + route.third.source = '.third == "third"' + "#, + ) + .unwrap(); + + let mut transform = Route::new(&config, &Default::default()).unwrap(); + let mut outputs = TransformOutputsBuf::new_with_capacity( + output_names + .iter() + .map(|output_name| { + TransformOutput::new(DataType::all(), HashMap::new()) + .with_port(output_name.to_owned()) + }) + .collect(), + 1, + ); + + transform.transform(event.clone(), &mut outputs); + for output_name in output_names { + let events: Vec<_> = outputs.drain_named(output_name).collect(); + assert_eq!(events.len(), 0); + } + } + #[tokio::test] async fn route_metrics_with_output_tag() { init_test(); diff --git a/website/cue/reference/components/transforms/base/route.cue b/website/cue/reference/components/transforms/base/route.cue index 873757ec82411..ad6e02580c813 100644 --- a/website/cue/reference/components/transforms/base/route.cue +++ b/website/cue/reference/components/transforms/base/route.cue @@ -1,20 +1,37 @@ package metadata -base: components: transforms: route: configuration: route: { - description: """ - A table of route identifiers to logical conditions representing the filter of the route. +base: components: transforms: route: configuration: { + reroute_unmatched: { + description: """ + Reroutes unmatched events to a named output instead of silently discarding them. - Each route can then be referenced as an input by other components with the name - `.`. If an event doesn’t match any route, it is sent to the - `._unmatched` output. + Normally, if an event doesn't match any defined route, it is sent to the `._unmatched` + output for further processing. In some cases, you may want to simply discard unmatched events and not + process them any further. - Both `_unmatched`, as well as `_default`, are reserved output names and thus cannot be used - as a route name. - """ - required: false - type: object: options: "*": { - description: "An individual route." - required: true - type: condition: {} + In these cases, `reroute_unmatched` can be set to `false` to disable the `._unmatched` + output and instead silently discard any unmatched events. + """ + required: false + type: bool: default: true + } + route: { + description: """ + A table of route identifiers to logical conditions representing the filter of the route. + + Each route can then be referenced as an input by other components with the name + `.`. If an event doesn’t match any route, and if `reroute_unmatched` + is set to `true` (the default), it is sent to the `._unmatched` output. + Otherwise, the unmatched event is instead silently discarded. + + Both `_unmatched`, as well as `_default`, are reserved output names and thus cannot be used + as a route name. + """ + required: false + type: object: options: "*": { + description: "An individual route." + required: true + type: condition: {} + } } }