feat(route transform): Add option to enable/disable unmatched output (v…

…ectordotdev#18309)

* feat(route transform): Add option to enable/disable unmatched output

This commit adds a new boolean option `reroute_unmatched` to the `route` transform.

It is inspired on the `reroute_dropped` option in the `remap` transform, allowing the user to control
if they want or not the `<transform_name>._unmatched` output to be created and used. For backwards
compatibility, this new option defaults to `true`.

Users that are not interested in processing unmatched events can use this option to avoid the following
warning on Vector startup, and also to remove the unnecessary `_unmatched` output in `vector top`:
```
WARN vector::config::loading: Transform "route._unmatched" has no consumers
```

Signed-off-by: Hugo Hromic <[email protected]>

* Update existing `can_serialize_remap` test

* Add test for the new `reroute_unmatched` option

---------

Signed-off-by: Hugo Hromic <[email protected]>

src/transforms/route.rs

@@ -19,6 +19,7 @@ pub(crate) const UNMATCHED_ROUTE: &str = "_unmatched";
 #[derive(Clone)]
 pub struct Route {
     conditions: Vec<(String, Condition)>,
+    reroute_unmatched: bool,
 }
 
 impl Route {
@@ -28,7 +29,10 @@ impl Route {
             let condition = condition.build(&context.enrichment_tables)?;
             conditions.push((output_name.clone(), condition));
         }
-        Ok(Self { conditions })
+        Ok(Self {
+            conditions,
+            reroute_unmatched: config.reroute_unmatched,
+        })
     }
 }
 
@@ -47,7 +51,7 @@ impl SyncTransform for Route {
                 check_failed += 1;
             }
         }
-        if check_failed == self.conditions.len() {
+        if self.reroute_unmatched && check_failed == self.conditions.len() {
             output.push(Some(UNMATCHED_ROUTE), event);
         }
     }
@@ -61,11 +65,24 @@ impl SyncTransform for Route {
 #[derive(Clone, Debug)]
 #[serde(deny_unknown_fields)]
 pub struct RouteConfig {
+    /// Reroutes unmatched events to a named output instead of silently discarding them.
+    ///
+    /// Normally, if an event doesn't match any defined route, it is sent to the `<transform_name>._unmatched`
+    /// output for further processing. In some cases, you may want to simply discard unmatched events and not
+    /// process them any further.
+    ///
+    /// In these cases, `reroute_unmatched` can be set to `false` to disable the `<transform_name>._unmatched`
+    /// output and instead silently discard any unmatched events.
+    #[serde(default = "crate::serde::default_true")]
+    #[configurable(metadata(docs::human_name = "Reroute Unmatched Events"))]
+    reroute_unmatched: bool,
+
     /// A table of route identifiers to logical conditions representing the filter of the route.
     ///
     /// Each route can then be referenced as an input by other components with the name
-    /// `<transform_name>.<route_id>`. If an event doesn’t match any route, it is sent to the
-    /// `<transform_name>._unmatched` output.
+    /// `<transform_name>.<route_id>`. If an event doesn’t match any route, and if `reroute_unmatched`
+    /// is set to `true` (the default), it is sent to the `<transform_name>._unmatched` output.
+    /// Otherwise, the unmatched event is instead silently discarded.
     ///
     /// Both `_unmatched`, as well as `_default`, are reserved output names and thus cannot be used
     /// as a route name.
@@ -76,6 +93,7 @@ pub struct RouteConfig {
 impl GenerateConfig for RouteConfig {
     fn generate_config() -> toml::Value {
         toml::Value::try_from(Self {
+            reroute_unmatched: true,
             route: IndexMap::new(),
         })
         .unwrap()
@@ -118,10 +136,12 @@ impl TransformConfig for RouteConfig {
                     .with_port(output_name)
             })
             .collect();
-        result.push(
-            TransformOutput::new(DataType::all(), clone_input_definitions(input_definitions))
-                .with_port(UNMATCHED_ROUTE),
-        );
+        if self.reroute_unmatched {
+            result.push(
+                TransformOutput::new(DataType::all(), clone_input_definitions(input_definitions))
+                    .with_port(UNMATCHED_ROUTE),
+            );
+        }
         result
     }
 
@@ -162,7 +182,7 @@ mod test {
 
         assert_eq!(
             serde_json::to_string(&config).unwrap(),
-            r#"{"route":{"first":{"type":"vrl","source":".message == \"hello world\"","runtime":"ast"}}}"#
+            r#"{"reroute_unmatched":true,"route":{"first":{"type":"vrl","source":".message == \"hello world\"","runtime":"ast"}}}"#
         );
     }
 
@@ -293,6 +313,45 @@ mod test {
         }
     }
 
+    #[test]
+    fn route_no_unmatched_output() {
+        let output_names = vec!["first", "second", "third", UNMATCHED_ROUTE];
+        let event = Event::try_from(serde_json::json!({"message": "NOPE"})).unwrap();
+        let config = toml::from_str::<RouteConfig>(
+            r#"
+            reroute_unmatched = false
+
+            route.first.type = "vrl"
+            route.first.source = '.message == "hello world"'
+
+            route.second.type = "vrl"
+            route.second.source = '.second == "second"'
+
+            route.third.type = "vrl"
+            route.third.source = '.third == "third"'
+        "#,
+        )
+        .unwrap();
+
+        let mut transform = Route::new(&config, &Default::default()).unwrap();
+        let mut outputs = TransformOutputsBuf::new_with_capacity(
+            output_names
+                .iter()
+                .map(|output_name| {
+                    TransformOutput::new(DataType::all(), HashMap::new())
+                        .with_port(output_name.to_owned())
+                })
+                .collect(),
+            1,
+        );
+
+        transform.transform(event.clone(), &mut outputs);
+        for output_name in output_names {
+            let events: Vec<_> = outputs.drain_named(output_name).collect();
+            assert_eq!(events.len(), 0);
+        }
+    }
+
     #[tokio::test]
     async fn route_metrics_with_output_tag() {
         init_test();

website/cue/reference/components/transforms/base/route.cue

@@ -1,20 +1,37 @@
 package metadata
 
-base: components: transforms: route: configuration: route: {
-	description: """
-		A table of route identifiers to logical conditions representing the filter of the route.
+base: components: transforms: route: configuration: {
+	reroute_unmatched: {
+		description: """
+			Reroutes unmatched events to a named output instead of silently discarding them.
 
-		Each route can then be referenced as an input by other components with the name
-		`<transform_name>.<route_id>`. If an event doesn’t match any route, it is sent to the
-		`<transform_name>._unmatched` output.
+			Normally, if an event doesn't match any defined route, it is sent to the `<transform_name>._unmatched`
+			output for further processing. In some cases, you may want to simply discard unmatched events and not
+			process them any further.
 
-		Both `_unmatched`, as well as `_default`, are reserved output names and thus cannot be used
-		as a route name.
-		"""
-	required: false
-	type: object: options: "*": {
-		description: "An individual route."
-		required:    true
-		type: condition: {}
+			In these cases, `reroute_unmatched` can be set to `false` to disable the `<transform_name>._unmatched`
+			output and instead silently discard any unmatched events.
+			"""
+		required: false
+		type: bool: default: true
+	}
+	route: {
+		description: """
+			A table of route identifiers to logical conditions representing the filter of the route.
+
+			Each route can then be referenced as an input by other components with the name
+			`<transform_name>.<route_id>`. If an event doesn’t match any route, and if `reroute_unmatched`
+			is set to `true` (the default), it is sent to the `<transform_name>._unmatched` output.
+			Otherwise, the unmatched event is instead silently discarded.
+
+			Both `_unmatched`, as well as `_default`, are reserved output names and thus cannot be used
+			as a route name.
+			"""
+		required: false
+		type: object: options: "*": {
+			description: "An individual route."
+			required:    true
+			type: condition: {}
+		}
 	}
 }