Add regex capture and rename capture_bytes to capture (#458)

googleforgames · Feb 9, 2022 · 0fb0994 · 0fb0994
1 parent 2fdc8ac
commit 0fb0994
Show file tree

Hide file tree

Showing 24 changed files with 649 additions and 382 deletions.
diff --git a/Cargo.toml b/Cargo.toml
@@ -56,6 +56,7 @@ prost-types = "=0.9.0"
 rand = "0.8.4"
 serde = { version = "1.0.130", features = ["derive", "rc"] }
 serde_json = "1.0.68"
+serde_regex = "1.1.0"
 serde_yaml = "0.8.21"
 snap = "1.0.5"
 tokio = { version = "1.16.1", features = ["rt-multi-thread", "signal", "test-util", "parking_lot"] }
@@ -70,6 +71,7 @@ eyre = "0.6.5"
 stable-eyre = "0.2.2"
 ipnetwork = "0.18.0"
 futures = "0.3.17"
+regex = "1.5.4"
 
 [target.'cfg(target_os = "linux")'.dependencies]
 sys-info = "0.9.0"

diff --git a/build.rs b/build.rs
@@ -28,10 +28,11 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         "proto/data-plane-api/envoy/type/metadata/v3/metadata.proto",
         "proto/data-plane-api/envoy/type/tracing/v3/custom_tag.proto",
         "proto/udpa/xds/core/v3/resource_name.proto",
-        "proto/quilkin/extensions/filters/capture_bytes/v1alpha1/capture_bytes.proto",
+        "proto/quilkin/extensions/filters/capture/v1alpha1/capture.proto",
         "proto/quilkin/extensions/filters/compress/v1alpha1/compress.proto",
         "proto/quilkin/extensions/filters/concatenate_bytes/v1alpha1/concatenate_bytes.proto",
         "proto/quilkin/extensions/filters/debug/v1alpha1/debug.proto",
+        "proto/quilkin/extensions/filters/debug/v1alpha1/debug.proto",
         "proto/quilkin/extensions/filters/firewall/v1alpha1/firewall.proto",
         "proto/quilkin/extensions/filters/load_balancer/v1alpha1/load_balancer.proto",
         "proto/quilkin/extensions/filters/local_rate_limit/v1alpha1/local_rate_limit.proto",

diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md
@@ -10,7 +10,7 @@
 - [Proxy](./proxy.md)
     - [Proxy Configuration](./proxy-configuration.md)
 - [Filters](./filters.md)
-    - [Capture Bytes](./filters/capture_bytes.md)
+    - [Capture](./filters/capture.md)
     - [Concatenate Bytes](./filters/concatenate_bytes.md)
     - [Compress](./filters/compress.md)
     - [Debug](./filters/debug.md)

diff --git a/docs/src/filters.md b/docs/src/filters.md
@@ -96,7 +96,7 @@ The following metadata are currently used by Quilkin core and built-in filters.
 
 | Name | Type | Description |
 |------|------|-------------|
-| `quilkin.dev/captured_bytes` | `Bytes` | The default key under which the [CaptureBytes] filter puts the byte slices it extracts from each packet. |
+| `quilkin.dev/captured` | `Bytes` | The default key under which the [Capture] filter puts the byte slices it extracts from each packet. |
 
 ### Built-in filters <a name="built-in-filters"></a>
 Quilkin includes several filters out of the box.
@@ -132,7 +132,7 @@ properties:
 required: [ 'name', 'config' ]
 ```
 
-[CaptureBytes]: ./filters/capture_bytes.md
+[Capture]: ./filters/capture.md
 [TokenRouter]: ./filters/token_router.md
 [Debug]: ./filters/debug.md
 [LocalRateLimit]: ./filters/local_rate_limit.md

diff --git a/docs/src/filters/capture_bytes.md → docs/src/filters/capture.md b/docs/src/filters/capture_bytes.md → docs/src/filters/capture.md
@@ -8,9 +8,25 @@ This is often used as a way of retrieving authentication tokens from a packet, a
 [ConcatenateBytes](./concatenate_bytes.md) and 
 [TokenRouter](token_router.md) filter to provide common packet routing utilities.
 
+### Capture strategies
+
+There are multiple strategies for capturing bytes from the packet.
+
+#### Suffix
+Captures bytes from the end of the packet.
+
+#### Prefix
+Captures bytes from the start of the packet.
+
+#### Regex
+Captures bytes using a regular expression. Unlike other capture strategies,
+the regular expression can return one or many values if there are
+multiple matches.
+
+
 #### Filter name
 ```text
-quilkin.extensions.filters.capture_bytes.v1alpha1.CaptureBytes
+quilkin.extensions.filters.capture.v1alpha1.Capture
 ```
 
 ### Configuration Examples
@@ -19,12 +35,12 @@ quilkin.extensions.filters.capture_bytes.v1alpha1.CaptureBytes
 version: v1alpha1
 static:
   filters:
-    - name: quilkin.extensions.filters.capture_bytes.v1alpha1.CaptureBytes
+    - name: quilkin.extensions.filters.capture.v1alpha1.Capture
       config:
-          strategy: PREFIX
           metadataKey: myapp.com/myownkey
-          size: 3
-          remove: false
+          prefix:
+            size: 3
+            remove: false
   endpoints:
     - address: 127.0.0.1:7001
 # ";
@@ -33,12 +49,12 @@ static:
 # quilkin::Builder::from(std::sync::Arc::new(config)).validate().unwrap();
 ```
 
-### Configuration Options ([Rust Doc](../../api/quilkin/filters/capture_bytes/struct.Config.html))
+### Configuration Options ([Rust Doc](../../api/quilkin/filters/capture/struct.Config.html))
 
 ```yaml
 properties:
   strategy:
-    type: string
+    type: object
     description: |
       The selected strategy for capturing the series of bytes from the incoming packet.
        - SUFFIX: Retrieve bytes from the end of the packet.
@@ -47,7 +63,7 @@ properties:
     enum: ['PREFIX', 'SUFFIX']
   metadataKey:
     type: string
-    default: quilkin.dev/captured_bytes
+    default: quilkin.dev/captured
     description: | 
       The key under which the captured bytes are stored in the Filter invocation values.
   size:
@@ -65,7 +81,7 @@ properties:
 
 ### Metrics
 
-* `quilkin_filter_CaptureBytes_packets_dropped`  
+* `quilkin_filter_Capture_packets_dropped`  
   A counter of the total number of packets that have been dropped due to their length being less than the configured
   `size`.
 

diff --git a/docs/src/filters/token_router.md b/docs/src/filters/token_router.md
@@ -38,15 +38,15 @@ static:
 # quilkin::Builder::from(std::sync::Arc::new(config)).validate().unwrap();
 ```
 
-View the [CaptureBytes](./capture_bytes.md) filter documentation for more details.
+View the [CaptureBytes](./capture.md) filter documentation for more details.
 
 ### Configuration Options ([Rust Doc](../../api/quilkin/filters/token_router/struct.Config.html))
 
 ```yaml
 properties:
   metadataKey:
     type: string
-    default: quilkin.dev/captured_bytes
+    default: quilkin.dev/captured
     description: | 
       The key under which the token is stored in the Filter dynamic metadata.
 ```
@@ -68,7 +68,7 @@ properties:
 In combination with several other filters, the `TokenRouter` can be utilised as an authentication and access control
 mechanism for all incoming packets.
 
-Capturing the authentication token from an incoming packet can be implemented via the [CaptureByte](./capture_bytes.md)
+Capturing the authentication token from an incoming packet can be implemented via the [CaptureByte](./capture.md)
 filter, with an example outlined below, or any other filter that populates the configured dynamic metadata key for the
 authentication token to reside.
 
@@ -82,10 +82,11 @@ For example, a configuration would look like:
 version: v1alpha1
 static:
   filters:
-    - name: quilkin.extensions.filters.capture_bytes.v1alpha1.CaptureBytes # Capture and remove the authentication token
+    - name: quilkin.extensions.filters.capture.v1alpha1.Capture # Capture and remove the authentication token
       config:
-          size: 3
-          remove: true
+        suffix:
+            size: 3
+            remove: true
     - name: quilkin.extensions.filters.token_router.v1alpha1.TokenRouter
   endpoints: 
     - address: 127.0.0.1:26000

diff --git a/macros/src/include.rs b/macros/src/include.rs
@@ -42,12 +42,9 @@ impl ToTokens for IncludeProto {
     fn to_tokens(&self, tokens: &mut TokenStream) {
         let id = &self.id;
 
-        let doc_hidden: syn::Attribute = syn::parse_quote!(#![doc(hidden)]);
         let tonic_include_proto: syn::Stmt = syn::parse_quote!(tonic::include_proto!(#id););
-        let items: Vec<syn::Item> = vec![
-            syn::Item::Verbatim(doc_hidden.to_token_stream()),
-            syn::Item::Verbatim(tonic_include_proto.to_token_stream()),
-        ];
+        let items: Vec<syn::Item> =
+            vec![syn::Item::Verbatim(tonic_include_proto.to_token_stream())];
 
         let module = id.split('.').rev().fold::<Vec<_>, _>(items, |acc, module| {
             let module = syn::Ident::new(module, Span::mixed_site());

diff --git a/...apture_bytes/v1alpha1/capture_bytes.proto → ...ns/filters/capture/v1alpha1/capture.proto b/...apture_bytes/v1alpha1/capture_bytes.proto → ...ns/filters/capture/v1alpha1/capture.proto
@@ -16,23 +16,30 @@
 
 syntax = "proto3";
 
-package quilkin.extensions.filters.capture_bytes.v1alpha1;
+package quilkin.extensions.filters.capture.v1alpha1;
 
 import "google/protobuf/wrappers.proto";
 
-message CaptureBytes {
-  enum Strategy {
-    Prefix = 0;
-    Suffix = 1;
+message Capture {
+  message Suffix {
+      uint32 size = 1;
+      google.protobuf.BoolValue remove = 2;
   }
 
-  message StrategyValue {
-    Strategy value = 1;
+  message Prefix {
+      uint32 size = 1;
+      google.protobuf.BoolValue remove = 2;
   }
 
-  StrategyValue strategy = 1;
-  uint32 size = 2;
-  google.protobuf.StringValue metadata_key = 3;
-  google.protobuf.BoolValue remove = 4;
+  message Regex {
+      google.protobuf.StringValue regex = 1;
+  }
+
+  google.protobuf.StringValue metadata_key = 1;
+  oneof strategy {
+      Prefix prefix = 2;
+      Suffix suffix = 3;
+      Regex regex = 4;
+  }
 }
 
diff --git a/src/filters.rs b/src/filters.rs
@@ -26,7 +26,7 @@ mod write;
 pub(crate) mod chain;
 pub(crate) mod manager;
 
-pub mod capture_bytes;
+pub mod capture;
 pub mod compress;
 pub mod concatenate_bytes;
 pub mod debug;