feat(pii-scrubbing): PII scrub span.data by default

CHANGELOG.md

@@ -7,7 +7,7 @@
 - Indicate if OS-version may be frozen with '>=' prefix. ([#1945](https://github.com/getsentry/relay/pull/1945))
 - Normalize monitor slug parameters into slugs. ([#1913](https://github.com/getsentry/relay/pull/1913))
 - Smart trim loggers for Java platforms. ([#1941](https://github.com/getsentry/relay/pull/1941))
-
+- PII scrub `span.data` by default. ([#1953](https://github.com/getsentry/relay/pull/1953))
 
 ## 23.3.0
 

py/CHANGELOG.md

@@ -3,6 +3,7 @@
 ## Unreleased
 
 - Add `thread.state` field to protocol. ([#1896](https://github.com/getsentry/relay/pull/1896))
+- PII scrub `span.data` by default. ([#1953](https://github.com/getsentry/relay/pull/1953))
 
 ## 0.8.19
 

relay-general/src/pii/processor.rs

@@ -385,16 +385,14 @@ fn insert_replacement_chunks(rule: &RuleRef, text: &str, output: &mut Vec<Chunk<
 
 #[cfg(test)]
 mod tests {
-    use std::collections::BTreeMap;
-
     use insta::assert_debug_snapshot;
 
     use super::*;
     use crate::pii::{DataScrubbingConfig, PiiConfig, ReplaceRedaction};
     use crate::processor::process_value;
     use crate::protocol::{
-        Addr, DataElement, DebugImage, DebugMeta, Event, ExtraValue, Headers, HttpElement,
-        LogEntry, NativeDebugImage, Request, Span, TagEntry, Tags,
+        Addr, DebugImage, DebugMeta, Event, ExtraValue, Headers, LogEntry, NativeDebugImage,
+        Request, Span, TagEntry, Tags,
     };
     use crate::testutils::assert_annotated_snapshot;
     use crate::types::{Annotated, FromValue, Object, Value};
@@ -1033,17 +1031,17 @@ mod tests {
     }
 
     #[test]
-    fn test_scrub_span_data_not_scrubbed() {
-        let mut span = Annotated::new(Span {
-            data: Annotated::new(DataElement {
-                http: Annotated::new(HttpElement {
-                    query: Annotated::new(Value::String("dance=true".to_owned())),
-                    ..Default::default()
-                }),
-                ..Default::default()
-            }),
-            ..Default::default()
-        });
+    fn test_scrub_span_data_http_not_scrubbed() {
+        let mut span: Annotated<Span> = Annotated::from_json(
+            r#"{
+                "data": {
+                    "http": {
+                        "query": "dance=true"
+                    }
+                }
+            }"#,
+        )
+        .unwrap();
 
         let ds_config = DataScrubbingConfig {
             scrub_data: true,
@@ -1058,19 +1056,18 @@ mod tests {
     }
 
     #[test]
-    fn test_scrub_span_data_is_scrubbed() {
-        let mut span = Annotated::new(Span {
-            data: Annotated::new(DataElement {
-                http: Annotated::new(HttpElement {
-                    query: Annotated::new(Value::String(
-                        "ccnumber=5105105105105100&process_id=123".to_owned(),
-                    )),
-                    ..Default::default()
-                }),
-                ..Default::default()
-            }),
-            ..Default::default()
-        });
+    fn test_scrub_span_data_http_strings_are_scrubbed() {
+        let mut span: Annotated<Span> = Annotated::from_json(
+            r#"{
+                "data": {
+                    "http": {
+                        "query": "ccnumber=5105105105105100&process_id=123",
+                        "fragment": "ccnumber=5105105105105100,process_id=123"
+                    }
+                }
+            }"#,
+        )
+        .unwrap();
 
         let ds_config = DataScrubbingConfig {
             scrub_data: true,
@@ -1085,28 +1082,52 @@ mod tests {
     }
 
     #[test]
-    fn test_scrub_span_data_object_is_scrubbed() {
-        let mut span = Annotated::new(Span {
-            data: Annotated::new(DataElement {
-                http: Annotated::new(HttpElement {
-                    query: Annotated::new(Value::Object({
-                        let mut map = BTreeMap::new();
-                        map.insert(
-                            "ccnumber".to_owned(),
-                            Annotated::new(Value::String("5105105105105100".to_owned())),
-                        );
-                        map.insert(
-                            "process_id".to_owned(),
-                            Annotated::new(Value::String("123".to_owned())),
-                        );
-                        map
-                    })),
-                    ..Default::default()
-                }),
-                ..Default::default()
-            }),
+    fn test_scrub_span_data_http_objects_are_scrubbed() {
+        let mut span: Annotated<Span> = Annotated::from_json(
+            r#"{
+                "data": {
+                    "http": {
+                        "query": {
+                            "ccnumber": "5105105105105100",
+                            "process_id": "123"
+                        },
+                        "fragment": {
+                            "ccnumber": "5105105105105100",
+                            "process_id": "123"
+                        }
+                    }
+                }
+            }"#,
+        )
+        .unwrap();
+
+        let ds_config = DataScrubbingConfig {
+            scrub_data: true,
+            scrub_defaults: true,
             ..Default::default()
-        });
+        };
+        let pii_config = ds_config.pii_config().unwrap().as_ref().unwrap();
+        let mut pii_processor = PiiProcessor::new(pii_config.compiled());
+
+        process_value(&mut span, &mut pii_processor, ProcessingState::root()).unwrap();
+        assert_annotated_snapshot!(span);
+    }
+
+    #[test]
+    fn test_scrub_span_data_untyped_props_are_scrubbed() {
+        let mut span: Annotated<Span> = Annotated::from_json(
+            r#"{
+                "data": {
+                    "untyped": "ccnumber=5105105105105100",
+                    "more_untyped": {
+                        "typed": "no",
+                        "scrubbed": "yes",
+                        "ccnumber": "5105105105105100"
+                    }
+                }
+            }"#,
+        )
+        .unwrap();
 
         let ds_config = DataScrubbingConfig {
             scrub_data: true,

relay-general/src/pii/snapshots/relay_general__pii__processor__tests__scrub_span_data_http_not_scrubbed.snap

@@ -0,0 +1,11 @@
+---
+source: relay-general/src/pii/processor.rs
+expression: span
+---
+{
+  "data": {
+    "http": {
+      "query": "dance=true"
+    }
+  }
+}

relay-general/src/pii/snapshots/relay_general__pii__processor__tests__scrub_span_data_http_objects_are_scrubbed.snap

@@ -0,0 +1,54 @@
+---
+source: relay-general/src/pii/processor.rs
+expression: span
+---
+{
+  "data": {
+    "http": {
+      "fragment": {
+        "ccnumber": "[Filtered]",
+        "process_id": "123"
+      },
+      "query": {
+        "ccnumber": "[Filtered]",
+        "process_id": "123"
+      }
+    }
+  },
+  "_meta": {
+    "data": {
+      "http": {
+        "fragment": {
+          "ccnumber": {
+            "": {
+              "rem": [
+                [
+                  "@creditcard:filter",
+                  "s",
+                  0,
+                  10
+                ]
+              ],
+              "len": 16
+            }
+          }
+        },
+        "query": {
+          "ccnumber": {
+            "": {
+              "rem": [
+                [
+                  "@creditcard:filter",
+                  "s",
+                  0,
+                  10
+                ]
+              ],
+              "len": 16
+            }
+          }
+        }
+      }
+    }
+  }
+}

relay-general/src/pii/snapshots/relay_general__pii__processor__tests__scrub_span_data_is_scrubbed.snap → relay-general/src/pii/snapshots/relay_general__pii__processor__tests__scrub_span_data_http_strings_are_scrubbed.snap

@@ -5,12 +5,26 @@ expression: span
 {
   "data": {
     "http": {
+      "fragment": "ccnumber=[Filtered],process_id=123",
       "query": "ccnumber=[Filtered]&process_id=123"
     }
   },
   "_meta": {
     "data": {
       "http": {
+        "fragment": {
+          "": {
+            "rem": [
+              [
+                "@creditcard:filter",
+                "s",
+                9,
+                19
+              ]
+            ],
+            "len": 40
+          }
+        },
         "query": {
           "": {
             "rem": [

relay-general/src/pii/snapshots/relay_general__pii__processor__tests__scrub_span_data_object_is_scrubbed.snap

@@ -5,6 +5,10 @@ expression: span
 {
   "data": {
     "http": {
+      "fragment": {
+        "ccnumber": "[Filtered]",
+        "process_id": "123"
+      },
       "query": {
         "ccnumber": "[Filtered]",
         "process_id": "123"
@@ -14,6 +18,21 @@ expression: span
   "_meta": {
     "data": {
       "http": {
+        "fragment": {
+          "ccnumber": {
+            "": {
+              "rem": [
+                [
+                  "@creditcard:filter",
+                  "s",
+                  0,
+                  10
+                ]
+              ],
+              "len": 16
+            }
+          }
+        },
         "query": {
           "ccnumber": {
             "": {

relay-general/src/pii/snapshots/relay_general__pii__processor__tests__scrub_span_data_string_is_scrubbed.snap

@@ -0,0 +1,44 @@
+---
+source: relay-general/src/pii/processor.rs
+expression: span
+---
+{
+  "data": {
+    "http": {
+      "fragment": "ccnumber=[Filtered],process_id=123",
+      "query": "ccnumber=[Filtered]&process_id=123"
+    }
+  },
+  "_meta": {
+    "data": {
+      "http": {
+        "fragment": {
+          "": {
+            "rem": [
+              [
+                "@creditcard:filter",
+                "s",
+                9,
+                19
+              ]
+            ],
+            "len": 40
+          }
+        },
+        "query": {
+          "": {
+            "rem": [
+              [
+                "@creditcard:filter",
+                "s",
+                9,
+                19
+              ]
+            ],
+            "len": 40
+          }
+        }
+      }
+    }
+  }
+}

relay-general/src/pii/snapshots/relay_general__pii__processor__tests__scrub_span_data_untyped_props_are_scrubbed.snap

@@ -0,0 +1,46 @@
+---
+source: relay-general/src/pii/processor.rs
+expression: span
+---
+{
+  "data": {
+    "more_untyped": {
+      "ccnumber": "[Filtered]",
+      "scrubbed": "yes",
+      "typed": "no"
+    },
+    "untyped": "ccnumber=[Filtered]"
+  },
+  "_meta": {
+    "data": {
+      "more_untyped": {
+        "ccnumber": {
+          "": {
+            "rem": [
+              [
+                "@creditcard:filter",
+                "s",
+                0,
+                10
+              ]
+            ],
+            "len": 16
+          }
+        }
+      },
+      "untyped": {
+        "": {
+          "rem": [
+            [
+              "@creditcard:filter",
+              "s",
+              9,
+              19
+            ]
+          ],
+          "len": 25
+        }
+      }
+    }
+  }
+}