Merge pull request #3 from samvaity/redactor-default

Add default recording redactors to proxy sanitization

sdk/core/azure-core-test/src/main/java/com/azure/core/test/http/TestProxyPlaybackClient.java

@@ -26,7 +26,7 @@
 import java.util.Queue;
 import java.util.stream.Collectors;
 
-import static com.azure.core.test.utils.TestProxyUtils.getRegexSanitizerRequests;
+import static com.azure.core.test.utils.TestProxyUtils.getSanitizerRequests;
 import static com.azure.core.test.utils.TestProxyUtils.loadSanitizers;
 
 /**
@@ -96,7 +96,7 @@ public Mono<HttpResponse> send(HttpRequest request) {
     }
 
     private void addProxySanitization() {
-        getRegexSanitizerRequests(this.sanitizers)
+        getSanitizerRequests(this.sanitizers)
             .forEach(request -> {
                 request.setHeader("x-recording-id", xRecordingId);
                 client.sendSync(request, Context.NONE);

sdk/core/azure-core-test/src/main/java/com/azure/core/test/http/TestProxyTestServer.java

@@ -17,8 +17,8 @@
 public class TestProxyTestServer implements Closeable {
     private final DisposableServer server;
 
-    private static final String TEST_RESPONSE_BODY = "{\"modelId\":\"0cd2728b-210e-4c05-b706-f70554276bcc\",\"createdDateTime\":\"2022-08-31T00:00:00Z\",\"apiVersion\":\"2022-08-31\"}";
-
+    private static final String TEST_JSON_RESPONSE_BODY = "{\"modelId\":\"0cd2728b-210e-4c05-b706-f70554276bcc\",\"createdDateTime\":\"2022-08-31T00:00:00Z\",\"apiVersion\":\"2022-08-31\"}";
+    private static final String TEST_XML_RESPONSE_BODY = "{\"Body\":\"<UserDelegationKey><SignedTid>sensitiveInformation=</SignedTid></UserDelegationKey>\",\"primaryKey\":\"<PrimaryKey>fakePrimaryKey</PrimaryKey>\", \"TableName\":\"listtable09bf2a3d\"}";
     /**
      * Constructor for TestProxyTestServer
      */
@@ -35,14 +35,18 @@ public TestProxyTestServer() {
                     }
                     return res.status(HttpResponseStatus.OK).sendString(Mono.just("echoheaders"));
                 })
-                .get("/fr/models", (req, res) -> {
+                .get("/fr/path/1", (req, res) -> {
                     for (Map.Entry<String, String> requestHeader : req.requestHeaders()) {
                         res.addHeader(requestHeader.getKey(), requestHeader.getValue());
                     }
                     return res.status(HttpResponseStatus.OK)
                         .addHeader("Content-Type","application/json")
-                        .sendString(Mono.just(TEST_RESPONSE_BODY));
-                }))
+                        .sendString(Mono.just(TEST_JSON_RESPONSE_BODY));
+                })
+                .get("/fr/path/2",
+                    (req, res) -> res.status(HttpResponseStatus.OK)
+                        .addHeader("Content-Type","application/json")
+                        .sendString(Mono.just(TEST_XML_RESPONSE_BODY))))
             .bindNow();
     }
 

sdk/core/azure-core-test/src/main/java/com/azure/core/test/models/TestProxySanitizer.java

@@ -11,6 +11,8 @@ public class TestProxySanitizer {
     private final String regexKey;
     private final String redactedValue;
 
+    private String groupForReplace;
+
     /**
      * Creates an instance of TestProxySanitizer
      * @param regexKey the regex key to lookup for redaction
@@ -46,4 +48,23 @@ public String getRegex() {
     public String getRedactedValue() {
         return redactedValue;
     }
+
+    /**
+     * Get the group for replace
+     * @return the group for replace.
+     */
+    public String getGroupForReplace() {
+        return groupForReplace;
+    }
+
+    /**
+     * Set the group for replace.
+     *
+     * @param groupForReplace th value to set
+     * @return
+     */
+    public TestProxySanitizer setGroupForReplace(String groupForReplace) {
+        this.groupForReplace = groupForReplace;
+        return this;
+    }
 }

sdk/core/azure-core-test/src/main/java/com/azure/core/test/models/TestProxySanitizerType.java

@@ -15,6 +15,8 @@ public enum TestProxySanitizerType {
      * Sanitize the response body.
      */
     BODY("BodyKeySanitizer"),
+
+    BODY_REGEX("BodyRegexSanitizer"),
     /**
      * Sanitize the request/response headers.
      */

sdk/core/azure-core-test/src/main/java/com/azure/core/test/policy/TestProxyRecordPolicy.java

@@ -28,7 +28,7 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 
-import static com.azure.core.test.utils.TestProxyUtils.getRegexSanitizerRequests;
+import static com.azure.core.test.utils.TestProxyUtils.getSanitizerRequests;
 import static com.azure.core.test.utils.TestProxyUtils.loadSanitizers;
 
 
@@ -107,7 +107,7 @@ public Mono<HttpResponse> process(HttpPipelineCallContext context, HttpPipelineN
     }
 
     private void addProxySanitization() {
-        getRegexSanitizerRequests(this.sanitizers)
+        getSanitizerRequests(this.sanitizers)
             .forEach(request -> {
                 request.setHeader("x-recording-id", xRecordingId);
                 client.sendSync(request, Context.NONE);

sdk/core/azure-core-test/src/main/java/com/azure/core/test/utils/TestProxyUtils.java

@@ -31,12 +31,20 @@ public class TestProxyUtils {
         Arrays.asList("authHeader", "accountKey", "accessToken", "accountName", "applicationId", "apiKey",
             "connectionString", "url", "host", "password", "userName"));
 
+    private static final List<String> BODY_REGEX_TO_REDACT
+        = new ArrayList<>(Arrays.asList("(?:<Value>)(?<secret>.*)(?:</Value>)", "(?:Password=)(?<secret>.*)(?:;)",
+        "(?:User ID=)(?<secret>.*)(?:;)", "(?:<PrimaryKey>)(?<secret>.*)(?:</PrimaryKey>)",
+        "(?:<SecondaryKey>)(?<secret>.*)(?:</SecondaryKey>)"));
+
     private static final String URL_REGEX =
         "^(?:https?:\\\\/\\\\/)?(?:[^@\\\\/\\\\n]+@)?(?:www\\\\.)?([^:\\\\/?\\\\n]+)";
     private static final String URL_REDACTION_VALUE = "https://REDACTED";
     private static final List<String> HEADERS_TO_REDACT = new ArrayList<>(Arrays.asList("Ocp-Apim-Subscription-Key"));
     private static final String REDACTED_VALUE = "REDACTED";
 
+    private static final String DELEGATION_KEY_CLIENTID_REGEX = "(?:<SignedOid>)(?<secret>.*)(?:</SignedOid>)";
+    private static final String DELEGATION_KEY_TENANTID_REGEX = "(?:<SignedTid>)(?<secret>.*)(?:</SignedTid>)";
+
     /**
      * Get the proxy URL.
      *
@@ -80,25 +88,31 @@ public static void changeHeaders(HttpRequest request, String xRecordingId, Strin
 
     public static List<TestProxySanitizer> loadSanitizers() {
         List<TestProxySanitizer> sanitizers = new ArrayList<>();
-        sanitizers.add(getDefaultUrlSanitizer());
-        sanitizers.addAll(getDefaultBodySanitizers());
-        sanitizers.addAll(getDefaultHeaderSanitizers());
+        sanitizers.addAll(addDefaultRegexSanitizers());
+        sanitizers.add(addDefaultUrlSanitizer());
+        sanitizers.addAll(addDefaultBodySanitizers());
+        sanitizers.addAll(addDefaultHeaderSanitizers());
         return sanitizers;
     }
 
     public static String createUrlRegexRequestBody(String regexValue, String redactedValue) {
         return String.format("{\"value\":\"%s\",\"regex\":\"%s\"}", redactedValue, regexValue);
     }
 
-    public static String createBodyRegexRequestBody(String regexValue, String redactedValue) {
+    public static String createBodyJsonKeyRequestBody(String regexValue, String redactedValue) {
         return String.format("{\"value\":\"%s\",\"jsonPath\":\"%s\"}", redactedValue, regexValue);
     }
 
+
+    public static String createBodyRegexRequestBody(String regexValue, String redactedValue, String groupForReplace) {
+        return String.format("{\"value\":\"%s\",\"regex\":\"%s\",\"groupForReplace\":\"%s\"}", redactedValue, regexValue, groupForReplace);
+    }
+
     public static String createHeaderRegexRequestBody(String regexValue, String redactedValue) {
         return String.format("{\"value\":\"%s\",\"key\":\"%s\"}", redactedValue, regexValue);
     }
 
-    public static List<HttpRequest> getRegexSanitizerRequests(List<TestProxySanitizer> sanitizers) {
+    public static List<HttpRequest> getSanitizerRequests(List<TestProxySanitizer> sanitizers) {
         return sanitizers.stream().map(testProxySanitizer -> {
             String requestBody;
             String sanitizerType;
@@ -108,8 +122,13 @@ public static List<HttpRequest> getRegexSanitizerRequests(List<TestProxySanitize
                         createUrlRegexRequestBody(testProxySanitizer.getRegex(), testProxySanitizer.getRedactedValue());
                     sanitizerType = TestProxySanitizerType.URL.name;
                     break;
-                case BODY:
+                case BODY_REGEX:
                     requestBody = createBodyRegexRequestBody(testProxySanitizer.getRegex(),
+                        testProxySanitizer.getRedactedValue(), testProxySanitizer.getGroupForReplace());
+                    sanitizerType = TestProxySanitizerType.BODY_REGEX.name;
+                    break;
+                case BODY:
+                    requestBody = createBodyJsonKeyRequestBody(testProxySanitizer.getRegex(),
                         testProxySanitizer.getRedactedValue());
                     sanitizerType = TestProxySanitizerType.BODY.name;
                     break;
@@ -119,7 +138,7 @@ public static List<HttpRequest> getRegexSanitizerRequests(List<TestProxySanitize
                     sanitizerType = TestProxySanitizerType.HEADER.name;
                     break;
                 default:
-                    throw new RuntimeException("Sanitizer type not supported");
+                    throw new RuntimeException(String.format("Sanitizer type {%s} not supported", testProxySanitizer.getType()));
             }
             HttpRequest request
                 = new HttpRequest(HttpMethod.POST, String.format("%s/Admin/AddSanitizer", TestProxyUtils.getProxyUrl()))
@@ -129,21 +148,41 @@ public static List<HttpRequest> getRegexSanitizerRequests(List<TestProxySanitize
         }).collect(Collectors.toList());
     }
 
-    private static TestProxySanitizer getDefaultUrlSanitizer() {
+    private static TestProxySanitizer addDefaultUrlSanitizer() {
         return new TestProxySanitizer(URL_REGEX, URL_REDACTION_VALUE, TestProxySanitizerType.URL);
     }
 
-    private static List<TestProxySanitizer> getDefaultBodySanitizers() {
+    private static List<TestProxySanitizer> addDefaultBodySanitizers() {
         return JSON_PROPERTIES_TO_REDACT.stream()
-            .map(jsonProperty -> new TestProxySanitizer(String.format("$..%s", jsonProperty), REDACTED_VALUE,
-                TestProxySanitizerType.BODY))
+            .map(jsonProperty ->
+                new TestProxySanitizer(String.format("$..%s", jsonProperty), REDACTED_VALUE,
+                    TestProxySanitizerType.BODY))
             .collect(Collectors.toList());
     }
 
-    private static List<TestProxySanitizer> getDefaultHeaderSanitizers() {
+    private static List<TestProxySanitizer> addDefaultRegexSanitizers() {
+        List<TestProxySanitizer> userDelegationSanitizers = getUserDelegationSanitizers();
+
+        userDelegationSanitizers.addAll(BODY_REGEX_TO_REDACT.stream()
+            .map(bodyRegex -> new TestProxySanitizer(bodyRegex, REDACTED_VALUE, TestProxySanitizerType.BODY_REGEX).setGroupForReplace("secret"))
+            .collect(Collectors.toList()));
+
+        // can add default url and header regex sanitizer same way
+        return userDelegationSanitizers;
+
+    }
+
+    private static List<TestProxySanitizer> addDefaultHeaderSanitizers() {
         return HEADERS_TO_REDACT.stream()
-            .map(
-                headerProperty -> new TestProxySanitizer(headerProperty, REDACTED_VALUE, TestProxySanitizerType.HEADER))
+            .map(headerProperty ->
+                new TestProxySanitizer(headerProperty, REDACTED_VALUE, TestProxySanitizerType.HEADER))
             .collect(Collectors.toList());
     }
+
+    private static List<TestProxySanitizer> getUserDelegationSanitizers() {
+        List<TestProxySanitizer> userDelegationSanitizers = new ArrayList<>();
+        userDelegationSanitizers.add(new TestProxySanitizer(DELEGATION_KEY_CLIENTID_REGEX, REDACTED_VALUE, TestProxySanitizerType.BODY_REGEX).setGroupForReplace("secret"));
+        userDelegationSanitizers.add(new TestProxySanitizer(DELEGATION_KEY_TENANTID_REGEX, REDACTED_VALUE, TestProxySanitizerType.BODY_REGEX).setGroupForReplace("secret"));
+        return userDelegationSanitizers;
+    }
 }

sdk/core/azure-core-test/src/test/java/com/azure/core/test/TestProxyTests.java

@@ -37,6 +37,7 @@
 import java.util.Map;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 @SuppressWarnings("deprecation")
 public class TestProxyTests extends TestBase {
@@ -49,6 +50,7 @@ public class TestProxyTests extends TestBase {
 
     static {
         customSanitizer.add(new TestProxySanitizer("$..modelId", REDACTED, TestProxySanitizerType.BODY));
+        customSanitizer.add(new TestProxySanitizer("TableName\\\"*:*\\\"(?<tablename>.*)\\\"", REDACTED, TestProxySanitizerType.BODY_REGEX).setGroupForReplace("tablename"));
     }
 
     @BeforeAll
@@ -57,14 +59,11 @@ public static void setupClass() {
         server = new TestProxyTestServer();
         TestBase.setupClass();
     }
-
     @AfterAll
     public static void teardownClass() {
         TestBase.teardownClass();
         server.close();
     }
-
-
     @Test
     @Tag("Record")
     public void testBasicRecord() {
@@ -163,7 +162,7 @@ public void testRecordWithRedaction() {
         try {
             url = new UrlBuilder()
                 .setHost("localhost")
-                .setPath("/fr/models")
+                .setPath("/fr/path/1")
                 .setPort(3000)
                 .setScheme("http")
                 .toUrl();
@@ -183,7 +182,7 @@ public void testRecordWithRedaction() {
         RecordedTestProxyData recordedTestProxyData = readDataFromFile();
         RecordedTestProxyData.TestProxyDataRecord record = recordedTestProxyData.getTestProxyDataRecords().get(0);
         // default sanitizers
-        assertEquals("https://REDACTED:3000/fr/models", record.getUri());
+        assertEquals("https://REDACTED:3000/fr/path/1", record.getUri());
         assertEquals(REDACTED, record.getHeaders().get("Ocp-Apim-Subscription-Key"));
         // custom sanitizers
         assertEquals(REDACTED, record.getResponse().get("modelId"));
@@ -216,6 +215,50 @@ public void testPlaybackWithRedaction() {
         assertEquals(200, response.getStatusCode());
     }
 
+    @Test
+    @Tag("Record")
+    public void testBodyRegexRedactRecord() {
+        HttpURLConnectionHttpClient client = new HttpURLConnectionHttpClient();
+
+        interceptorManager.addRecordSanitizers(customSanitizer);
+
+        HttpPipeline pipeline = new HttpPipelineBuilder()
+            .httpClient(client)
+            .policies(interceptorManager.getRecordPolicy()).build();
+        URL url;
+        try {
+            url = new UrlBuilder()
+                .setHost("localhost")
+                .setPath("/fr/path/2")
+                .setPort(3000)
+                .setScheme("http")
+                .toUrl();
+        } catch (MalformedURLException e) {
+            throw new RuntimeException(e);
+        }
+
+        HttpRequest request = new HttpRequest(HttpMethod.GET, url);
+        request.setHeader("Content-Type", "application/json");
+
+        HttpResponse response = pipeline.sendSync(request, Context.NONE);
+
+        assertEquals(response.getStatusCode(), 200);
+
+        assertEquals(200, response.getStatusCode());
+
+        RecordedTestProxyData recordedTestProxyData = readDataFromFile();
+        RecordedTestProxyData.TestProxyDataRecord record = recordedTestProxyData.getTestProxyDataRecords().get(0);
+        // default regex sanitizers
+        assertEquals("https://REDACTED:3000/fr/path/2", record.getUri());
+
+        // user delegation sanitizers
+        assertTrue(record.getResponse().get("Body").contains("<UserDelegationKey><SignedTid>REDACTED</SignedTid></UserDelegationKey>"));
+        assertTrue(record.getResponse().get("primaryKey").contains("<PrimaryKey>REDACTED</PrimaryKey>"));
+
+        // custom body regex
+        assertTrue(record.getResponse().get("TableName").equals(REDACTED));
+    }
+
     private RecordedTestProxyData readDataFromFile() {
         String filePath = InterceptorManager.getRecordFolder() + "\\" + this.testContextManager.getTestPlaybackRecordingName() + ".json";
         File recordFile = new File(filePath);

sdk/core/azure-core-test/src/test/resources/session-records/TestProxyTests.testBodyRegexRedactRecord.json

@@ -0,0 +1,25 @@
+{
+  "Entries": [
+    {
+      "RequestUri": "https://REDACTED:3000/fr/path/2",
+      "RequestMethod": "GET",
+      "RequestHeaders": {
+        "Accept": "text/html, image/gif, image/jpeg, *; q=.2, */*; q=.2",
+        "Connection": "keep-alive",
+        "User-Agent": "Java/11.0.9"
+      },
+      "RequestBody": null,
+      "StatusCode": 200,
+      "ResponseHeaders": {
+        "Content-Length": "154",
+        "Content-Type": "application/json"
+      },
+      "ResponseBody": {
+        "Body": "\u003CUserDelegationKey\u003E\u003CSignedTid\u003EREDACTED\u003C/SignedTid\u003E\u003C/UserDelegationKey\u003E",
+        "primaryKey": "\u003CPrimaryKey\u003EREDACTED\u003C/PrimaryKey\u003E",
+        "TableName": "REDACTED"
+      }
+    }
+  ],
+  "Variables": {}
+}

sdk/core/azure-core-test/src/test/resources/session-records/TestProxyTests.testRecordWithRedaction.json

@@ -1,7 +1,7 @@
 {
   "Entries": [
     {
-      "RequestUri": "https://REDACTED:3000/fr/models",
+      "RequestUri": "https://REDACTED:3000/fr/path/1",
       "RequestMethod": "GET",
       "RequestHeaders": {
         "Accept": "text/html, image/gif, image/jpeg, *; q=.2, */*; q=.2",
@@ -19,7 +19,7 @@
         "Content-Type": "application/json",
         "Host": "localhost:3000",
         "Ocp-Apim-Subscription-Key": "REDACTED",
-        "traceparent": "00-991ec280d6eb610706ed90b754f450d5-f327e47ae4f71dc1-00",
+        "traceparent": "00-18e2d72d3db49d1a4bae9e6a4c82dc96-8f32ab073c009f2a-00",
         "User-Agent": "Java/11.0.9"
       },
       "ResponseBody": {
@@ -29,8 +29,5 @@
       }
     }
   ],
-  "Variables": {
-    "0": "test21822",
-    "1": "2023-01-05T21:43:38.703776200Z"
-  }
+  "Variables": {}
 }