diff --git a/sdk/core/azure-core-test/src/main/java/com/azure/core/test/http/TestProxyPlaybackClient.java b/sdk/core/azure-core-test/src/main/java/com/azure/core/test/http/TestProxyPlaybackClient.java index 6bcf2faed99ad..101d42490a3d4 100644 --- a/sdk/core/azure-core-test/src/main/java/com/azure/core/test/http/TestProxyPlaybackClient.java +++ b/sdk/core/azure-core-test/src/main/java/com/azure/core/test/http/TestProxyPlaybackClient.java @@ -26,7 +26,7 @@ import java.util.Queue; import java.util.stream.Collectors; -import static com.azure.core.test.utils.TestProxyUtils.getRegexSanitizerRequests; +import static com.azure.core.test.utils.TestProxyUtils.getSanitizerRequests; import static com.azure.core.test.utils.TestProxyUtils.loadSanitizers; /** @@ -96,7 +96,7 @@ public Mono send(HttpRequest request) { } private void addProxySanitization() { - getRegexSanitizerRequests(this.sanitizers) + getSanitizerRequests(this.sanitizers) .forEach(request -> { request.setHeader("x-recording-id", xRecordingId); client.sendSync(request, Context.NONE); diff --git a/sdk/core/azure-core-test/src/main/java/com/azure/core/test/http/TestProxyTestServer.java b/sdk/core/azure-core-test/src/main/java/com/azure/core/test/http/TestProxyTestServer.java index 52f52c0f3b322..dde689e03b91a 100644 --- a/sdk/core/azure-core-test/src/main/java/com/azure/core/test/http/TestProxyTestServer.java +++ b/sdk/core/azure-core-test/src/main/java/com/azure/core/test/http/TestProxyTestServer.java @@ -17,8 +17,8 @@ public class TestProxyTestServer implements Closeable { private final DisposableServer server; - private static final String TEST_RESPONSE_BODY = "{\"modelId\":\"0cd2728b-210e-4c05-b706-f70554276bcc\",\"createdDateTime\":\"2022-08-31T00:00:00Z\",\"apiVersion\":\"2022-08-31\"}"; - + private static final String TEST_JSON_RESPONSE_BODY = "{\"modelId\":\"0cd2728b-210e-4c05-b706-f70554276bcc\",\"createdDateTime\":\"2022-08-31T00:00:00Z\",\"apiVersion\":\"2022-08-31\"}"; + private static final String TEST_XML_RESPONSE_BODY = "{\"Body\":\"sensitiveInformation=\",\"primaryKey\":\"fakePrimaryKey\", \"TableName\":\"listtable09bf2a3d\"}"; /** * Constructor for TestProxyTestServer */ @@ -35,14 +35,18 @@ public TestProxyTestServer() { } return res.status(HttpResponseStatus.OK).sendString(Mono.just("echoheaders")); }) - .get("/fr/models", (req, res) -> { + .get("/fr/path/1", (req, res) -> { for (Map.Entry requestHeader : req.requestHeaders()) { res.addHeader(requestHeader.getKey(), requestHeader.getValue()); } return res.status(HttpResponseStatus.OK) .addHeader("Content-Type","application/json") - .sendString(Mono.just(TEST_RESPONSE_BODY)); - })) + .sendString(Mono.just(TEST_JSON_RESPONSE_BODY)); + }) + .get("/fr/path/2", + (req, res) -> res.status(HttpResponseStatus.OK) + .addHeader("Content-Type","application/json") + .sendString(Mono.just(TEST_XML_RESPONSE_BODY)))) .bindNow(); } diff --git a/sdk/core/azure-core-test/src/main/java/com/azure/core/test/models/TestProxySanitizer.java b/sdk/core/azure-core-test/src/main/java/com/azure/core/test/models/TestProxySanitizer.java index 7f5d459f183e3..c2e909808e586 100644 --- a/sdk/core/azure-core-test/src/main/java/com/azure/core/test/models/TestProxySanitizer.java +++ b/sdk/core/azure-core-test/src/main/java/com/azure/core/test/models/TestProxySanitizer.java @@ -11,6 +11,8 @@ public class TestProxySanitizer { private final String regexKey; private final String redactedValue; + private String groupForReplace; + /** * Creates an instance of TestProxySanitizer * @param regexKey the regex key to lookup for redaction @@ -46,4 +48,23 @@ public String getRegex() { public String getRedactedValue() { return redactedValue; } + + /** + * Get the group for replace + * @return the group for replace. + */ + public String getGroupForReplace() { + return groupForReplace; + } + + /** + * Set the group for replace. + * + * @param groupForReplace th value to set + * @return + */ + public TestProxySanitizer setGroupForReplace(String groupForReplace) { + this.groupForReplace = groupForReplace; + return this; + } } diff --git a/sdk/core/azure-core-test/src/main/java/com/azure/core/test/models/TestProxySanitizerType.java b/sdk/core/azure-core-test/src/main/java/com/azure/core/test/models/TestProxySanitizerType.java index 8bf3f91c60d6c..0b53ff642a458 100644 --- a/sdk/core/azure-core-test/src/main/java/com/azure/core/test/models/TestProxySanitizerType.java +++ b/sdk/core/azure-core-test/src/main/java/com/azure/core/test/models/TestProxySanitizerType.java @@ -15,6 +15,8 @@ public enum TestProxySanitizerType { * Sanitize the response body. */ BODY("BodyKeySanitizer"), + + BODY_REGEX("BodyRegexSanitizer"), /** * Sanitize the request/response headers. */ diff --git a/sdk/core/azure-core-test/src/main/java/com/azure/core/test/policy/TestProxyRecordPolicy.java b/sdk/core/azure-core-test/src/main/java/com/azure/core/test/policy/TestProxyRecordPolicy.java index c53a287329386..dd329879e5eeb 100644 --- a/sdk/core/azure-core-test/src/main/java/com/azure/core/test/policy/TestProxyRecordPolicy.java +++ b/sdk/core/azure-core-test/src/main/java/com/azure/core/test/policy/TestProxyRecordPolicy.java @@ -28,7 +28,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; -import static com.azure.core.test.utils.TestProxyUtils.getRegexSanitizerRequests; +import static com.azure.core.test.utils.TestProxyUtils.getSanitizerRequests; import static com.azure.core.test.utils.TestProxyUtils.loadSanitizers; @@ -107,7 +107,7 @@ public Mono process(HttpPipelineCallContext context, HttpPipelineN } private void addProxySanitization() { - getRegexSanitizerRequests(this.sanitizers) + getSanitizerRequests(this.sanitizers) .forEach(request -> { request.setHeader("x-recording-id", xRecordingId); client.sendSync(request, Context.NONE); diff --git a/sdk/core/azure-core-test/src/main/java/com/azure/core/test/utils/TestProxyUtils.java b/sdk/core/azure-core-test/src/main/java/com/azure/core/test/utils/TestProxyUtils.java index 2e007a9222d46..0e3ce09a7bead 100644 --- a/sdk/core/azure-core-test/src/main/java/com/azure/core/test/utils/TestProxyUtils.java +++ b/sdk/core/azure-core-test/src/main/java/com/azure/core/test/utils/TestProxyUtils.java @@ -31,12 +31,20 @@ public class TestProxyUtils { Arrays.asList("authHeader", "accountKey", "accessToken", "accountName", "applicationId", "apiKey", "connectionString", "url", "host", "password", "userName")); + private static final List BODY_REGEX_TO_REDACT + = new ArrayList<>(Arrays.asList("(?:)(?.*)(?:)", "(?:Password=)(?.*)(?:;)", + "(?:User ID=)(?.*)(?:;)", "(?:)(?.*)(?:)", + "(?:)(?.*)(?:)")); + private static final String URL_REGEX = "^(?:https?:\\\\/\\\\/)?(?:[^@\\\\/\\\\n]+@)?(?:www\\\\.)?([^:\\\\/?\\\\n]+)"; private static final String URL_REDACTION_VALUE = "https://REDACTED"; private static final List HEADERS_TO_REDACT = new ArrayList<>(Arrays.asList("Ocp-Apim-Subscription-Key")); private static final String REDACTED_VALUE = "REDACTED"; + private static final String DELEGATION_KEY_CLIENTID_REGEX = "(?:)(?.*)(?:)"; + private static final String DELEGATION_KEY_TENANTID_REGEX = "(?:)(?.*)(?:)"; + /** * Get the proxy URL. * @@ -80,9 +88,10 @@ public static void changeHeaders(HttpRequest request, String xRecordingId, Strin public static List loadSanitizers() { List sanitizers = new ArrayList<>(); - sanitizers.add(getDefaultUrlSanitizer()); - sanitizers.addAll(getDefaultBodySanitizers()); - sanitizers.addAll(getDefaultHeaderSanitizers()); + sanitizers.addAll(addDefaultRegexSanitizers()); + sanitizers.add(addDefaultUrlSanitizer()); + sanitizers.addAll(addDefaultBodySanitizers()); + sanitizers.addAll(addDefaultHeaderSanitizers()); return sanitizers; } @@ -90,15 +99,20 @@ public static String createUrlRegexRequestBody(String regexValue, String redacte return String.format("{\"value\":\"%s\",\"regex\":\"%s\"}", redactedValue, regexValue); } - public static String createBodyRegexRequestBody(String regexValue, String redactedValue) { + public static String createBodyJsonKeyRequestBody(String regexValue, String redactedValue) { return String.format("{\"value\":\"%s\",\"jsonPath\":\"%s\"}", redactedValue, regexValue); } + + public static String createBodyRegexRequestBody(String regexValue, String redactedValue, String groupForReplace) { + return String.format("{\"value\":\"%s\",\"regex\":\"%s\",\"groupForReplace\":\"%s\"}", redactedValue, regexValue, groupForReplace); + } + public static String createHeaderRegexRequestBody(String regexValue, String redactedValue) { return String.format("{\"value\":\"%s\",\"key\":\"%s\"}", redactedValue, regexValue); } - public static List getRegexSanitizerRequests(List sanitizers) { + public static List getSanitizerRequests(List sanitizers) { return sanitizers.stream().map(testProxySanitizer -> { String requestBody; String sanitizerType; @@ -108,8 +122,13 @@ public static List getRegexSanitizerRequests(List getRegexSanitizerRequests(List getRegexSanitizerRequests(List getDefaultBodySanitizers() { + private static List addDefaultBodySanitizers() { return JSON_PROPERTIES_TO_REDACT.stream() - .map(jsonProperty -> new TestProxySanitizer(String.format("$..%s", jsonProperty), REDACTED_VALUE, - TestProxySanitizerType.BODY)) + .map(jsonProperty -> + new TestProxySanitizer(String.format("$..%s", jsonProperty), REDACTED_VALUE, + TestProxySanitizerType.BODY)) .collect(Collectors.toList()); } - private static List getDefaultHeaderSanitizers() { + private static List addDefaultRegexSanitizers() { + List userDelegationSanitizers = getUserDelegationSanitizers(); + + userDelegationSanitizers.addAll(BODY_REGEX_TO_REDACT.stream() + .map(bodyRegex -> new TestProxySanitizer(bodyRegex, REDACTED_VALUE, TestProxySanitizerType.BODY_REGEX).setGroupForReplace("secret")) + .collect(Collectors.toList())); + + // can add default url and header regex sanitizer same way + return userDelegationSanitizers; + + } + + private static List addDefaultHeaderSanitizers() { return HEADERS_TO_REDACT.stream() - .map( - headerProperty -> new TestProxySanitizer(headerProperty, REDACTED_VALUE, TestProxySanitizerType.HEADER)) + .map(headerProperty -> + new TestProxySanitizer(headerProperty, REDACTED_VALUE, TestProxySanitizerType.HEADER)) .collect(Collectors.toList()); } + + private static List getUserDelegationSanitizers() { + List userDelegationSanitizers = new ArrayList<>(); + userDelegationSanitizers.add(new TestProxySanitizer(DELEGATION_KEY_CLIENTID_REGEX, REDACTED_VALUE, TestProxySanitizerType.BODY_REGEX).setGroupForReplace("secret")); + userDelegationSanitizers.add(new TestProxySanitizer(DELEGATION_KEY_TENANTID_REGEX, REDACTED_VALUE, TestProxySanitizerType.BODY_REGEX).setGroupForReplace("secret")); + return userDelegationSanitizers; + } } diff --git a/sdk/core/azure-core-test/src/test/java/com/azure/core/test/TestProxyTests.java b/sdk/core/azure-core-test/src/test/java/com/azure/core/test/TestProxyTests.java index 98062561c7674..23089da389f8c 100644 --- a/sdk/core/azure-core-test/src/test/java/com/azure/core/test/TestProxyTests.java +++ b/sdk/core/azure-core-test/src/test/java/com/azure/core/test/TestProxyTests.java @@ -37,6 +37,7 @@ import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; @SuppressWarnings("deprecation") public class TestProxyTests extends TestBase { @@ -49,6 +50,7 @@ public class TestProxyTests extends TestBase { static { customSanitizer.add(new TestProxySanitizer("$..modelId", REDACTED, TestProxySanitizerType.BODY)); + customSanitizer.add(new TestProxySanitizer("TableName\\\"*:*\\\"(?.*)\\\"", REDACTED, TestProxySanitizerType.BODY_REGEX).setGroupForReplace("tablename")); } @BeforeAll @@ -57,14 +59,11 @@ public static void setupClass() { server = new TestProxyTestServer(); TestBase.setupClass(); } - @AfterAll public static void teardownClass() { TestBase.teardownClass(); server.close(); } - - @Test @Tag("Record") public void testBasicRecord() { @@ -163,7 +162,7 @@ public void testRecordWithRedaction() { try { url = new UrlBuilder() .setHost("localhost") - .setPath("/fr/models") + .setPath("/fr/path/1") .setPort(3000) .setScheme("http") .toUrl(); @@ -183,7 +182,7 @@ public void testRecordWithRedaction() { RecordedTestProxyData recordedTestProxyData = readDataFromFile(); RecordedTestProxyData.TestProxyDataRecord record = recordedTestProxyData.getTestProxyDataRecords().get(0); // default sanitizers - assertEquals("https://REDACTED:3000/fr/models", record.getUri()); + assertEquals("https://REDACTED:3000/fr/path/1", record.getUri()); assertEquals(REDACTED, record.getHeaders().get("Ocp-Apim-Subscription-Key")); // custom sanitizers assertEquals(REDACTED, record.getResponse().get("modelId")); @@ -216,6 +215,50 @@ public void testPlaybackWithRedaction() { assertEquals(200, response.getStatusCode()); } + @Test + @Tag("Record") + public void testBodyRegexRedactRecord() { + HttpURLConnectionHttpClient client = new HttpURLConnectionHttpClient(); + + interceptorManager.addRecordSanitizers(customSanitizer); + + HttpPipeline pipeline = new HttpPipelineBuilder() + .httpClient(client) + .policies(interceptorManager.getRecordPolicy()).build(); + URL url; + try { + url = new UrlBuilder() + .setHost("localhost") + .setPath("/fr/path/2") + .setPort(3000) + .setScheme("http") + .toUrl(); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + + HttpRequest request = new HttpRequest(HttpMethod.GET, url); + request.setHeader("Content-Type", "application/json"); + + HttpResponse response = pipeline.sendSync(request, Context.NONE); + + assertEquals(response.getStatusCode(), 200); + + assertEquals(200, response.getStatusCode()); + + RecordedTestProxyData recordedTestProxyData = readDataFromFile(); + RecordedTestProxyData.TestProxyDataRecord record = recordedTestProxyData.getTestProxyDataRecords().get(0); + // default regex sanitizers + assertEquals("https://REDACTED:3000/fr/path/2", record.getUri()); + + // user delegation sanitizers + assertTrue(record.getResponse().get("Body").contains("REDACTED")); + assertTrue(record.getResponse().get("primaryKey").contains("REDACTED")); + + // custom body regex + assertTrue(record.getResponse().get("TableName").equals(REDACTED)); + } + private RecordedTestProxyData readDataFromFile() { String filePath = InterceptorManager.getRecordFolder() + "\\" + this.testContextManager.getTestPlaybackRecordingName() + ".json"; File recordFile = new File(filePath); diff --git a/sdk/core/azure-core-test/src/test/resources/session-records/TestProxyTests.testBodyRegexRedactRecord.json b/sdk/core/azure-core-test/src/test/resources/session-records/TestProxyTests.testBodyRegexRedactRecord.json new file mode 100644 index 0000000000000..f30857ebf83ea --- /dev/null +++ b/sdk/core/azure-core-test/src/test/resources/session-records/TestProxyTests.testBodyRegexRedactRecord.json @@ -0,0 +1,25 @@ +{ + "Entries": [ + { + "RequestUri": "https://REDACTED:3000/fr/path/2", + "RequestMethod": "GET", + "RequestHeaders": { + "Accept": "text/html, image/gif, image/jpeg, *; q=.2, */*; q=.2", + "Connection": "keep-alive", + "User-Agent": "Java/11.0.9" + }, + "RequestBody": null, + "StatusCode": 200, + "ResponseHeaders": { + "Content-Length": "154", + "Content-Type": "application/json" + }, + "ResponseBody": { + "Body": "\u003CUserDelegationKey\u003E\u003CSignedTid\u003EREDACTED\u003C/SignedTid\u003E\u003C/UserDelegationKey\u003E", + "primaryKey": "\u003CPrimaryKey\u003EREDACTED\u003C/PrimaryKey\u003E", + "TableName": "REDACTED" + } + } + ], + "Variables": {} +} diff --git a/sdk/core/azure-core-test/src/test/resources/session-records/TestProxyTests.testRecordWithRedaction.json b/sdk/core/azure-core-test/src/test/resources/session-records/TestProxyTests.testRecordWithRedaction.json index 13304b2d5a538..c0e57749b072e 100644 --- a/sdk/core/azure-core-test/src/test/resources/session-records/TestProxyTests.testRecordWithRedaction.json +++ b/sdk/core/azure-core-test/src/test/resources/session-records/TestProxyTests.testRecordWithRedaction.json @@ -1,7 +1,7 @@ { "Entries": [ { - "RequestUri": "https://REDACTED:3000/fr/models", + "RequestUri": "https://REDACTED:3000/fr/path/1", "RequestMethod": "GET", "RequestHeaders": { "Accept": "text/html, image/gif, image/jpeg, *; q=.2, */*; q=.2", @@ -19,7 +19,7 @@ "Content-Type": "application/json", "Host": "localhost:3000", "Ocp-Apim-Subscription-Key": "REDACTED", - "traceparent": "00-991ec280d6eb610706ed90b754f450d5-f327e47ae4f71dc1-00", + "traceparent": "00-18e2d72d3db49d1a4bae9e6a4c82dc96-8f32ab073c009f2a-00", "User-Agent": "Java/11.0.9" }, "ResponseBody": { @@ -29,8 +29,5 @@ } } ], - "Variables": { - "0": "test21822", - "1": "2023-01-05T21:43:38.703776200Z" - } + "Variables": {} }