From fb6062a02ee9807feb9b1f716013b4c7cfebf7a8 Mon Sep 17 00:00:00 2001 From: Scott Beddall <45376673+scbedd@users.noreply.github.com> Date: Tue, 12 Sep 2023 13:25:31 -0700 Subject: [PATCH] Support Bulk Sanitizer Add (#6926) * add code and tests for bulk sanitizers * add docs reflecting new bulk sanitizer add --- .../AdminTests.cs | 119 ++++++++++++++++++ .../Azure.Sdk.Tools.TestProxy/Admin.cs | 33 ++++- .../Common/HttpRequestInteractions.cs | 27 ++++ .../Common/SanitizerList.cs | 10 ++ .../Azure.Sdk.Tools.TestProxy/README.md | 32 ++++- 5 files changed, 219 insertions(+), 2 deletions(-) create mode 100644 tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/SanitizerList.cs diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/AdminTests.cs b/tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/AdminTests.cs index f52d8d72f52..843c590daef 100644 --- a/tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/AdminTests.cs +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/AdminTests.cs @@ -3,6 +3,7 @@ using Azure.Sdk.Tools.TestProxy.Matchers; using Azure.Sdk.Tools.TestProxy.Sanitizers; using Azure.Sdk.Tools.TestProxy.Transforms; +using Microsoft.AspNetCore.DataProtection.KeyManagement; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging.Abstractions; @@ -11,6 +12,7 @@ using System.Linq; using System.Net; using System.Reflection; +using System.Text.RegularExpressions; using System.Threading.Tasks; using Xunit; @@ -29,6 +31,123 @@ public class AdminTests { private NullLoggerFactory _nullLogger = new NullLoggerFactory(); + [Fact] + public async void TestAddSanitizersThrowsOnEmptyArray() + { + RecordingHandler testRecordingHandler = new RecordingHandler(Directory.GetCurrentDirectory()); + var httpContext = new DefaultHttpContext(); + + string requestBody = @"[]"; + + httpContext.Request.Body = TestHelpers.GenerateStreamRequestBody(requestBody); + httpContext.Request.ContentLength = httpContext.Request.Body.Length; + testRecordingHandler.Sanitizers.Clear(); + + var controller = new Admin(testRecordingHandler, _nullLogger) + { + ControllerContext = new ControllerContext() + { + HttpContext = httpContext + } + }; + var assertion = await Assert.ThrowsAsync( + async () => await controller.AddSanitizers() + ); + + assertion.StatusCode.Equals(HttpStatusCode.BadRequest); + } + + [Fact] + + public async void TestAddSanitizersHandlesPopulatedArray() + { + RecordingHandler testRecordingHandler = new RecordingHandler(Directory.GetCurrentDirectory()); + var httpContext = new DefaultHttpContext(); + + string requestBody = @"[ + { + ""Name"": ""GeneralRegexSanitizer"", + ""Body"": { + ""regex"": ""[a-zA-Z]?"", + ""value"": ""hello_there"", + ""condition"": { + ""UriRegex"": "".+/Tables"" + } + } + }, + { + ""Name"": ""HeaderRegexSanitizer"", + ""Body"": { + ""key"": ""Location"", + ""value"": ""https://fakeazsdktestaccount.table.core.windows.net/Tables"" + } + } +]"; + + httpContext.Request.Body = TestHelpers.GenerateStreamRequestBody(requestBody); + httpContext.Request.ContentLength = httpContext.Request.Body.Length; + testRecordingHandler.Sanitizers.Clear(); + + var controller = new Admin(testRecordingHandler, _nullLogger) + { + ControllerContext = new ControllerContext() + { + HttpContext = httpContext + } + }; + await controller.AddSanitizers(); + + + Assert.Equal(2, testRecordingHandler.Sanitizers.Count); + + Assert.True(testRecordingHandler.Sanitizers[0] is GeneralRegexSanitizer); + Assert.True(testRecordingHandler.Sanitizers[1] is HeaderRegexSanitizer); + } + + [Fact] + public async void TestAddSanitizersThrowsOnSingleBadInput() + { + RecordingHandler testRecordingHandler = new RecordingHandler(Directory.GetCurrentDirectory()); + var httpContext = new DefaultHttpContext(); + + string requestBody = @"[ + { + ""Name"": ""GeneralRegexSanitizer"", + ""Body"": { + ""regex"": ""[a-zA-Z]?"", + ""value"": ""hello_there"", + ""condition"": { + ""UriRegex"": "".+/Tables"" + } + } + }, + { + ""Name"": ""BadRegexIdentifier"", + ""Body"": { + ""key"": ""Location"", + ""value"": ""https://fakeazsdktestaccount.table.core.windows.net/Tables"" + } + } +]"; + + httpContext.Request.Body = TestHelpers.GenerateStreamRequestBody(requestBody); + httpContext.Request.ContentLength = httpContext.Request.Body.Length; + testRecordingHandler.Sanitizers.Clear(); + + var controller = new Admin(testRecordingHandler, _nullLogger) + { + ControllerContext = new ControllerContext() + { + HttpContext = httpContext + } + }; + var assertion = await Assert.ThrowsAsync( + async () => await controller.AddSanitizers() + ); + + assertion.StatusCode.Equals(HttpStatusCode.BadRequest); + } + [Fact] public async void TestAddSanitizerThrowsOnInvalidAbstractionId() { diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Admin.cs b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Admin.cs index d81b7ce8588..f8d2f1ec7d3 100644 --- a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Admin.cs +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Admin.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. using Azure.Sdk.Tools.TestProxy.Common; @@ -8,6 +8,7 @@ using Microsoft.Extensions.Logging; using System; using System.Collections.Generic; +using System.Linq; using System.Net; using System.Text.Json; using System.Threading.Tasks; @@ -81,6 +82,36 @@ public async Task AddSanitizer() } } + [HttpPost] + public async Task AddSanitizers() + { + DebugLogger.LogAdminRequestDetails(_logger, Request); + var recordingId = RecordingHandler.GetHeader(Request, "x-recording-id", allowNulls: true); + + // parse all of them first, any exceptions should pop here + var workload = (await HttpRequestInteractions.GetBody>(Request)).Select(s => (RecordedTestSanitizer)GetSanitizer(s.Name, s.Body)).ToList(); + + if (workload.Count == 0) + { + throw new HttpException(HttpStatusCode.BadRequest, "When bulk adding sanitizers, ensure there is at least one sanitizer added in each batch. Received 0 work items."); + } + + // register them all + foreach(var sanitizer in workload) + { + if (recordingId != null) + { + _recordingHandler.AddSanitizerToRecording(recordingId, sanitizer); + } + else + { + _recordingHandler.Sanitizers.Add(sanitizer); + } + } + + } + + [HttpPost] public async Task SetMatcher() { diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/HttpRequestInteractions.cs b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/HttpRequestInteractions.cs index 7de8a0a04e1..60bd3ae8164 100644 --- a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/HttpRequestInteractions.cs +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/HttpRequestInteractions.cs @@ -4,6 +4,7 @@ using Azure.Core; using Azure.Sdk.Tools.TestProxy.Common.Exceptions; using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; using System; using System.IO; @@ -61,6 +62,32 @@ public static string GetBodyKey(JsonDocument document, string key, bool allowNul return value; } + public async static Task GetBody(HttpRequest req) + { + if (req.ContentLength > 0) + { + try + { + using (var jsonDocument = await JsonDocument.ParseAsync(req.Body, options: new JsonDocumentOptions() { AllowTrailingCommas = true })) + { + return JsonSerializer.Deserialize(jsonDocument.RootElement.GetRawText(), new JsonSerializerOptions() { }); + } + + } + catch (Exception e) + { + req.Body.Position = 0; + using (StreamReader readstream = new StreamReader(req.Body, Encoding.UTF8)) + { + string bodyContent = readstream.ReadToEnd(); + throw new HttpException(HttpStatusCode.BadRequest, $"The body of this request is invalid JSON. Content: {bodyContent}. Exception detail: {e.Message}"); + } + } + } + + return default(T); + } + public async static Task GetBody(HttpRequest req) { if (req.ContentLength > 0) diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/SanitizerList.cs b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/SanitizerList.cs new file mode 100644 index 00000000000..fe8dc075db0 --- /dev/null +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/SanitizerList.cs @@ -0,0 +1,10 @@ +using System.Collections.Generic; +using System.Text.Json; + +namespace Azure.Sdk.Tools.TestProxy.Common +{ + public class SanitizerBody { + public string Name { get; set; } + public JsonDocument Body { get; set; } + } +} diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/README.md b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/README.md index fcadbc371d8..f2334f1682d 100644 --- a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/README.md +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/README.md @@ -524,7 +524,7 @@ Add a more expansive Header sanitizer that uses a target group instead of filter ```jsonc // POST to URI /Admin/AddSanitizer -// dictionary dictionary +// headers { "x-abstraction-identifier": "HeaderRegexSanitizer" } @@ -549,6 +549,36 @@ Each sanitizer is optionally prefaced with the **specific part** of the request/ A sanitizer that does _not_ include this prefix is something different, and probably applies at the session level instead on an individual request/response pair. +#### Passing sanitizers in bulk + +In some cases, users need to register a lot (10+) of sanitizers. In this case, going back and forth with the proxy server individually is not very efficient. To ameliorate this, the proxy honors multiple sanitizers in the same request if the user utilizes `/Admin/AddSanitizers`. + +```jsonc +// POST to URI /Admin/AddSanitizers +// note the request body is simply an array of objects +[ + { + "Name": "GeneralRegexSanitizer", + "Body": { + "regex": "[a-zA-Z]?", + "value": "hello_there", + "condition": { + "UriRegex": ".+/Tables" + } + } + }, + { + "Name": "HeaderRegexSanitizer", + "Body": { // <-- the contents of this property mirror what would be passed in the request body for individual AddSanitizer() + "key": "Location", + "value": "fakeaccount", + "regex": "https\\:\\/\\/(?[a-z]+)\\.(?:table|blob|queue)\\.core\\.windows\\.net", + "groupForReplace": "account" + } + } +] +``` + ### For Sanitizers, Matchers, or Transforms in general When invoked as basic requests to the `Admin` controller, these settings will be applied to **all** further requests and responses. Both `Playback` and `Recording`. Where applicable.