Add proxy audit log and other assorted fixes

tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/AdminTests.cs

@@ -939,33 +939,7 @@ public async Task TestAddSanitizerContinuesWithTwoRequiredParams()
         }
 
         [Fact]
-        public async Task RemoveSanitizerErrorsForInvalidIdOnRecording()
-        {
-            RecordingHandler testRecordingHandler = new RecordingHandler(Directory.GetCurrentDirectory());
-            var httpContext = new DefaultHttpContext();
-            await testRecordingHandler.StartPlaybackAsync("Test.RecordEntries/oauth_request_with_variables.json", httpContext.Response);
-            var recordingId = httpContext.Response.Headers["x-recording-id"];
-            httpContext.Request.Headers["x-recording-id"] = recordingId;
-            httpContext.Response.Body = new MemoryStream();
-            var controller = new Admin(testRecordingHandler, _nullLogger)
-            {
-                ControllerContext = new ControllerContext()
-                {
-                    HttpContext = httpContext
-                }
-            };
-
-            var testSet = new RemoveSanitizerList() { Sanitizers = new List<string>() { "0" } };
-
-            var assertion = await Assert.ThrowsAsync<HttpException>(
-                async () => await controller.RemoveSanitizers(testSet)
-            );
-
-            Assert.Contains("Unable to remove 1 sanitizer. Detailed list follows: \nThe requested sanitizer for removal \"0\" is not active on recording/playback with id", assertion.Message);
-        }
-
-        [Fact]
-        public async Task RemoveSanitizerErrorsForInvalidId()
+        public async Task RemoveSanitizersSilentlyAcceptsInvalidSanitizer()
         {
             RecordingHandler testRecordingHandler = new RecordingHandler(Directory.GetCurrentDirectory());
             var httpContext = new DefaultHttpContext();
@@ -980,11 +954,9 @@ public async Task RemoveSanitizerErrorsForInvalidId()
 
             var testSet = new RemoveSanitizerList() { Sanitizers = new List<string>() { "AZSDK00-1" } };
 
-            var assertion = await Assert.ThrowsAsync<HttpException>(
-                async () => await controller.RemoveSanitizers(testSet)
-            );
+            await controller.RemoveSanitizers(testSet);
 
-            Assert.Equal("Unable to remove 1 sanitizer. Detailed list follows: \nThe requested sanitizer for removal \"AZSDK00-1\" is not active at the session level.", assertion.Message);
+            Assert.Equal(200, httpContext.Response.StatusCode);
         }
 
         [Fact]

tools/test-proxy/Azure.Sdk.Tools.TestProxy/Admin.cs

@@ -70,39 +70,26 @@ public async Task RemoveSanitizers([FromBody]RemoveSanitizerList sanitizerList)
             var recordingId = RecordingHandler.GetHeader(Request, "x-recording-id", allowNulls: true);
 
             var removedSanitizers = new List<string>();
-            var exceptionsList = new List<string>();
 
             if (sanitizerList.Sanitizers.Count == 0)
             {
                 throw new HttpException(HttpStatusCode.BadRequest, "At least one sanitizerId for removal must be provided.");
             }
 
             foreach(var sanitizerId in sanitizerList.Sanitizers) {
-                try
+                var removedId = await _recordingHandler.UnregisterSanitizer(sanitizerId, recordingId);
+                if (!string.IsNullOrWhiteSpace(removedId))
                 {
-                    var removedId = await _recordingHandler.UnregisterSanitizer(sanitizerId, recordingId);
                     removedSanitizers.Add(sanitizerId);
                 }
-                catch (HttpException ex) {
-                    exceptionsList.Add(ex.Message);
-                }
             }
 
-            if (exceptionsList.Count > 0)
-            {
-                var varExceptionMessage = $"Unable to remove {exceptionsList.Count} sanitizer{(exceptionsList.Count > 1 ? 's' : string.Empty)}. Detailed list follows: \n"
-                    + string.Join("\n", exceptionsList);
-                throw new HttpException(HttpStatusCode.BadRequest, varExceptionMessage);
-            }
-            else
-            {
-                var json = JsonSerializer.Serialize(new { Removed = removedSanitizers });
+            var json = JsonSerializer.Serialize(new { Removed = removedSanitizers });
 
-                Response.ContentType = "application/json";
-                Response.ContentLength = json.Length;
+            Response.ContentType = "application/json";
+            Response.ContentLength = json.Length;
 
-                await Response.WriteAsync(json);
-            }
+            await Response.WriteAsync(json);
         }
 
         [HttpGet]
@@ -171,22 +158,11 @@ public async Task AddSanitizers()
                 throw new HttpException(HttpStatusCode.BadRequest, "When bulk adding sanitizers, ensure there is at least one sanitizer added in each batch. Received 0 work items.");
             }
 
-            var registeredSanitizers = new List<string>();
+            // we need check if a recording id is present BEFORE the loop, as we want to encapsulate the entire
+            // sanitizer add operation in a single lock, rather than gathering and releasing a sanitizer lock
+            // for the session/recording on _each_ sanitizer addition.
 
-            // register them all
-            foreach(var sanitizer in workload)
-            {
-                if (recordingId != null)
-                {
-                    var registeredId = await _recordingHandler.RegisterSanitizer(sanitizer, recordingId);
-                    registeredSanitizers.Add(registeredId);
-                }
-                else
-                {
-                    var registeredId = await _recordingHandler.RegisterSanitizer(sanitizer);
-                    registeredSanitizers.Add(registeredId);
-                }
-            }
+            var registeredSanitizers = await _recordingHandler.RegisterSanitizers(workload, recordingId);
 
             if (recordingId != null)
             {

tools/test-proxy/Azure.Sdk.Tools.TestProxy/Audit.cs

@@ -0,0 +1,93 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using Azure.Sdk.Tools.TestProxy.Common;
+using Azure.Sdk.Tools.TestProxy.Common.Exceptions;
+using Azure.Sdk.Tools.TestProxy.Store;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Logging;
+using System;
+using System.Collections.Generic;
+using System.Net;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Azure.Sdk.Tools.TestProxy
+{
+    [ApiController]
+    [Route("[controller]/[action]")]
+    public sealed class Audit : ControllerBase
+    {
+        private readonly ILogger _logger;
+        private readonly RecordingHandler _recordingHandler;
+
+        public Audit(RecordingHandler recordingHandler, ILoggerFactory loggerFactory)
+        {
+            _recordingHandler = recordingHandler;
+            _logger = loggerFactory.CreateLogger<Record>();
+        }
+
+        [HttpGet]
+        public async Task GetAuditLogs()
+        {
+
+            var allAuditSessions = _recordingHandler.RetrieveOngoingAuditLogs();
+            allAuditSessions.AddRange(_recordingHandler.AuditSessions.Values);
+
+            StringBuilder stringBuilder = new StringBuilder();
+
+            foreach (var auditLogQueue in allAuditSessions) {
+                while (auditLogQueue.TryDequeue(out var logItem))
+                {
+                    stringBuilder.Append(logItem.ToCsvString() + Environment.NewLine);
+                }
+            }
+
+            Response.ContentType = "text/plain";
+
+            await Response.WriteAsync(stringBuilder.ToString());
+        }
+
+
+        [HttpPost]
+        public async Task Push([FromBody()] IDictionary<string, object> options = null)
+        {
+            DebugLogger.LogAdminRequestDetails(_logger, Request);
+
+            var pathToAssets = RecordingHandler.GetAssetsJsonLocation(StoreResolver.ParseAssetsJsonBody(options), _recordingHandler.ContextDirectory);
+
+            await _recordingHandler.Store.Push(pathToAssets);
+        }
+
+        [HttpPost]
+        [AllowEmptyBody]
+        public async Task Stop([FromBody()] IDictionary<string, string> variables = null)
+        {
+            string id = RecordingHandler.GetHeader(Request, "x-recording-id");
+            bool save = true;
+            EntryRecordMode mode = RecordingHandler.GetRecordMode(Request);
+
+            if (mode != EntryRecordMode.Record && mode != EntryRecordMode.DontRecord)
+            {
+                throw new HttpException(HttpStatusCode.BadRequest, "When stopping a recording and providing a \"x-recording-skip\" value, only value \"request-response\" is accepted.");
+            }
+
+            if (mode == EntryRecordMode.DontRecord)
+            {
+                save = false;
+            }
+
+            DebugLogger.LogAdminRequestDetails(_logger, Request);
+
+            await _recordingHandler.StopRecording(id, variables: variables, saveRecording: save);
+        }
+
+        public async Task HandleRequest()
+        {
+            string id = RecordingHandler.GetHeader(Request, "x-recording-id");
+
+            await _recordingHandler.HandleRecordRequestAsync(id, Request, Response);
+        }
+    }
+}

tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/AuditLogItem.cs

@@ -0,0 +1,39 @@
+using System;
+
+namespace Azure.Sdk.Tools.TestProxy.Common
+{
+    public class AuditLogItem
+    {
+        public string RecordingId { get; set; }
+
+        public DateTime Timestamp { get; set; }
+
+        public string Uri { get; set; }
+
+        public string Verb { get; set; }
+
+        public string Message { get; set; }
+
+        public AuditLogItem(string recordingId, string requestUri, string requestMethod) {
+            RecordingId = recordingId;
+            Timestamp = DateTime.UtcNow;
+
+            Uri = requestUri;
+            Verb = requestMethod;
+        }
+
+        public string ToCsvString()
+        {
+            return $"{RecordingId},{Timestamp.ToString("o")},{Verb},{Uri},{Message}";
+        }
+
+        public AuditLogItem(string recordingId, string message)
+        {
+            RecordingId = recordingId;
+            Timestamp = DateTime.UtcNow;
+
+            Message = message;
+        }
+
+    }
+}

tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/DebugLogger.cs

@@ -5,6 +5,7 @@
 using System.Collections.Generic;
 using System.Text.Json;
 using Microsoft.AspNetCore.Http.Extensions;
+using System.Collections.Concurrent;
 
 namespace Azure.Sdk.Tools.TestProxy.Common
 {

tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/ModifiableRecordSession.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Linq;
 using System.Net.Http;
@@ -9,13 +10,13 @@ namespace Azure.Sdk.Tools.TestProxy.Common
 {
     public class ModifiableRecordSession
     {
-        public RecordMatcher CustomMatcher { get; set;}
+        public RecordMatcher CustomMatcher { get; set; }
 
         public RecordSession Session { get; }
 
         public ModifiableRecordSession(SanitizerDictionary sanitizerRegistry, string sessionId)
         {
-            lock(sanitizerRegistry.SessionSanitizerLock)
+            lock (sanitizerRegistry.SessionSanitizerLock)
             {
                 this.AppliedSanitizers = sanitizerRegistry.SessionSanitizers.ToList();
             }
@@ -42,18 +43,17 @@ public ModifiableRecordSession(RecordSession session, SanitizerDictionary saniti
 
         public List<ResponseTransform> AdditionalTransforms { get; } = new List<ResponseTransform>();
 
-        public SemaphoreSlim SanitizerLock = new SemaphoreSlim(1);
-
         public List<string> AppliedSanitizers { get; set; } = new List<string>();
         public List<string> ForRemoval { get; } = new List<string>();
 
         public string SourceRecordingId { get; set; }
 
         public int PlaybackResponseTime { get; set; }
 
+        public ConcurrentQueue<AuditLogItem> AuditLog { get; set; } = new ConcurrentQueue<AuditLogItem>();
         public async void ResetExtensions(SanitizerDictionary sanitizerDictionary)
         {
-            await SanitizerLock.WaitAsync();
+            await Session.EntryLock.WaitAsync();
             try
             {
                 AdditionalTransforms.Clear();
@@ -66,7 +66,7 @@ public async void ResetExtensions(SanitizerDictionary sanitizerDictionary)
             }
             finally
             {
-                SanitizerLock.Release();
+                Session.EntryLock.Release();
             }
         }
     }

tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/RecordSession.cs

@@ -3,8 +3,6 @@
 
 using System;
 using System.Collections.Generic;
-using System.IO;
-using System.Linq;
 using System.Text.Json;
 using System.Threading;
 using System.Threading.Tasks;