(Storage) Add recorded tests (#6609)

* Switch from MSTest to NUnit 3

* Add recorded tests

* Test fixes
- Change samples for cross-platform file size deltas
- Make tests more reliable with MemoryStream.ToArray() instead .GetBuffer()

* Improve the StorageRequestFailedException error message
Here's an example of an error message with additional info specified:

Server failed to authenticate the request. Make sure the value of Authorization header is formed correctly including the signature.
RequestId:6b6d5a57-201e-007b-2291-250f92000000
Time:2019-06-18T04:53:02.9793056Z
Status: 403 (Server failed to authenticate the request. Make sure the value of Authorization header is formed correctly including the signature.)

ErrorCode: AuthenticationFailed

Additional Information:
AuthenticationErrorDetail: The MAC signature found in the HTTP request 'T8bvpwE2Fp9cKr9yAfo1kblARYwUoRLovmH1nIlPrpQ=' is not the same as any computed signature. Server used following string to sign: 'GET

x-ms-client-request-id:097eb32f-12eb-476b-8fe2-e912b96e6974
x-ms-date:Tue, 18 Jun 2019 04:53:01 GMT
x-ms-return-client-request-id:true
x-ms-version:2018-11-09
/storageteglazatesting/
comp:properties
restype:account'.

Headers:
Server: Microsoft-HTTPAPI/2.0
x-ms-request-id: 6b6d5a57-201e-007b-2291-250f92000000
x-ms-error-code: AuthenticationFailed
Date: Tue, 18 Jun 2019 04:53:02 GMT
Content-Length: 779
Content-Type: application/xml

sdk/core/Azure.Core/src/Pipeline/HttpHeader.cs

@@ -50,6 +50,7 @@ public bool Equals(HttpHeader other)
         public static class Names
         {
             public static string Date => "Date";
+            public static string XMsDate => "x-ms-date";
             public static string ContentType => "Content-Type";
             public static string XMsRequestId => "x-ms-request-id";
             public static string UserAgent => "User-Agent";

sdk/core/Azure.Core/src/ResponseExceptionExtensions.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT License.
 
 using System;
+using System.Globalization;
 using System.IO;
 using System.Text;
 using System.Threading.Tasks;
@@ -45,7 +46,7 @@ public static async Task<string> CreateRequestFailedMessageAsync(string message,
             StringBuilder messageBuilder = new StringBuilder()
                 .AppendLine(message)
                 .Append("Status: ")
-                .Append(response.Status.ToString())
+                .Append(response.Status.ToString(CultureInfo.InvariantCulture))
                 .Append(" (")
                 .Append(response.ReasonPhrase)
                 .AppendLine(")");
@@ -59,7 +60,7 @@ public static async Task<string> CreateRequestFailedMessageAsync(string message,
 
                 using (var streamReader = new StreamReader(response.ContentStream, encoding))
                 {
-                    string content = async ? await streamReader.ReadToEndAsync() : streamReader.ReadToEnd();
+                    string content = async ? await streamReader.ReadToEndAsync().ConfigureAwait(false) : streamReader.ReadToEnd();
 
                     messageBuilder.AppendLine(content);
                 }

sdk/core/Azure.Core/src/ResponseHeaders.cs

@@ -17,7 +17,11 @@ public ResponseHeaders(Response response)
             _response = response;
         }
 
-        public DateTimeOffset? Date => TryGetValue(HttpHeader.Names.Date, out var value) ? (DateTimeOffset?)DateTimeOffset.Parse(value) : null;
+        public DateTimeOffset? Date =>
+            TryGetValue(HttpHeader.Names.Date, out var value) ||
+            TryGetValue(HttpHeader.Names.XMsDate, out value) ?
+                (DateTimeOffset?)DateTimeOffset.Parse(value) :
+                null;
 
         public string ContentType => TryGetValue(HttpHeader.Names.ContentType, out var value) ? value : null;
 

sdk/core/Azure.Core/src/Shared/ArrayBufferWriter.cs

@@ -39,7 +39,9 @@ public ArrayBufferWriter()
         public ArrayBufferWriter(int initialCapacity)
         {
             if (initialCapacity <= 0)
+#pragma warning disable CA2208 // Instantiate argument exceptions correctly
                 throw new ArgumentException(nameof(initialCapacity));
+#pragma warning restore CA2208 // Instantiate argument exceptions correctly
 
             _buffer = new T[initialCapacity];
             _index = 0;
@@ -98,7 +100,9 @@ public void Clear()
         public void Advance(int count)
         {
             if (count < 0)
+#pragma warning disable CA2208 // Instantiate argument exceptions correctly
                 throw new ArgumentException(nameof(count));
+#pragma warning restore CA2208 // Instantiate argument exceptions correctly
 
             if (_index > _buffer.Length - count)
                 ThrowInvalidOperationException_AdvancedTooFar(_buffer.Length);
@@ -155,7 +159,9 @@ public Span<T> GetSpan(int sizeHint = 0)
         private void CheckAndResizeBuffer(int sizeHint)
         {
             if (sizeHint < 0)
+#pragma warning disable CA2208 // Instantiate argument exceptions correctly
                 throw new ArgumentException(nameof(sizeHint));
+#pragma warning restore CA2208 // Instantiate argument exceptions correctly
 
             if (sizeHint == 0)
             {

sdk/core/Azure.Core/tests/HeadersTests.cs

@@ -39,6 +39,26 @@ public void DateReturnsDateHeaderValue()
             Assert.AreEqual(new DateTimeOffset(2013, 9, 29, 1, 2, 3, TimeSpan.Zero), mockResponse.Headers.Date);
         }
 
+        [Test]
+        public void DateReturnsXMsDateHeaderValue()
+        {
+            var mockResponse = new MockResponse(200);
+            mockResponse.AddHeader(new HttpHeader("x-ms-date", "Sun, 29 Sep 2013 01:02:03 GMT"));
+
+            Assert.AreEqual(new DateTimeOffset(2013, 9, 29, 1, 2, 3, TimeSpan.Zero), mockResponse.Headers.Date);
+        }
+
+        [Test]
+        public void DateReturnsDateHeaderValueFirst()
+        {
+            var mockResponse = new MockResponse(200);
+            mockResponse.AddHeader(new HttpHeader("x-ms-date", "Sun, 29 Sep 2013 01:02:03 GMT"));
+            mockResponse.AddHeader(new HttpHeader("Date", "Sun, 29 Sep 2013 09:02:03 GMT"));
+
+            Assert.AreEqual(new DateTimeOffset(2013, 9, 29, 9, 2, 3, TimeSpan.Zero), mockResponse.Headers.Date);
+        }
+
+
         [Test]
         public void DateReturnsNullForNoHeader()
         {

sdk/core/Azure.Core/tests/TestFramework/PlaybackTransport.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
+using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Threading.Tasks;
@@ -14,21 +15,62 @@ public class PlaybackTransport : MockTransport
 
         private readonly RecordMatcher _matcher;
 
-        public PlaybackTransport(RecordSession session, RecordMatcher matcher)
+        private readonly Random _random;
+
+        public PlaybackTransport(RecordSession session, RecordMatcher matcher, Random random)
         {
             _session = session;
             _matcher = matcher;
+            _random = random;
         }
 
         public override void Process(HttpPipelineMessage message)
         {
+            // Some tests will check if the Request Content is being read (to
+            // verify their Progress handling) so we'll just copy it to a
+            // MemoryStream
+            if (message.Request.Content != null &&
+                message.Request.Content.TryComputeLength(out long length) &&
+                length > 0)
+            {
+                using (MemoryStream stream = new MemoryStream((int)length))
+                {
+                    message.Request.Content.WriteTo(stream, message.CancellationToken);
+                }
+            }
+
             message.Response = GetResponse(_session.Lookup(message.Request, _matcher));
         }
 
-        public override Task ProcessAsync(HttpPipelineMessage message)
+        public override async Task ProcessAsync(HttpPipelineMessage message)
         {
+            // Some tests will check if the Request Content is being read (to
+            // verify their Progress handling) so we'll just copy it to a
+            // MemoryStream asynchronously
+            if (message.Request.Content != null &&
+                message.Request.Content.TryComputeLength(out long length) &&
+                length > 0)
+            {
+                using (MemoryStream stream = new MemoryStream((int)length))
+                {
+                    await message.Request.Content.WriteToAsync(stream, message.CancellationToken).ConfigureAwait(false);
+                }
+            }
+
             message.Response = GetResponse(_session.Lookup(message.Request, _matcher));
-            return Task.CompletedTask;
+        }
+
+        public override Request CreateRequest()
+        {
+            lock (_random)
+            {
+                // Force a call to random.NewGuid so we keep the random seed
+                // unified between record/playback
+                _random.NewGuid();
+
+                // TODO: Pavel will think about ways to unify this
+            }
+            return base.CreateRequest();
         }
 
         public Response GetResponse(RecordEntry recordEntry)

sdk/core/Azure.Core/tests/TestFramework/RecordMatcher.cs

@@ -20,6 +20,7 @@ public RecordMatcher(RecordedTestSanitizer sanitizer)
         public HashSet<string> ExcludeHeaders = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
         {
             "Date",
+            "x-ms-date",
             "x-ms-client-request-id",
             "User-Agent"
         };

sdk/core/Azure.Core/tests/TestFramework/RecordedTestBase.cs

@@ -31,7 +31,7 @@ protected RecordedTestBase(bool isAsync, RecordedTestMode mode) : base(isAsync)
             Mode = mode;
         }
 
-        private static RecordedTestMode GetModeFromEnvironment()
+        internal static RecordedTestMode GetModeFromEnvironment()
         {
             string modeString = Environment.GetEnvironmentVariable(ModeEnvironmentVariableName);
 

sdk/core/Azure.Core/tests/TestFramework/RecordedTestSanitizer.cs

@@ -7,7 +7,7 @@ namespace Azure.Core.Testing
 {
     public class RecordedTestSanitizer
     {
-        private const string SanitizeValue = "Sanitized";
+        protected const string SanitizeValue = "Sanitized";
         private static readonly string[] SanitizeValueArray = { SanitizeValue };
 
         private static readonly string[] SanitizedHeaders = { "Authorization" };

sdk/core/Azure.Core/tests/TestFramework/TestRecording.cs

@@ -10,9 +10,10 @@
 
 namespace Azure.Core.Testing
 {
-    public class TestRecording: IDisposable
+    public class TestRecording : IDisposable
     {
         private const string RandomSeedVariableKey = "RandomSeed";
+        private const string DateTimeOffsetNowVariableKey = "DateTimeOffsetNow";
 
         public TestRecording(RecordedTestMode mode, string sessionFile, RecordedTestSanitizer sanitizer, RecordMatcher matcher)
         {
@@ -97,6 +98,50 @@ public Random Random
             }
         }
 
+        /// <summary>
+        /// The moment in time that this test is being run.
+        /// </summary>
+        private DateTimeOffset? _now;
+
+        /// <summary>
+        /// Gets the moment in time that this test is being run.  This is useful
+        /// for any test recordings that capture the current time.
+        /// </summary>
+        public DateTimeOffset Now
+        {
+            get
+            {
+                if (_now == null)
+                {
+                    switch (Mode)
+                    {
+                        case RecordedTestMode.Live:
+                            _now = DateTimeOffset.Now;
+                            break;
+                        case RecordedTestMode.Record:
+                            // While we can cache DateTimeOffset.Now for playing back tests,
+                            // a number of auth mechanisms are time sensitive and will require
+                            // values in the present when re-recording
+                            _now = DateTimeOffset.Now;
+                            _session.Variables[DateTimeOffsetNowVariableKey] = _now.Value.ToString("O"); // Use the "Round-Trip Format"
+                            break;
+                        case RecordedTestMode.Playback:
+                            _now = DateTimeOffset.Parse(_session.Variables[DateTimeOffsetNowVariableKey]);
+                            break;
+                        default:
+                            throw new ArgumentOutOfRangeException();
+                    }
+                }
+                return _now.Value;
+            }
+        }
+
+        /// <summary>
+        /// Gets the moment in time that this test is being run in UTC format.
+        /// This is useful for any test recordings that capture the current time.
+        /// </summary>
+        public DateTimeOffset UtcNow => Now.ToUniversalTime();
+
         private RecordSession Load()
         {
             using FileStream fileStream = File.OpenRead(_sessionFile);
@@ -142,7 +187,7 @@ public HttpPipelineTransport CreateTransport(HttpPipelineTransport currentTransp
                 case RecordedTestMode.Record:
                     return new RecordTransport(_session, currentTransport, entry => !_disableRecording.Value, Random);
                 case RecordedTestMode.Playback:
-                    return new PlaybackTransport(_session, _matcher);
+                    return new PlaybackTransport(_session, _matcher, Random);
                 default:
                     throw new ArgumentOutOfRangeException(nameof(Mode), Mode, null);
             }

sdk/storage/Azure.Storage.Blobs/src/StorageRequestFailedException.cs

@@ -45,11 +45,7 @@ static partial void CustomizeFromXml(XElement root, StorageError error)
         /// <param name="response">The failed response.</param>
         /// <returns>A StorageRequestFailedException.</returns>
         public Exception CreateException(Azure.Response response)
-            => new StorageRequestFailedException(response, this.Message)
-                {
-                    ErrorCode = this.Code,
-                    AdditionalInformation = this.AdditionalInformation
-                };
+            => new StorageRequestFailedException(response, this.Message, null, this.Code, this.AdditionalInformation);
     }
 
     /// <summary>
@@ -63,6 +59,6 @@ internal partial class ConditionNotMetError
         /// <param name="response">The failed response.</param>
         /// <returns>A StorageRequestFailedException.</returns>
         public Exception CreateException(Azure.Response response)
-            => new StorageRequestFailedException(response) { ErrorCode = this.ErrorCode };
+            => new StorageRequestFailedException(response, null, null, this.ErrorCode);
     }
 }