Ignore live-only tests when in Playback or Record modes (#7430)

Fixes #7416 * Ignore live-only tests when in Playback or Record modes * Attribute to other live-only tests throughout repo
Azure · Sep 3, 2019 · 9be3f20 · 9be3f20
1 parent 38e2c46
commit 9be3f20
Show file tree

Hide file tree

Showing 34 changed files with 134 additions and 76 deletions.
diff --git a/...tion/Azure.Data.AppConfiguration/samples/Azure.Data.AppConfiguration.Samples.Tests.csproj b/...tion/Azure.Data.AppConfiguration/samples/Azure.Data.AppConfiguration.Samples.Tests.csproj
@@ -2,6 +2,7 @@
   <PropertyGroup>
     <TargetFrameworks>$(RequiredTargetFrameworks)</TargetFrameworks>
   </PropertyGroup>
+  <Import Project="..\..\..\core\Azure.Core\tests\LiveOnly.props" />
   <ItemGroup>
     <PackageReference Include="nunit" />
     <PackageReference Include="NUnit3TestAdapter" />

diff --git a/sdk/appconfiguration/Azure.Data.AppConfiguration/samples/Sample1_HelloWorld.cs b/sdk/appconfiguration/Azure.Data.AppConfiguration/samples/Sample1_HelloWorld.cs
@@ -2,13 +2,14 @@
 // Licensed under the MIT License. See License.txt in the project root for
 // license information.
 
+using Azure.Core.Testing;
 using NUnit.Framework;
 using System;
 using System.Diagnostics;
 
 namespace Azure.Data.AppConfiguration.Samples
 {
-    [Category("Live")]
+    [LiveOnly]
     public partial class ConfigurationSamples
     {
         [Test]

diff --git a/sdk/appconfiguration/Azure.Data.AppConfiguration/samples/Sample2_HelloWorldExtended.cs b/sdk/appconfiguration/Azure.Data.AppConfiguration/samples/Sample2_HelloWorldExtended.cs
@@ -2,6 +2,7 @@
 // Licensed under the MIT License. See License.txt in the project root for
 // license information.
 
+using Azure.Core.Testing;
 using NUnit.Framework;
 using System;
 using System.Collections.Generic;
@@ -10,7 +11,7 @@
 
 namespace Azure.Data.AppConfiguration.Samples
 {
-    [Category("Live")]
+    [LiveOnly]
     public partial class ConfigurationSamples
     {
         [Test]

diff --git a/sdk/appconfiguration/Azure.Data.AppConfiguration/samples/Sample4_Logging.cs b/sdk/appconfiguration/Azure.Data.AppConfiguration/samples/Sample4_Logging.cs
@@ -2,14 +2,15 @@
 // Licensed under the MIT License. See License.txt in the project root for
 // license information.
 
+using Azure.Core.Testing;
 using NUnit.Framework;
 using System;
 using System.Diagnostics;
 using System.Diagnostics.Tracing;
 
 namespace Azure.Data.AppConfiguration.Samples
 {
-    [Category("Live")]
+    [LiveOnly]
     public partial class ConfigurationSamples
     {
         // ConfigurationClient logs lots of useful information automatically to .NET's EventSource.

diff --git a/sdk/appconfiguration/Azure.Data.AppConfiguration/samples/Sample6_ConfiguringRetries.cs b/sdk/appconfiguration/Azure.Data.AppConfiguration/samples/Sample6_ConfiguringRetries.cs
@@ -2,13 +2,14 @@
 // Licensed under the MIT License. See License.txt in the project root for
 // license information.
 
+using Azure.Core.Testing;
 using NUnit.Framework;
 using System;
 using Azure.Core.Pipeline;
 
 namespace Azure.Data.AppConfiguration.Samples
 {
-    [Category("Live")]
+    [LiveOnly]
     public partial class ConfigurationSamples
     {
         [Test]

diff --git a/sdk/appconfiguration/Azure.Data.AppConfiguration/samples/Sample7_ConfiguringPipeline.cs b/sdk/appconfiguration/Azure.Data.AppConfiguration/samples/Sample7_ConfiguringPipeline.cs
@@ -2,14 +2,15 @@
 // Licensed under the MIT License. See License.txt in the project root for
 // license information.
 
+using Azure.Core.Testing;
 using Azure.Core.Pipeline;
 using NUnit.Framework;
 using System;
 using System.Net.Http;
 
 namespace Azure.Data.AppConfiguration.Samples
 {
-    [Category("Live")]
+    [LiveOnly]
     public partial class ConfigurationSamples
     {
         HttpClient s_client = new HttpClient();

diff --git a/sdk/core/Azure.Core/tests/LiveOnly.props b/sdk/core/Azure.Core/tests/LiveOnly.props
@@ -0,0 +1,7 @@
+<Project ToolsVersion="15.0">
+  <ItemGroup>
+    <Compile Include="$(MSBuildThisFileDirectory)\TestFramework\LiveOnlyAttribute.cs" Link="TestFramework\LiveOnlyAttribute.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)\TestFramework\RecordedTestMode.cs" Link="TestFramework\RecordedTestMode.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)\TestFramework\RecordedTestUtilities.cs" Link="TestFramework\RecordedTestUtilities.cs" />
+  </ItemGroup>
+</Project>
diff --git a/sdk/core/Azure.Core/tests/TestFramework.props b/sdk/core/Azure.Core/tests/TestFramework.props
@@ -11,6 +11,7 @@
   </ItemGroup>
 
   <ItemGroup Condition="'$(ExcludeRecordingFramework)' == 'true'">
+    <Compile Remove="$(MSBuildThisFileDirectory)\TestFramework\LiveOnlyAttribute.cs" />
     <Compile Remove="$(MSBuildThisFileDirectory)\TestFramework\PlaybackTransport.cs" />
     <Compile Remove="$(MSBuildThisFileDirectory)\TestFramework\*Record*.cs" />
   </ItemGroup>

diff --git a/sdk/core/Azure.Core/tests/TestFramework/LiveOnlyAttribute.cs b/sdk/core/Azure.Core/tests/TestFramework/LiveOnlyAttribute.cs
@@ -0,0 +1,36 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System;
+using NUnit.Framework;
+using NUnit.Framework.Interfaces;
+using NUnit.Framework.Internal;
+
+namespace Azure.Core.Testing
+{
+    /// <summary>
+    /// Attribute on test assemblies, classes, or methods that run only against live resources.
+    /// </summary>
+    [AttributeUsage(AttributeTargets.Method | AttributeTargets.Class | AttributeTargets.Assembly, AllowMultiple = true, Inherited = true)]
+    public class LiveOnlyAttribute : NUnitAttribute, IApplyToTest
+    {
+        /// <summary>
+        /// Modifies the <paramref name="test"/> by adding categories to it and changing the run state as needed.
+        /// </summary>
+        /// <param name="test">The <see cref="Test"/> to modify.</param>
+        public void ApplyToTest(Test test)
+        {
+            test.Properties.Add("Category", "Live");
+
+            if (test.RunState != RunState.NotRunnable)
+            {
+                RecordedTestMode mode = RecordedTestUtilities.GetModeFromEnvironment();
+                if (mode != RecordedTestMode.Live)
+                {
+                    test.RunState = RunState.Ignored;
+                    test.Properties.Set("_SKIPREASON", $"Live tests will not run when AZURE_TEST_MODE is {mode}");
+                }
+            }
+        }
+    }
+}
diff --git a/sdk/core/Azure.Core/tests/TestFramework/RecordedTestBase.cs b/sdk/core/Azure.Core/tests/TestFramework/RecordedTestBase.cs
@@ -1,7 +1,6 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
-using System;
 using System.IO;
 using NUnit.Framework;
 
@@ -10,8 +9,6 @@ namespace Azure.Core.Testing
     [Category("Recorded")]
     public abstract class RecordedTestBase: ClientTestBase
     {
-        private const string ModeEnvironmentVariableName = "AZURE_TEST_MODE";
-
         protected RecordedTestSanitizer Sanitizer { get; set; }
 
         protected RecordMatcher Matcher { get; set; }
@@ -20,7 +17,7 @@ public abstract class RecordedTestBase: ClientTestBase
 
         protected RecordedTestMode Mode { get; }
 
-        protected RecordedTestBase(bool isAsync) : this(isAsync, GetModeFromEnvironment())
+        protected RecordedTestBase(bool isAsync) : this(isAsync, RecordedTestUtilities.GetModeFromEnvironment())
         {
         }
 
@@ -31,19 +28,6 @@ protected RecordedTestBase(bool isAsync, RecordedTestMode mode) : base(isAsync)
             Mode = mode;
         }
 
-        internal static RecordedTestMode GetModeFromEnvironment()
-        {
-            string modeString = Environment.GetEnvironmentVariable(ModeEnvironmentVariableName);
-
-            RecordedTestMode mode = RecordedTestMode.Playback;
-            if (!string.IsNullOrEmpty(modeString))
-            {
-                mode = (RecordedTestMode)Enum.Parse(typeof(RecordedTestMode), modeString, true);
-            }
-
-            return mode;
-        }
-
         private string GetSessionFilePath(string name = null)
         {
             TestContext.TestAdapter testAdapter = TestContext.CurrentContext.Test;

diff --git a/sdk/core/Azure.Core/tests/TestFramework/RecordedTestUtilities.cs b/sdk/core/Azure.Core/tests/TestFramework/RecordedTestUtilities.cs
@@ -0,0 +1,26 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System;
+
+namespace Azure.Core.Testing
+{
+    internal static class RecordedTestUtilities
+    {
+        private const string ModeEnvironmentVariableName = "AZURE_TEST_MODE";
+
+        internal static RecordedTestMode GetModeFromEnvironment()
+        {
+            string modeString = Environment.GetEnvironmentVariable(ModeEnvironmentVariableName);
+
+            RecordedTestMode mode = RecordedTestMode.Playback;
+            if (!string.IsNullOrEmpty(modeString))
+            {
+                mode = (RecordedTestMode)Enum.Parse(typeof(RecordedTestMode), modeString, true);
+            }
+
+            return mode;
+        }
+
+    }
+}
diff --git a/sdk/core/Azure.Core/tests/samples/Sample1_HelloWorld.cs b/sdk/core/Azure.Core/tests/samples/Sample1_HelloWorld.cs
@@ -3,14 +3,15 @@
 // license information.
 
 using Azure.Core.Pipeline;
+using Azure.Core.Testing;
 using NUnit.Framework;
 using System;
 using System.IO;
 using System.Threading.Tasks;
 
 namespace Azure.Core.Samples
 {
-    [Category("Live")]
+    [LiveOnly]
     public partial class BaseSamples
     {
         [Test]

diff --git a/...keyvault/Azure.Security.KeyVault.Keys/samples/Azure.Security.KeyVault.Keys.Samples.csproj b/...keyvault/Azure.Security.KeyVault.Keys/samples/Azure.Security.KeyVault.Keys.Samples.csproj
@@ -2,6 +2,7 @@
   <PropertyGroup>
     <TargetFrameworks>$(RequiredTargetFrameworks)</TargetFrameworks>
   </PropertyGroup>
+  <Import Project="..\..\..\core\Azure.Core\tests\LiveOnly.props" />
   <ItemGroup>
     <PackageReference Include="nunit" />
     <PackageReference Include="NUnit3TestAdapter" />

diff --git a/sdk/keyvault/Azure.Security.KeyVault.Keys/samples/Sample1_HelloWorld.cs b/sdk/keyvault/Azure.Security.KeyVault.Keys/samples/Sample1_HelloWorld.cs
@@ -2,19 +2,19 @@
 // Licensed under the MIT License. See License.txt in the project root for
 // license information.
 
+using Azure.Core.Testing;
 using Azure.Identity;
 using NUnit.Framework;
 using System;
 using System.Diagnostics;
-using System.Security.Cryptography;
 using System.Threading;
 
 namespace Azure.Security.KeyVault.Keys.Samples
 {
     /// <summary>
     /// Sample demonstrates how to set, get, update and delete a key using the synchronous methods of the KeyClient.
     /// </summary>
-    [Category("Live")]
+    [LiveOnly]
     public partial class HelloWorld
     {
         [Test]

diff --git a/sdk/keyvault/Azure.Security.KeyVault.Keys/samples/Sample1_HelloWorldAsync.cs b/sdk/keyvault/Azure.Security.KeyVault.Keys/samples/Sample1_HelloWorldAsync.cs
@@ -2,24 +2,23 @@
 // Licensed under the MIT License. See License.txt in the project root for
 // license information.
 
+using Azure.Core.Testing;
 using Azure.Identity;
 using NUnit.Framework;
 using System;
 using System.Diagnostics;
-using System.Security.Cryptography;
-using System.Threading;
 using System.Threading.Tasks;
 
 namespace Azure.Security.KeyVault.Keys.Samples
 {
     /// <summary>
     /// Sample demonstrates how to set, get, update and delete a key using the asynchronous methods of the KeyClient.
     /// </summary>
-    [Category("Live")]
+    [LiveOnly]
     public partial class HelloWorld
     {
         [Test]
-        public async Task HelloWorldASync()
+        public async Task HelloWorldAsync()
         {
             // Environment variable with the Key Vault endpoint.
             string keyVaultUrl = Environment.GetEnvironmentVariable("AZURE_KEYVAULT_URL");

diff --git a/sdk/keyvault/Azure.Security.KeyVault.Keys/samples/Sample2_BackupAndRestore.cs b/sdk/keyvault/Azure.Security.KeyVault.Keys/samples/Sample2_BackupAndRestore.cs
@@ -2,6 +2,7 @@
 // Licensed under the MIT License. See License.txt in the project root for
 // license information.
 
+using Azure.Core.Testing;
 using Azure.Identity;
 using NUnit.Framework;
 using System;
@@ -14,7 +15,7 @@ namespace Azure.Security.KeyVault.Keys.Samples
     /// Sample demonstrates how to backup and restore keys in the Key Vault
     /// using the synchronous methods of the KeyClient.
     /// </summary>
-    [Category("Live")]
+    [LiveOnly]
     public partial class BackupAndRestore
     {
         [Test]

diff --git a/sdk/keyvault/Azure.Security.KeyVault.Keys/samples/Sample2_BackupAndRestoreAsync.cs b/sdk/keyvault/Azure.Security.KeyVault.Keys/samples/Sample2_BackupAndRestoreAsync.cs
@@ -2,11 +2,11 @@
 // Licensed under the MIT License. See License.txt in the project root for
 // license information.
 
+using Azure.Core.Testing;
 using Azure.Identity;
 using NUnit.Framework;
 using System;
 using System.IO;
-using System.Threading;
 using System.Threading.Tasks;
 
 namespace Azure.Security.KeyVault.Keys.Samples
@@ -15,7 +15,7 @@ namespace Azure.Security.KeyVault.Keys.Samples
     /// Sample demonstrates how to backup and restore keys in the Key Vault
     /// using the asynchronous methods of the KeyClient.
     /// </summary>
-    [Category("Live")]
+    [LiveOnly]
     public partial class BackupAndRestore
     {
         [Test]

diff --git a/sdk/keyvault/Azure.Security.KeyVault.Keys/samples/Sample3_GetKeys.cs b/sdk/keyvault/Azure.Security.KeyVault.Keys/samples/Sample3_GetKeys.cs
@@ -2,12 +2,12 @@
 // Licensed under the MIT License. See License.txt in the project root for
 // license information.
 
+using Azure.Core.Testing;
 using Azure.Identity;
 using NUnit.Framework;
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
-using System.Security.Cryptography;
 using System.Threading;
 
 namespace Azure.Security.KeyVault.Keys.Samples
@@ -17,7 +17,7 @@ namespace Azure.Security.KeyVault.Keys.Samples
     /// and list deleted keys in a soft-delete enabled Key Vault
     /// using the synchronous methods of the KeyClient.
     /// </summary>
-    [Category("Live")]
+    [LiveOnly]
     public partial class GetKeys
     {
         [Test]

diff --git a/sdk/keyvault/Azure.Security.KeyVault.Keys/samples/Sample3_GetKeysAsync.cs b/sdk/keyvault/Azure.Security.KeyVault.Keys/samples/Sample3_GetKeysAsync.cs
@@ -2,12 +2,11 @@
 // Licensed under the MIT License. See License.txt in the project root for
 // license information.
 
+using Azure.Core.Testing;
 using Azure.Identity;
 using NUnit.Framework;
 using System;
-using System.Collections.Generic;
 using System.Diagnostics;
-using System.Threading;
 using System.Threading.Tasks;
 
 namespace Azure.Security.KeyVault.Keys.Samples
@@ -17,7 +16,7 @@ namespace Azure.Security.KeyVault.Keys.Samples
     /// and list deleted keys in a soft-delete enabled Key Vault
     /// using the asynchronous methods of the KeyClient.
     /// </summary>
-    [Category("Live")]
+    [LiveOnly]
     public partial class GetKeys
     {
         [Test]

diff --git a/sdk/keyvault/Azure.Security.KeyVault.Keys/samples/Sample4_EncryptDecrypt.cs b/sdk/keyvault/Azure.Security.KeyVault.Keys/samples/Sample4_EncryptDecrypt.cs
@@ -1,5 +1,5 @@
-using Azure.Identity;
-using Azure.Security.KeyVault.Keys;
+using Azure.Core.Testing;
+using Azure.Identity;
 using Azure.Security.KeyVault.Keys.Cryptography;
 using NUnit.Framework;
 using System;
@@ -13,7 +13,7 @@ namespace Azure.Security.KeyVault.Keys.Samples
     /// <summary>
     /// Sample demonstrates how to encrypt and decrypt a single block of plain text with an RSA key using the synchronous methods of the CryptographyClient.
     /// </summary>
-    [Category("Live")]
+    [LiveOnly]
     public partial class Sample4_EncryptDecypt
     {
         [Test]