Ignore live-only tests when in Playback or Record modes

Partially fixes Azure#7416

sdk/core/Azure.Core/tests/LiveOnly.props

@@ -0,0 +1,7 @@
+<Project ToolsVersion="15.0">
+  <ItemGroup>
+    <Compile Include="$(MSBuildThisFileDirectory)\TestFramework\LiveOnlyAttribute.cs" Link="TestFramework\LiveOnlyAttribute.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)\TestFramework\RecordedTestMode.cs" Link="TestFramework\RecordedTestMode.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)\TestFramework\RecordedTestUtilities.cs" Link="TestFramework\RecordedTestUtilities.cs" />
+  </ItemGroup>
+</Project>

sdk/core/Azure.Core/tests/TestFramework.props

@@ -11,6 +11,7 @@
   </ItemGroup>
 
   <ItemGroup Condition="'$(ExcludeRecordingFramework)' == 'true'">
+    <Compile Remove="$(MSBuildThisFileDirectory)\TestFramework\LiveOnlyAttribute.cs" />
     <Compile Remove="$(MSBuildThisFileDirectory)\TestFramework\PlaybackTransport.cs" />
     <Compile Remove="$(MSBuildThisFileDirectory)\TestFramework\*Record*.cs" />
   </ItemGroup>

sdk/core/Azure.Core/tests/TestFramework/LiveOnlyAttribute.cs

@@ -0,0 +1,36 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System;
+using NUnit.Framework;
+using NUnit.Framework.Interfaces;
+using NUnit.Framework.Internal;
+
+namespace Azure.Core.Testing
+{
+    /// <summary>
+    /// Attribute on test assemblies, classes, or methods that run only against live resources.
+    /// </summary>
+    [AttributeUsage(AttributeTargets.Method | AttributeTargets.Class | AttributeTargets.Assembly, AllowMultiple = true, Inherited = true)]
+    public class LiveOnlyAttribute : NUnitAttribute, IApplyToTest
+    {
+        /// <summary>
+        /// Modifies the <paramref name="test"/> by adding categories to it and changing the run state as needed.
+        /// </summary>
+        /// <param name="test">The <see cref="Test"/> to modify.</param>
+        public void ApplyToTest(Test test)
+        {
+            test.Properties.Add("Category", "Live");
+
+            if (test.RunState != RunState.NotRunnable)
+            {
+                RecordedTestMode mode = RecordedTestUtilities.GetModeFromEnvironment();
+                if (mode != RecordedTestMode.Live)
+                {
+                    test.RunState = RunState.Ignored;
+                    test.Properties.Set("_SKIPREASON", $"Live tests will not run when AZURE_TEST_MODE is {mode}");
+                }
+            }
+        }
+    }
+}

sdk/core/Azure.Core/tests/TestFramework/RecordedTestBase.cs

@@ -1,7 +1,6 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
-using System;
 using System.IO;
 using NUnit.Framework;
 
@@ -10,8 +9,6 @@ namespace Azure.Core.Testing
     [Category("Recorded")]
     public abstract class RecordedTestBase: ClientTestBase
     {
-        private const string ModeEnvironmentVariableName = "AZURE_TEST_MODE";
-
         protected RecordedTestSanitizer Sanitizer { get; set; }
 
         protected RecordMatcher Matcher { get; set; }
@@ -20,7 +17,7 @@ public abstract class RecordedTestBase: ClientTestBase
 
         protected RecordedTestMode Mode { get; }
 
-        protected RecordedTestBase(bool isAsync) : this(isAsync, GetModeFromEnvironment())
+        protected RecordedTestBase(bool isAsync) : this(isAsync, RecordedTestUtilities.GetModeFromEnvironment())
         {
         }
 
@@ -31,19 +28,6 @@ protected RecordedTestBase(bool isAsync, RecordedTestMode mode) : base(isAsync)
             Mode = mode;
         }
 
-        internal static RecordedTestMode GetModeFromEnvironment()
-        {
-            string modeString = Environment.GetEnvironmentVariable(ModeEnvironmentVariableName);
-
-            RecordedTestMode mode = RecordedTestMode.Playback;
-            if (!string.IsNullOrEmpty(modeString))
-            {
-                mode = (RecordedTestMode)Enum.Parse(typeof(RecordedTestMode), modeString, true);
-            }
-
-            return mode;
-        }
-
         private string GetSessionFilePath(string name = null)
         {
             TestContext.TestAdapter testAdapter = TestContext.CurrentContext.Test;

sdk/core/Azure.Core/tests/TestFramework/RecordedTestUtilities.cs

@@ -0,0 +1,26 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System;
+
+namespace Azure.Core.Testing
+{
+    internal static class RecordedTestUtilities
+    {
+        private const string ModeEnvironmentVariableName = "AZURE_TEST_MODE";
+
+        internal static RecordedTestMode GetModeFromEnvironment()
+        {
+            string modeString = Environment.GetEnvironmentVariable(ModeEnvironmentVariableName);
+
+            RecordedTestMode mode = RecordedTestMode.Playback;
+            if (!string.IsNullOrEmpty(modeString))
+            {
+                mode = (RecordedTestMode)Enum.Parse(typeof(RecordedTestMode), modeString, true);
+            }
+
+            return mode;
+        }
+
+    }
+}

sdk/core/Azure.Core/tests/samples/Sample1_HelloWorld.cs

@@ -3,14 +3,15 @@
 // license information.
 
 using Azure.Core.Pipeline;
+using Azure.Core.Testing;
 using NUnit.Framework;
 using System;
 using System.IO;
 using System.Threading.Tasks;
 
 namespace Azure.Core.Samples
 {
-    [Category("Live")]
+    [LiveOnly]
     public partial class BaseSamples
     {
         [Test]

sdk/keyvault/Azure.Security.KeyVault.Keys/samples/Azure.Security.KeyVault.Keys.Samples.csproj

@@ -2,6 +2,7 @@
   <PropertyGroup>
     <TargetFrameworks>$(RequiredTargetFrameworks)</TargetFrameworks>
   </PropertyGroup>
+  <Import Project="..\..\..\core\Azure.Core\tests\LiveOnly.props" />
   <ItemGroup>
     <PackageReference Include="nunit" />
     <PackageReference Include="NUnit3TestAdapter" />

sdk/keyvault/Azure.Security.KeyVault.Keys/samples/Sample1_HelloWorld.cs

@@ -2,19 +2,19 @@
 // Licensed under the MIT License. See License.txt in the project root for
 // license information.
 
+using Azure.Core.Testing;
 using Azure.Identity;
 using NUnit.Framework;
 using System;
 using System.Diagnostics;
-using System.Security.Cryptography;
 using System.Threading;
 
 namespace Azure.Security.KeyVault.Keys.Samples
 {
     /// <summary>
     /// Sample demonstrates how to set, get, update and delete a key using the synchronous methods of the KeyClient.
     /// </summary>
-    [Category("Live")]
+    [LiveOnly]
     public partial class HelloWorld
     {
         [Test]

sdk/keyvault/Azure.Security.KeyVault.Keys/samples/Sample1_HelloWorldAsync.cs

@@ -2,24 +2,23 @@
 // Licensed under the MIT License. See License.txt in the project root for
 // license information.
 
+using Azure.Core.Testing;
 using Azure.Identity;
 using NUnit.Framework;
 using System;
 using System.Diagnostics;
-using System.Security.Cryptography;
-using System.Threading;
 using System.Threading.Tasks;
 
 namespace Azure.Security.KeyVault.Keys.Samples
 {
     /// <summary>
     /// Sample demonstrates how to set, get, update and delete a key using the asynchronous methods of the KeyClient.
     /// </summary>
-    [Category("Live")]
+    [LiveOnly]
     public partial class HelloWorld
     {
         [Test]
-        public async Task HelloWorldASync()
+        public async Task HelloWorldAsync()
         {
             // Environment variable with the Key Vault endpoint.
             string keyVaultUrl = Environment.GetEnvironmentVariable("AZURE_KEYVAULT_URL");

sdk/keyvault/Azure.Security.KeyVault.Keys/samples/Sample2_BackupAndRestore.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT License. See License.txt in the project root for
 // license information.
 
+using Azure.Core.Testing;
 using Azure.Identity;
 using NUnit.Framework;
 using System;
@@ -14,7 +15,7 @@ namespace Azure.Security.KeyVault.Keys.Samples
     /// Sample demonstrates how to backup and restore keys in the Key Vault
     /// using the synchronous methods of the KeyClient.
     /// </summary>
-    [Category("Live")]
+    [LiveOnly]
     public partial class BackupAndRestore
     {
         [Test]

sdk/keyvault/Azure.Security.KeyVault.Keys/samples/Sample2_BackupAndRestoreAsync.cs

@@ -2,11 +2,11 @@
 // Licensed under the MIT License. See License.txt in the project root for
 // license information.
 
+using Azure.Core.Testing;
 using Azure.Identity;
 using NUnit.Framework;
 using System;
 using System.IO;
-using System.Threading;
 using System.Threading.Tasks;
 
 namespace Azure.Security.KeyVault.Keys.Samples
@@ -15,7 +15,7 @@ namespace Azure.Security.KeyVault.Keys.Samples
     /// Sample demonstrates how to backup and restore keys in the Key Vault
     /// using the asynchronous methods of the KeyClient.
     /// </summary>
-    [Category("Live")]
+    [LiveOnly]
     public partial class BackupAndRestore
     {
         [Test]

sdk/keyvault/Azure.Security.KeyVault.Keys/samples/Sample3_GetKeys.cs

@@ -2,12 +2,12 @@
 // Licensed under the MIT License. See License.txt in the project root for
 // license information.
 
+using Azure.Core.Testing;
 using Azure.Identity;
 using NUnit.Framework;
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
-using System.Security.Cryptography;
 using System.Threading;
 
 namespace Azure.Security.KeyVault.Keys.Samples
@@ -17,7 +17,7 @@ namespace Azure.Security.KeyVault.Keys.Samples
     /// and list deleted keys in a soft-delete enabled Key Vault
     /// using the synchronous methods of the KeyClient.
     /// </summary>
-    [Category("Live")]
+    [LiveOnly]
     public partial class GetKeys
     {
         [Test]

sdk/keyvault/Azure.Security.KeyVault.Keys/samples/Sample3_GetKeysAsync.cs

@@ -2,12 +2,11 @@
 // Licensed under the MIT License. See License.txt in the project root for
 // license information.
 
+using Azure.Core.Testing;
 using Azure.Identity;
 using NUnit.Framework;
 using System;
-using System.Collections.Generic;
 using System.Diagnostics;
-using System.Threading;
 using System.Threading.Tasks;
 
 namespace Azure.Security.KeyVault.Keys.Samples
@@ -17,7 +16,7 @@ namespace Azure.Security.KeyVault.Keys.Samples
     /// and list deleted keys in a soft-delete enabled Key Vault
     /// using the asynchronous methods of the KeyClient.
     /// </summary>
-    [Category("Live")]
+    [LiveOnly]
     public partial class GetKeys
     {
         [Test]

sdk/keyvault/Azure.Security.KeyVault.Keys/samples/Sample4_EncryptDecrypt.cs

@@ -1,5 +1,5 @@
-using Azure.Identity;
-using Azure.Security.KeyVault.Keys;
+using Azure.Core.Testing;
+using Azure.Identity;
 using Azure.Security.KeyVault.Keys.Cryptography;
 using NUnit.Framework;
 using System;
@@ -13,7 +13,7 @@ namespace Azure.Security.KeyVault.Keys.Samples
     /// <summary>
     /// Sample demonstrates how to encrypt and decrypt a single block of plain text with an RSA key using the synchronous methods of the CryptographyClient.
     /// </summary>
-    [Category("Live")]
+    [LiveOnly]
     public partial class Sample4_EncryptDecypt
     {
         [Test]

sdk/keyvault/Azure.Security.KeyVault.Keys/samples/Sample4_EncryptDecryptAsync.cs

@@ -1,8 +1,8 @@
-using Azure.Identity;
+using Azure.Core.Testing;
+using Azure.Identity;
 using Azure.Security.KeyVault.Keys.Cryptography;
 using NUnit.Framework;
 using System;
-using System.Collections.Generic;
 using System.Diagnostics;
 using System.Text;
 using System.Threading.Tasks;
@@ -13,7 +13,7 @@ namespace Azure.Security.KeyVault.Keys.Samples
     /// <summary>
     /// Sample demonstrates how to encrypt and decrypt a single block of plain text with an RSA key using the asynchronous methods of the CryptographyClient.
     /// </summary>
-    [Category("Live")]
+    [LiveOnly]
     public partial class Sample4_EncryptDecypt
     {
         [Test]

sdk/keyvault/Azure.Security.KeyVault.Keys/samples/Sample5_SignVerify.cs

@@ -1,5 +1,5 @@
-using Azure.Identity;
-using Azure.Security.KeyVault.Keys;
+using Azure.Core.Testing;
+using Azure.Identity;
 using Azure.Security.KeyVault.Keys.Cryptography;
 using NUnit.Framework;
 using System;
@@ -14,7 +14,7 @@ namespace Azure.Security.KeyVault.Keys.Samples
     /// <summary>
     /// Sample demonstrates how to sign data with both a RSA key and an EC key using the synchronous methods of the CryptographyClient.
     /// </summary>
-    [Category("Live")]
+    [LiveOnly]
     public partial class Sample5_SignVerify
     {
         [Test]

sdk/keyvault/Azure.Security.KeyVault.Keys/samples/Sample5_SignVerifyAsync.cs

@@ -1,12 +1,11 @@
-using Azure.Identity;
-using Azure.Security.KeyVault.Keys;
+using Azure.Core.Testing;
+using Azure.Identity;
 using Azure.Security.KeyVault.Keys.Cryptography;
 using NUnit.Framework;
 using System;
 using System.Diagnostics;
 using System.Security.Cryptography;
 using System.Text;
-using System.Threading;
 using System.Threading.Tasks;
 
 namespace Azure.Security.KeyVault.Keys.Samples
@@ -15,7 +14,7 @@ namespace Azure.Security.KeyVault.Keys.Samples
     /// <summary>
     /// Sample demonstrates how to sign data with both a RSA key and an EC key using the synchronous methods of the CryptographyClient.
     /// </summary>
-    [Category("Live")]
+    [LiveOnly]
     public partial class Sample5_SignVerify
     {
         [Test]

sdk/keyvault/Azure.Security.KeyVault.Keys/samples/Sample6_WrapUnwrap.cs

@@ -1,5 +1,5 @@
-using Azure.Identity;
-using Azure.Security.KeyVault.Keys;
+using Azure.Core.Testing;
+using Azure.Identity;
 using Azure.Security.KeyVault.Keys.Cryptography;
 using NUnit.Framework;
 using System;
@@ -13,7 +13,7 @@ namespace Azure.Security.KeyVault.Keys.Samples
     /// <summary>
     /// Sample demonstrates how to wrap and unwrap a symmetric key with an RSA key using the synchronous methods of the CryptographyClient.
     /// </summary>
-    [Category("Live")]
+    [LiveOnly]
     public partial class Sample6_WrapUnwrap
     {
         [Test]