Introduce BinaryFormatter enablement switch; disable on wasm (#38963)

Applications are given the option of disabling BinaryFormatter.Serialize and BinaryFormatter.Deserialize, meaning they'll throw NotSupportedException if anybody tries calling them. WASM projects always disable these APIs via a hardcoded mechanism; there is no re-enablement mechanism for those projects.

This commit only introduces the switch. It does not disable it in any project types by default. (Exception: wasm, as mentioned above.) The related PR dotnet/sdk#12434 causes aspnet 5.0+ applications to disable BinaryFormatter functionality by default, but apps can re-enable if needed.

docs/workflow/trimming/feature-switches.md

@@ -9,7 +9,8 @@ configurations but their defaults might vary as any SDK can set the defaults dif
 | MSBuild Property Name | AppContext Setting | Description |
 |-|-|-|
 | DebuggerSupport | System.Diagnostics.Debugger.IsSupported | Any dependency that enables better debugging experience to be trimmed when set to false |
-| EnableUnsafeUTF7Encoding | System.Text.Encoding.EnableUnsafeUTF7Encoding |  Insecure UTF-7 encoding is trimmed when set to false |
+| EnableUnsafeUTF7Encoding | System.Text.Encoding.EnableUnsafeUTF7Encoding | Insecure UTF-7 encoding is trimmed when set to false |
+| EnableUnsafeBinaryFormatterSerialization | System.Runtime.Serialization.EnableUnsafeBinaryFormatterSerialization | BinaryFormatter serialization support is trimmed when set to false |
 | EventSourceSupport | System.Diagnostics.Tracing.EventSource.IsSupported | Any EventSource related code or logic is trimmed when set to false |
 | InvariantGlobalization | System.Globalization.Invariant | All globalization specific code and data is trimmed when set to true |
 | UseSystemResourceKeys | System.Resources.UseSystemResourceKeys |  Any localizable resources for system assemblies is trimmed when set to true |

src/libraries/Common/src/System/LocalAppContextSwitches.Common.cs

@@ -51,6 +51,11 @@ private static bool GetSwitchDefaultValue(string switchName)
                 return true;
             }
 
+            if (switchName == "System.Runtime.Serialization.EnableUnsafeBinaryFormatterSerialization")
+            {
+                return true;
+            }
+
             return false;
         }
     }

src/libraries/Common/tests/System/Collections/IEnumerable.Generic.Serialization.Tests.cs

@@ -9,7 +9,7 @@ namespace System.Collections.Tests
 {
     public abstract partial class IEnumerable_Generic_Tests<T> : TestBase<T>
     {
-        [Theory]
+        [ConditionalTheory(typeof(PlatformDetection), nameof(PlatformDetection.IsBinaryFormatterSupported))]
         [MemberData(nameof(ValidCollectionSizes))]
         public void IGenericSharedAPI_SerializeDeserialize(int count)
         {

src/libraries/Common/tests/System/Collections/IEnumerable.NonGeneric.Serialization.Tests.cs

@@ -11,7 +11,7 @@ namespace System.Collections.Tests
 {
     public abstract partial class IEnumerable_NonGeneric_Tests : TestBase
     {
-        [Theory]
+        [ConditionalTheory(typeof(PlatformDetection), nameof(PlatformDetection.IsBinaryFormatterSupported))]
         [MemberData(nameof(ValidCollectionSizes))]
         public void IGenericSharedAPI_SerializeDeserialize(int count)
         {

src/libraries/System.Collections/tests/Generic/Comparers/EqualityComparer.Generic.Serialization.Tests.cs

@@ -11,7 +11,7 @@ namespace System.Collections.Generic.Tests
 {
     public abstract partial class ComparersGenericTests<T>
     {
-        [Fact]
+        [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsBinaryFormatterSupported))]
         public void EqualityComparer_SerializationRoundtrip()
         {
             var bf = new BinaryFormatter();

src/libraries/System.Collections/tests/Generic/Dictionary/Dictionary.Tests.cs

@@ -399,7 +399,7 @@ private static IDictionary<T, T> CreateDictionary<T>(int size, Func<int, T> keyV
             return dict;
         }
 
-        [Fact]
+        [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsBinaryFormatterSupported))]
         public void ComparerSerialization()
         {
             // Strings switch between randomized and non-randomized comparers,

src/libraries/System.Collections/tests/Generic/HashSet/HashSet.Generic.Tests.cs

@@ -658,7 +658,7 @@ public void Remove_NonDefaultComparer_ComparerUsed(int capacity)
 
         #region Serialization
 
-        [Fact]
+        [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsBinaryFormatterSupported))]
         public void ComparerSerialization()
         {
             // Strings switch between randomized and non-randomized comparers,

src/libraries/System.ComponentModel.Annotations/tests/System/ComponentModel/DataAnnotations/ValidationExceptionTests.cs

@@ -80,7 +80,7 @@ public static void Ctor_String_ValidationAttribute_Object(string errorMessage)
             Assert.Same(value, ex.Value);
         }
 
-        [Fact]
+        [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsBinaryFormatterSupported))]
         public void Ctor_SerializationInfo_StreamingContext()
         {
             using (var stream = new MemoryStream())

src/libraries/System.ComponentModel.Primitives/tests/System/ComponentModel/InvalidAsynchronousStateExceptionTests.cs

@@ -42,7 +42,7 @@ public void Ctor_String_Exception(string message)
             Assert.Null(exception.ParamName);
         }
 
-        [Fact]
+        [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsBinaryFormatterSupported))]
         public void Ctor_SerializationInfo_StreamingContext()
         {
             using (var stream = new MemoryStream())

src/libraries/System.ComponentModel.Primitives/tests/System/ComponentModel/InvalidEnumArgumentExceptionTests.cs

@@ -87,7 +87,7 @@ public void Ctor_NullEnumClass_ThrowsArgumentNulException()
             AssertExtensions.Throws<ArgumentNullException, NullReferenceException>("enumClass", () => new InvalidEnumArgumentException("argumentName", 1, null));
         }
 
-        [Fact]
+        [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsBinaryFormatterSupported))]
         public void Ctor_SerializationInfo_StreamingContext()
         {
             using (var stream = new MemoryStream())

src/libraries/System.ComponentModel.TypeConverter/tests/Design/CheckoutExceptionTests.cs

@@ -60,7 +60,7 @@ public void Cancelled_Get_ReturnsExpected()
             Assert.Null(exception.InnerException);
         }
 
-        [Fact]
+        [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsBinaryFormatterSupported))]
         public void Ctor_SerializationInfo_StreamingContext()
         {
             using (var stream = new MemoryStream())

src/libraries/System.ComponentModel.TypeConverter/tests/LicenseExceptionTests.cs

@@ -73,7 +73,7 @@ public void Ctor_Type_Object_String_Exception(Type type, object instance, string
             Assert.Equal(message, exception.Message);
         }
 
-        [Fact]
+        [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsBinaryFormatterSupported))]
         public void Ctor_SerializationInfo_StreamingContext()
         {
             using (var stream = new MemoryStream())

src/libraries/System.ComponentModel.TypeConverter/tests/WarningExceptionTests.cs

@@ -74,7 +74,7 @@ public void Ctor_String_String_String(string message, string helpUrl, string hel
             Assert.Equal(message, exception.Message);
         }
 
-        [Fact]
+        [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsBinaryFormatterSupported))]
         public void Ctor_SerializationInfo_StreamingContext()
         {
             using (var stream = new MemoryStream())

src/libraries/System.Private.CoreLib/src/ILLink/ILLink.Substitutions.Shared.xml

@@ -12,5 +12,8 @@
     <type fullname="System.LocalAppContextSwitches">
       <method signature="System.Boolean get_EnableUnsafeUTF7Encoding()" body="stub" value="false" feature="System.Text.Encoding.EnableUnsafeUTF7Encoding" featurevalue="false" />
     </type>
+    <type fullname="System.Resources.ResourceReader">
+      <method signature="System.Boolean InitializeBinaryFormatter()" body="stub" value="false" feature="System.Runtime.Serialization.EnableUnsafeBinaryFormatterSerialization" featurevalue="false" />
+    </type>
   </assembly>
 </linker>

src/libraries/System.Private.CoreLib/src/Resources/Strings.resx

@@ -3757,4 +3757,7 @@
   <data name="Arg_MustBeHalf" xml:space="preserve">
     <value>Object must be of type Half.</value>
   </data>
+  <data name="BinaryFormatter_SerializationDisallowed" xml:space="preserve">
+    <value>BinaryFormatter serialization and deserialization are disabled within this application. See https://aka.ms/binaryformatter for more information.</value>
+  </data>
 </root>

src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems

@@ -819,6 +819,7 @@
     <Compile Include="$(MSBuildThisFileDirectory)System\Runtime\Serialization\SafeSerializationEventArgs.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Runtime\Serialization\SerializationException.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Runtime\Serialization\SerializationInfo.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)System\Runtime\Serialization\SerializationInfo.SerializationGuard.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Runtime\Serialization\SerializationInfoEnumerator.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Runtime\Serialization\StreamingContext.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Runtime\Versioning\ComponentGuaranteesAttribute.cs" />

src/libraries/System.Private.CoreLib/src/System/Resources/ResourceReader.Core.cs

@@ -5,6 +5,7 @@
 using System.Diagnostics;
 using System.IO;
 using System.Reflection;
+using System.Runtime.Serialization;
 using System.Text;
 using System.Threading;
 
@@ -50,7 +51,14 @@ private object DeserializeObject(int typeIndex)
 
             if (_binaryFormatter == null)
             {
-                InitializeBinaryFormatter();
+                if (!InitializeBinaryFormatter())
+                {
+                    // The linker trimmed away the BinaryFormatter implementation and we can't call into it.
+                    // We'll throw an exception with the same text that BinaryFormatter would have thrown
+                    // had we been able to call into it. Keep this resource string in sync with the same
+                    // resource from the Formatters assembly.
+                    throw new NotSupportedException(SR.BinaryFormatter_SerializationDisallowed);
+                }
             }
 
             Type type = FindType(typeIndex);
@@ -65,8 +73,11 @@ private object DeserializeObject(int typeIndex)
         }
 
         // Issue https://github.com/dotnet/runtime/issues/39290 tracks finding an alternative to BinaryFormatter
-        private void InitializeBinaryFormatter()
+        private bool InitializeBinaryFormatter()
         {
+            // If BinaryFormatter support is disabled for the app, the linker will replace this entire
+            // method body with "return false;", skipping all reflection code below.
+
             LazyInitializer.EnsureInitialized(ref s_binaryFormatterType, () =>
                 Type.GetType("System.Runtime.Serialization.Formatters.Binary.BinaryFormatter, System.Runtime.Serialization.Formatters, Version=0.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a",
                 throwOnError: true)!);
@@ -83,6 +94,8 @@ private void InitializeBinaryFormatter()
             });
 
             _binaryFormatter = Activator.CreateInstance(s_binaryFormatterType!)!;
+
+            return true; // initialization successful
         }
 
         // generic method that we specialize at runtime once we've loaded the BinaryFormatter type

src/libraries/System.Private.CoreLib/src/System/Runtime/Serialization/SerializationInfo.SerializationGuard.cs

@@ -0,0 +1,139 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Security;
+using System.Threading;
+
+namespace System.Runtime.Serialization
+{
+    /// <summary>The structure for holding all of the data needed for object serialization and deserialization.</summary>
+    public sealed partial class SerializationInfo
+    {
+        internal static AsyncLocal<bool> AsyncDeserializationInProgress { get; } = new AsyncLocal<bool>();
+
+#if !CORECLR
+        // On AoT, assume private members are reflection blocked, so there's no further protection required
+        // for the thread's DeserializationTracker
+        [ThreadStatic]
+        private static DeserializationTracker? t_deserializationTracker;
+
+        private static DeserializationTracker GetThreadDeserializationTracker() =>
+            t_deserializationTracker ??= new DeserializationTracker();
+#endif // !CORECLR
+
+        // Returns true if deserialization is currently in progress
+        public static bool DeserializationInProgress
+        {
+#if CORECLR
+            [DynamicSecurityMethod] // Methods containing StackCrawlMark local var must be marked DynamicSecurityMethod
+#endif
+            get
+            {
+                if (AsyncDeserializationInProgress.Value)
+                {
+                    return true;
+                }
+
+#if CORECLR
+                StackCrawlMark stackMark = StackCrawlMark.LookForMe;
+                DeserializationTracker tracker = Thread.GetThreadDeserializationTracker(ref stackMark);
+#else
+                DeserializationTracker tracker = GetThreadDeserializationTracker();
+#endif
+                bool result = tracker.DeserializationInProgress;
+                return result;
+            }
+        }
+
+        // Throws a SerializationException if dangerous deserialization is currently
+        // in progress
+        public static void ThrowIfDeserializationInProgress()
+        {
+            if (DeserializationInProgress)
+            {
+                throw new SerializationException(SR.Serialization_DangerousDeserialization);
+            }
+        }
+
+        // Throws a DeserializationBlockedException if dangerous deserialization is currently
+        // in progress and the AppContext switch Switch.System.Runtime.Serialization.SerializationGuard.{switchSuffix}
+        // is not true. The value of the switch is cached in cachedValue to avoid repeated lookups:
+        // 0: No value cached
+        // 1: The switch is true
+        // -1: The switch is false
+        public static void ThrowIfDeserializationInProgress(string switchSuffix, ref int cachedValue)
+        {
+            const string SwitchPrefix = "Switch.System.Runtime.Serialization.SerializationGuard.";
+            if (switchSuffix == null)
+            {
+                throw new ArgumentNullException(nameof(switchSuffix));
+            }
+            if (string.IsNullOrWhiteSpace(switchSuffix))
+            {
+                throw new ArgumentException(SR.Argument_EmptyName, nameof(switchSuffix));
+            }
+
+            if (cachedValue == 0)
+            {
+                if (AppContext.TryGetSwitch(SwitchPrefix + switchSuffix, out bool isEnabled) && isEnabled)
+                {
+                    cachedValue = 1;
+                }
+                else
+                {
+                    cachedValue = -1;
+                }
+            }
+
+            if (cachedValue == 1)
+            {
+                return;
+            }
+            else if (cachedValue == -1)
+            {
+                if (DeserializationInProgress)
+                {
+                    throw new SerializationException(SR.Format(SR.Serialization_DangerousDeserialization_Switch, SwitchPrefix + switchSuffix));
+                }
+            }
+            else
+            {
+                throw new ArgumentOutOfRangeException(nameof(cachedValue));
+            }
+        }
+
+        // Declares that the current thread and async context have begun deserialization.
+        // In this state, if the SerializationGuard or other related AppContext switches are set,
+        // actions likely to be dangerous during deserialization, such as starting a process will be blocked.
+        // Returns a DeserializationToken that must be disposed to remove the deserialization state.
+#if CORECLR
+        [DynamicSecurityMethod] // Methods containing StackCrawlMark local var must be marked DynamicSecurityMethod
+#endif
+        public static DeserializationToken StartDeserialization()
+        {
+            if (LocalAppContextSwitches.SerializationGuard)
+            {
+#if CORECLR
+                StackCrawlMark stackMark = StackCrawlMark.LookForMe;
+                DeserializationTracker tracker = Thread.GetThreadDeserializationTracker(ref stackMark);
+#else
+                DeserializationTracker tracker = GetThreadDeserializationTracker();
+#endif
+                if (!tracker.DeserializationInProgress)
+                {
+                    lock (tracker)
+                    {
+                        if (!tracker.DeserializationInProgress)
+                        {
+                            AsyncDeserializationInProgress.Value = true;
+                            tracker.DeserializationInProgress = true;
+                            return new DeserializationToken(tracker);
+                        }
+                    }
+                }
+            }
+
+            return new DeserializationToken(null);
+        }
+    }
+}