[mtouch] add mixed-mode support (#4751)

* [mtouch] add --interp-mixed option

When enabling this option, mtouch will AOT compile `mscorlib.dll`.  At
runtime that means every method that wasn't AOT'd will be executed by
the runtime interpreter.

* [mtouch] Add support to --interpreter to list the assemblies to (not) interpret.

* [msbuild] Simplify interpreter code to use a single variable.

* Fix whitespace.

* [mtouch] Move mtouch-specific code to mtouch-specific file.

* [msbuild] An empty string is a valid value for 'Interpreter', so make it a non-required property.

* [mtouch] Add sanity check for aot-compiling interpreted assemblies.

msbuild/Xamarin.iOS.Tasks.Core/Tasks/MTouchTaskBase.cs

@@ -106,8 +106,7 @@ public GccOptions ()
 		[Required]
 		public bool EnableSGenConc { get; set; }
 
-		[Required]
-		public bool UseInterpreter { get; set; }
+		public string Interpreter { get; set; }
 
 		[Required]
 		public bool LinkerDumpDependencies { get; set; }
@@ -405,8 +404,8 @@ protected override string GenerateCommandLineCommands ()
 			if (EnableSGenConc)
 				args.AddLine ("--sgen-conc");
 
-			if (UseInterpreter)
-				args.Add ("--interpreter");
+			if (!string.IsNullOrEmpty (Interpreter))
+				args.Add ($"--interpreter={Interpreter}");
 
 			switch (LinkMode.ToLowerInvariant ()) {
 			case "sdkonly": args.AddLine ("--linksdkonly"); break;

msbuild/Xamarin.iOS.Tasks.Core/Xamarin.iOS.Common.props

@@ -51,7 +51,6 @@ Copyright (C) 2013-2016 Xamarin. All rights reserved.
 		<MtouchUseThumb Condition="'$(MtouchUseThumb)' == ''">False</MtouchUseThumb>
 		<MtouchProjectDirectory>$(MSBuildProjectDirectory)</MtouchProjectDirectory>
 		<MtouchEnableSGenConc Condition="'$(MtouchEnableSGenConc)' == ''">False</MtouchEnableSGenConc>
-		<MtouchUseInterpreter Condition="'$(MtouchUseInterpreter)' == ''">False</MtouchUseInterpreter>
 		<MtouchVerbosity Condition="$(MtouchVerbosity) == ''">2</MtouchVerbosity>
 
 		<IsMacEnabled>true</IsMacEnabled>

msbuild/Xamarin.iOS.Tasks.Core/Xamarin.iOS.Common.targets

@@ -831,7 +831,7 @@ Copyright (C) 2013-2016 Xamarin. All rights reserved.
 			UseThumb="$(MtouchUseThumb)"
 			EnableBitcode="$(MtouchEnableBitcode)"
 			EnableSGenConc="$(MtouchEnableSGenConc)"
-			UseInterpreter="$(MtouchUseInterpreter)"
+			Interpreter="$(MtouchInterpreter)"
 			AppExtensionReferences="@(_ResolvedAppExtensionReferences)"
 			ArchiveSymbols="$(MonoSymbolArchive)"
 			Verbosity="$(MtouchVerbosity)"

tests/common/BundlerTool.cs

@@ -74,6 +74,7 @@ abstract class BundlerTool : Tool
 		public int Verbosity;
 		public int [] WarnAsError; // null array: nothing passed to mtouch/mmp. empty array: pass --warnaserror (which means makes all warnings errors).
 		public string [] XmlDefinitions;
+		public string Interpreter;
 
 		// These are a bit smarter
 		public bool NoPlatformAssemblyReference;
@@ -291,7 +292,12 @@ protected virtual void BuildArguments (StringBuilder sb)
 					sb.Append (" --xml:").Append (StringUtils.Quote (xd));
 			}
 
-
+			if (Interpreter != null) {
+				if (Interpreter.Length == 0)
+					sb.Append (" --interpreter");
+				else
+					sb.Append (" --interpreter=").Append (Interpreter);
+			}
 		}
 
 		public string CreateTemporaryDirectory ()

tests/mtouch/MTouch.cs

@@ -1780,6 +1780,25 @@ public class B
 			}
 		}
 
+		[Test]
+		public void MT0138 ()
+		{
+			using (var mtouch = new MTouchTool ()) {
+				var tmpdir = mtouch.CreateTemporaryDirectory ();
+				mtouch.CreateTemporaryCacheDirectory ();
+
+				mtouch.CreateTemporaryApp ();
+				mtouch.WarnAsError = new int [] { 138 }; // This is just to make mtouch bail out early instead of spending time building the app when that's not what we're interested in.
+				mtouch.Interpreter = "all,-all,foo,-bar,mscorlib.dll,mscorlib";
+				mtouch.AssertExecuteFailure (MTouchAction.BuildSim, "build");
+				mtouch.AssertError (138, "Cannot find the assembly 'foo', passed as an argument to --interpreter.");
+				mtouch.AssertError (138, "Cannot find the assembly 'bar', passed as an argument to --interpreter.");
+				mtouch.AssertError (138, "Cannot find the assembly 'mscorlib.dll', passed as an argument to --interpreter.");
+				// just the name, without the extension, is the right way.
+				mtouch.AssertErrorCount (3);
+			}
+		}
+
 		[Test]
 		[TestCase ("all")]
 		[TestCase ("-all")]

tests/xharness/Jenkins.cs

@@ -182,12 +182,15 @@ IEnumerable<TestData> GetTestData (RunTestTask test)
 					yield return new TestData { Variation = "Release (all optimizations)", MTouchExtraArgs = "--registrar:static --optimize:all", Debug = false, Profiling = false, Defines = "OPTIMIZEALL" };
 					yield return new TestData { Variation = "Debug (all optimizations)", MTouchExtraArgs = "--registrar:static --optimize:all", Debug = true, Profiling = false, Defines = "OPTIMIZEALL" };
 					yield return new TestData { Variation = "Debug (interpreter)", MTouchExtraArgs = "--interpreter", Debug = true, Profiling = false, Ignored = true, };
+					yield return new TestData { Variation = "Debug (interpreter -mscorlib)", MTouchExtraArgs = "--interpreter=-mscorlib", Debug = true, Profiling = false, Ignored = true, };
 					break;
 				case "mscorlib":
 					yield return new TestData { Variation = "Debug (interpreter)", MTouchExtraArgs = "--interpreter", Debug = true, Profiling = false, Ignored = true, Undefines = "FULL_AOT_RUNTIME" };
+					yield return new TestData { Variation = "Debug (interpreter -mscorlib)", MTouchExtraArgs = "--interpreter=-mscorlib", Debug = true, Profiling = false, Ignored = true, Undefines = "FULL_AOT_RUNTIME" };
 					break;
 				case "mini":
 					yield return new TestData { Variation = "Debug (interpreter)", MTouchExtraArgs = "--interpreter", Debug = true, Profiling = false, Undefines = "FULL_AOT_RUNTIME" };
+					yield return new TestData { Variation = "Debug (interpreter -mscorlib)", MTouchExtraArgs = "--interpreter=-mscorlib", Debug = true, Profiling = false, Undefines = "FULL_AOT_RUNTIME" };
 					break;
 				}
 				break;

tools/common/Application.cs

@@ -63,7 +63,6 @@ public partial class Application
 
 		public bool? EnableCoopGC;
 		public bool EnableSGenConc;
-		public bool UseInterpreter;
 		public MarshalObjectiveCExceptionMode MarshalObjectiveCExceptions;
 		public MarshalManagedExceptionMode MarshalManagedExceptions;
 

tools/mtouch/Application.cs

@@ -116,6 +116,8 @@ public bool EnableMSym {
 		public string AotOtherArguments = string.Empty;
 		public bool? LLVMAsmWriter;
 		public Dictionary<string, string> LLVMOptimizations = new Dictionary<string, string> ();
+		public bool UseInterpreter;
+		public List<string> InterpretedAssemblies = new List<string> ();
 
 		public Dictionary<string, string> EnvironmentVariables = new Dictionary<string, string> ();
 
@@ -129,6 +131,44 @@ public bool EnableMSym {
 
 		public bool? BuildDSym;
 
+		public bool IsInterpreted (string assembly)
+		{
+			// IsAOTCompiled and IsInterpreted are not opposites: mscorlib.dll can be both.
+			if (!UseInterpreter)
+				return false;
+
+			// Go through the list of assemblies to interpret in reverse order,
+			// so that the last option passed to mtouch takes precedence.
+			for (int i = InterpretedAssemblies.Count - 1; i >= 0; i--) {
+				var opt = InterpretedAssemblies [i];
+				if (opt == "all")
+					return true;
+				else if (opt == "-all")
+					return false;
+				else if (opt == assembly)
+					return true;
+				else if (opt [0] == '-' && opt.Substring (1) == assembly)
+					return false;
+			}
+
+			// There's an implicit 'all' at the start of the list.
+			return true;
+		}
+
+		public bool IsAOTCompiled (string assembly)
+		{
+			if (!UseInterpreter)
+				return true;
+
+			// IsAOTCompiled and IsInterpreted are not opposites: mscorlib.dll can be both:
+			// - mscorlib will always be processed by the AOT compiler to generate required wrapper functions for the interpreter to work
+			// - mscorlib might also be fully AOT-compiled (both when the interpreter is enabled and when it's not)
+			if (assembly == "mscorlib")
+				return true;
+
+			return !IsInterpreted (assembly);
+		}
+
 		// If we're targetting a 32 bit arch.
 		bool? is32bits;
 		public bool Is32Build {
@@ -462,7 +502,7 @@ public bool UseDlsym (string assembly)
 			if (EnableLLVMOnlyBitCode)
 				return false;
 
-			if (UseInterpreter)
+			if (IsInterpreted (Assembly.GetIdentity (assembly)))
 				return true;
 
 			switch (Platform) {
@@ -2124,7 +2164,20 @@ public void StripNativeCode ()
 
 		public void BundleAssemblies ()
 		{
-			var strip = !UseInterpreter && ManagedStrip && IsDeviceBuild && !EnableDebug && !PackageManagedDebugSymbols;
+			Assembly.StripAssembly strip = ((path) => 
+			{
+				if (!ManagedStrip)
+					return false;
+				if (!IsDeviceBuild)
+					return false;
+				if (EnableDebug)
+					return false;
+				if (PackageManagedDebugSymbols)
+					return false;
+				if (IsInterpreted (Assembly.GetIdentity (path)))
+					return false;
+				return true;
+			});
 
 			var grouped = Targets.SelectMany ((Target t) => t.Assemblies).GroupBy ((Assembly asm) => asm.Identity);
 			foreach (var @group in grouped) {

tools/mtouch/Assembly.cs

@@ -54,11 +54,7 @@ public HashSet<string> DependencyMap {
 
 		public bool IsAOTCompiled {
 			get {
-				if (App.UseInterpreter)
-					/* interpreter only requires a few stubs that are attached
-					 * to mscorlib.dll, other assemblies won't be AOT compiled */
-					return FileName == "mscorlib.dll";
-				return true;
+				return App.IsAOTCompiled (Identity);
 			}
 		}
 
@@ -108,15 +104,18 @@ public void ComputeDependencyMap (List<Exception> exceptions)
 			ComputeDependencies (exceptions);
 		}
 
+		public delegate bool StripAssembly (string path);
+
 		// returns false if the assembly was not copied (because it was already up-to-date).
-		public bool CopyAssembly (string source, string target, bool copy_debug_symbols = true, bool strip = false)
+		public bool CopyAssembly (string source, string target, bool copy_debug_symbols = true, StripAssembly strip = null)
 		{
 			var copied = false;
 
 			try {
-				if (!Application.IsUptodate (source, target) && (strip || !Cache.CompareAssemblies (source, target))) {
+				var strip_assembly = strip != null && strip (source);
+				if (!Application.IsUptodate (source, target) && (strip_assembly || !Cache.CompareAssemblies (source, target))) {
 					copied = true;
-					if (strip) {
+					if (strip_assembly) {
 						Driver.FileDelete (target);
 						Directory.CreateDirectory (Path.GetDirectoryName (target));
 						MonoTouch.Tuner.Stripper.Process (source, target);
@@ -195,7 +194,7 @@ public void CopyConfigToDirectory (string directory)
 		// Aot data is copied separately, because we might want to copy aot data 
 		// even if we don't want to copy the assembly (if 32/64-bit assemblies are identical, 
 		// only one is copied, but we still want the aotdata for both).
-		public void CopyToDirectory (string directory, bool reload = true, bool check_case = false, bool only_copy = false, bool copy_debug_symbols = true, bool strip = false)
+		public void CopyToDirectory (string directory, bool reload = true, bool check_case = false, bool only_copy = false, bool copy_debug_symbols = true, StripAssembly strip = null)
 		{
 			var target = Path.Combine (directory, FileName);
 

tools/mtouch/Target.cs

@@ -283,6 +283,29 @@ public void Initialize (bool show_warnings)
 			}
 
 			linker_flags = new CompilerFlags (this);
+
+			// Verify that there are no entries in our list of intepreted assemblies that doesn't match
+			// any of the assemblies we know about.
+			if (App.UseInterpreter) {
+				var exceptions = new List<Exception> ();
+				foreach (var entry in App.InterpretedAssemblies) {
+					var assembly = entry;
+					if (string.IsNullOrEmpty (assembly))
+						continue;
+
+					if (assembly [0] == '-')
+						assembly = assembly.Substring (1);
+
+					if (assembly == "all")
+						continue;
+
+					if (Assemblies.ContainsKey (assembly))
+						continue;
+
+					exceptions.Add (ErrorHelper.CreateWarning (138, $"Cannot find the assembly '{assembly}', passed as an argument to --interpreter."));
+				}
+				ErrorHelper.ThrowIfErrors (exceptions);
+			}
 		}
 
 		// This is to load the symbols for all assemblies, so that we can give better error messages
@@ -1134,6 +1157,8 @@ void AOTCompile ()
 			}
 
 			if (App.UseInterpreter)
+				/* TODO: not sure? we might have to continue here, depending on
+				 * the set of assemblies are AOT'd? */
 				return;
 
 			// Code in one assembly (either in a P/Invoke or a third-party library) can depend on a third-party library in another assembly.

tools/mtouch/mtouch.cs

@@ -455,7 +455,8 @@ public static string GetAotArguments (Application app, string filename, Abi abi,
 			bool enable_debug = app.EnableDebug;
 			bool enable_debug_symbols = app.PackageManagedDebugSymbols;
 			bool llvm_only = app.EnableLLVMOnlyBitCode;
-			bool interp = app.UseInterpreter;
+			bool interp = app.IsInterpreted (Assembly.GetIdentity (filename));
+			bool interp_full = !interp && app.UseInterpreter && fname == "mscorlib.dll";
 			bool is32bit = (abi & Abi.Arch32Mask) > 0;
 			string arch = abi.AsArchString ();
 
@@ -474,9 +475,15 @@ public static string GetAotArguments (Application app, string filename, Abi abi,
 			args.Append (app.AotArguments);
 			if (llvm_only)
 				args.Append ("llvmonly,");
-			else if (interp)
+			else if (interp) {
+				if (fname != "mscorlib.dll")
+					throw ErrorHelper.CreateError (99, $"Internal error: can only enable the interpreter for mscorlib.dll when AOT-compiling assemblies (tried to interpret {fname}). Please file an issue at https://github.com/xamarin/xamarin-macios/issues/new.");
 				args.Append ("interp,");
-			else
+			} else if (interp_full) {
+				if (fname != "mscorlib.dll")
+					throw ErrorHelper.CreateError (99, $"Internal error: can only enable the interpreter for mscorlib.dll when AOT-compiling assemblies (tried to interpret {fname}). Please file an issue at https://github.com/xamarin/xamarin-macios/issues/new."); 
+				args.Append ("interp,full,");
+			} else
 				args.Append ("full,");
 
 			var aname = Path.GetFileNameWithoutExtension (fname);
@@ -1247,7 +1254,14 @@ static Application ParseArguments (string [] args, out Action a)
 						app.LLVMOptimizations [asm] = opt;
 				}
 			},
-			{ "interpreter", "Enable the *experimental* interpreter.", v => { app.UseInterpreter = true; }},
+			{ "interpreter:", "Enable the *experimental* interpreter. Optionally takes a comma-separated list of assemblies to interpret (if prefixed with a minus sign, the assembly will be AOT-compiled instead). 'all' can be used to specify all assemblies. This argument can be specified multiple times.", v =>
+				{ 
+					app.UseInterpreter = true;
+					if (!string.IsNullOrEmpty (v)) {
+						app.InterpretedAssemblies.AddRange (v.Split (new char [] { ',' }, StringSplitOptions.RemoveEmptyEntries));
+					}
+				}
+			},
 			{ "http-message-handler=", "Specify the default HTTP message handler for HttpClient", v => { http_message_handler = v; }},
 			{ "output-format=", "Specify the output format for some commands. Possible values: Default, XML", v =>
 				{