From cdbf1227d2f41a3fb7ded30edb99fdeca03ef4f8 Mon Sep 17 00:00:00 2001 From: Bernhard Urban Date: Tue, 11 Sep 2018 14:06:53 +0200 Subject: [PATCH] [mtouch] add mixed-mode support (#4751) * [mtouch] add --interp-mixed option When enabling this option, mtouch will AOT compile `mscorlib.dll`. At runtime that means every method that wasn't AOT'd will be executed by the runtime interpreter. * [mtouch] Add support to --interpreter to list the assemblies to (not) interpret. * [msbuild] Simplify interpreter code to use a single variable. * Fix whitespace. * [mtouch] Move mtouch-specific code to mtouch-specific file. * [msbuild] An empty string is a valid value for 'Interpreter', so make it a non-required property. * [mtouch] Add sanity check for aot-compiling interpreted assemblies. --- .../Tasks/MTouchTaskBase.cs | 7 +-- .../Xamarin.iOS.Common.props | 1 - .../Xamarin.iOS.Common.targets | 2 +- tests/common/BundlerTool.cs | 8 ++- tests/mtouch/MTouch.cs | 19 +++++++ tests/xharness/Jenkins.cs | 3 + tools/common/Application.cs | 1 - tools/mtouch/Application.cs | 57 ++++++++++++++++++- tools/mtouch/Assembly.cs | 17 +++--- tools/mtouch/Target.cs | 25 ++++++++ tools/mtouch/mtouch.cs | 22 +++++-- 11 files changed, 139 insertions(+), 23 deletions(-) diff --git a/msbuild/Xamarin.iOS.Tasks.Core/Tasks/MTouchTaskBase.cs b/msbuild/Xamarin.iOS.Tasks.Core/Tasks/MTouchTaskBase.cs index c0159bc5bc34..7d58d6e37591 100644 --- a/msbuild/Xamarin.iOS.Tasks.Core/Tasks/MTouchTaskBase.cs +++ b/msbuild/Xamarin.iOS.Tasks.Core/Tasks/MTouchTaskBase.cs @@ -106,8 +106,7 @@ public GccOptions () [Required] public bool EnableSGenConc { get; set; } - [Required] - public bool UseInterpreter { get; set; } + public string Interpreter { get; set; } [Required] public bool LinkerDumpDependencies { get; set; } @@ -405,8 +404,8 @@ protected override string GenerateCommandLineCommands () if (EnableSGenConc) args.AddLine ("--sgen-conc"); - if (UseInterpreter) - args.Add ("--interpreter"); + if (!string.IsNullOrEmpty (Interpreter)) + args.Add ($"--interpreter={Interpreter}"); switch (LinkMode.ToLowerInvariant ()) { case "sdkonly": args.AddLine ("--linksdkonly"); break; diff --git a/msbuild/Xamarin.iOS.Tasks.Core/Xamarin.iOS.Common.props b/msbuild/Xamarin.iOS.Tasks.Core/Xamarin.iOS.Common.props index de1f2acdf112..a29b7e7a18e5 100644 --- a/msbuild/Xamarin.iOS.Tasks.Core/Xamarin.iOS.Common.props +++ b/msbuild/Xamarin.iOS.Tasks.Core/Xamarin.iOS.Common.props @@ -51,7 +51,6 @@ Copyright (C) 2013-2016 Xamarin. All rights reserved. False $(MSBuildProjectDirectory) False - False 2 true diff --git a/msbuild/Xamarin.iOS.Tasks.Core/Xamarin.iOS.Common.targets b/msbuild/Xamarin.iOS.Tasks.Core/Xamarin.iOS.Common.targets index db613f01c062..ff54f2301f14 100644 --- a/msbuild/Xamarin.iOS.Tasks.Core/Xamarin.iOS.Common.targets +++ b/msbuild/Xamarin.iOS.Tasks.Core/Xamarin.iOS.Common.targets @@ -831,7 +831,7 @@ Copyright (C) 2013-2016 Xamarin. All rights reserved. UseThumb="$(MtouchUseThumb)" EnableBitcode="$(MtouchEnableBitcode)" EnableSGenConc="$(MtouchEnableSGenConc)" - UseInterpreter="$(MtouchUseInterpreter)" + Interpreter="$(MtouchInterpreter)" AppExtensionReferences="@(_ResolvedAppExtensionReferences)" ArchiveSymbols="$(MonoSymbolArchive)" Verbosity="$(MtouchVerbosity)" diff --git a/tests/common/BundlerTool.cs b/tests/common/BundlerTool.cs index 498c95d6076e..65fba9979882 100644 --- a/tests/common/BundlerTool.cs +++ b/tests/common/BundlerTool.cs @@ -74,6 +74,7 @@ abstract class BundlerTool : Tool public int Verbosity; public int [] WarnAsError; // null array: nothing passed to mtouch/mmp. empty array: pass --warnaserror (which means makes all warnings errors). public string [] XmlDefinitions; + public string Interpreter; // These are a bit smarter public bool NoPlatformAssemblyReference; @@ -291,7 +292,12 @@ protected virtual void BuildArguments (StringBuilder sb) sb.Append (" --xml:").Append (StringUtils.Quote (xd)); } - + if (Interpreter != null) { + if (Interpreter.Length == 0) + sb.Append (" --interpreter"); + else + sb.Append (" --interpreter=").Append (Interpreter); + } } public string CreateTemporaryDirectory () diff --git a/tests/mtouch/MTouch.cs b/tests/mtouch/MTouch.cs index 34cca5475041..4644e1001ee1 100644 --- a/tests/mtouch/MTouch.cs +++ b/tests/mtouch/MTouch.cs @@ -1780,6 +1780,25 @@ public class B } } + [Test] + public void MT0138 () + { + using (var mtouch = new MTouchTool ()) { + var tmpdir = mtouch.CreateTemporaryDirectory (); + mtouch.CreateTemporaryCacheDirectory (); + + mtouch.CreateTemporaryApp (); + mtouch.WarnAsError = new int [] { 138 }; // This is just to make mtouch bail out early instead of spending time building the app when that's not what we're interested in. + mtouch.Interpreter = "all,-all,foo,-bar,mscorlib.dll,mscorlib"; + mtouch.AssertExecuteFailure (MTouchAction.BuildSim, "build"); + mtouch.AssertError (138, "Cannot find the assembly 'foo', passed as an argument to --interpreter."); + mtouch.AssertError (138, "Cannot find the assembly 'bar', passed as an argument to --interpreter."); + mtouch.AssertError (138, "Cannot find the assembly 'mscorlib.dll', passed as an argument to --interpreter."); + // just the name, without the extension, is the right way. + mtouch.AssertErrorCount (3); + } + } + [Test] [TestCase ("all")] [TestCase ("-all")] diff --git a/tests/xharness/Jenkins.cs b/tests/xharness/Jenkins.cs index fa67c59abd40..5cc986274557 100644 --- a/tests/xharness/Jenkins.cs +++ b/tests/xharness/Jenkins.cs @@ -182,12 +182,15 @@ IEnumerable GetTestData (RunTestTask test) yield return new TestData { Variation = "Release (all optimizations)", MTouchExtraArgs = "--registrar:static --optimize:all", Debug = false, Profiling = false, Defines = "OPTIMIZEALL" }; yield return new TestData { Variation = "Debug (all optimizations)", MTouchExtraArgs = "--registrar:static --optimize:all", Debug = true, Profiling = false, Defines = "OPTIMIZEALL" }; yield return new TestData { Variation = "Debug (interpreter)", MTouchExtraArgs = "--interpreter", Debug = true, Profiling = false, Ignored = true, }; + yield return new TestData { Variation = "Debug (interpreter -mscorlib)", MTouchExtraArgs = "--interpreter=-mscorlib", Debug = true, Profiling = false, Ignored = true, }; break; case "mscorlib": yield return new TestData { Variation = "Debug (interpreter)", MTouchExtraArgs = "--interpreter", Debug = true, Profiling = false, Ignored = true, Undefines = "FULL_AOT_RUNTIME" }; + yield return new TestData { Variation = "Debug (interpreter -mscorlib)", MTouchExtraArgs = "--interpreter=-mscorlib", Debug = true, Profiling = false, Ignored = true, Undefines = "FULL_AOT_RUNTIME" }; break; case "mini": yield return new TestData { Variation = "Debug (interpreter)", MTouchExtraArgs = "--interpreter", Debug = true, Profiling = false, Undefines = "FULL_AOT_RUNTIME" }; + yield return new TestData { Variation = "Debug (interpreter -mscorlib)", MTouchExtraArgs = "--interpreter=-mscorlib", Debug = true, Profiling = false, Undefines = "FULL_AOT_RUNTIME" }; break; } break; diff --git a/tools/common/Application.cs b/tools/common/Application.cs index 01b661a2b4d0..b111651dbf80 100644 --- a/tools/common/Application.cs +++ b/tools/common/Application.cs @@ -63,7 +63,6 @@ public partial class Application public bool? EnableCoopGC; public bool EnableSGenConc; - public bool UseInterpreter; public MarshalObjectiveCExceptionMode MarshalObjectiveCExceptions; public MarshalManagedExceptionMode MarshalManagedExceptions; diff --git a/tools/mtouch/Application.cs b/tools/mtouch/Application.cs index 3827c9deeb41..9d8d6d336441 100644 --- a/tools/mtouch/Application.cs +++ b/tools/mtouch/Application.cs @@ -116,6 +116,8 @@ public bool EnableMSym { public string AotOtherArguments = string.Empty; public bool? LLVMAsmWriter; public Dictionary LLVMOptimizations = new Dictionary (); + public bool UseInterpreter; + public List InterpretedAssemblies = new List (); public Dictionary EnvironmentVariables = new Dictionary (); @@ -129,6 +131,44 @@ public bool EnableMSym { public bool? BuildDSym; + public bool IsInterpreted (string assembly) + { + // IsAOTCompiled and IsInterpreted are not opposites: mscorlib.dll can be both. + if (!UseInterpreter) + return false; + + // Go through the list of assemblies to interpret in reverse order, + // so that the last option passed to mtouch takes precedence. + for (int i = InterpretedAssemblies.Count - 1; i >= 0; i--) { + var opt = InterpretedAssemblies [i]; + if (opt == "all") + return true; + else if (opt == "-all") + return false; + else if (opt == assembly) + return true; + else if (opt [0] == '-' && opt.Substring (1) == assembly) + return false; + } + + // There's an implicit 'all' at the start of the list. + return true; + } + + public bool IsAOTCompiled (string assembly) + { + if (!UseInterpreter) + return true; + + // IsAOTCompiled and IsInterpreted are not opposites: mscorlib.dll can be both: + // - mscorlib will always be processed by the AOT compiler to generate required wrapper functions for the interpreter to work + // - mscorlib might also be fully AOT-compiled (both when the interpreter is enabled and when it's not) + if (assembly == "mscorlib") + return true; + + return !IsInterpreted (assembly); + } + // If we're targetting a 32 bit arch. bool? is32bits; public bool Is32Build { @@ -462,7 +502,7 @@ public bool UseDlsym (string assembly) if (EnableLLVMOnlyBitCode) return false; - if (UseInterpreter) + if (IsInterpreted (Assembly.GetIdentity (assembly))) return true; switch (Platform) { @@ -2124,7 +2164,20 @@ public void StripNativeCode () public void BundleAssemblies () { - var strip = !UseInterpreter && ManagedStrip && IsDeviceBuild && !EnableDebug && !PackageManagedDebugSymbols; + Assembly.StripAssembly strip = ((path) => + { + if (!ManagedStrip) + return false; + if (!IsDeviceBuild) + return false; + if (EnableDebug) + return false; + if (PackageManagedDebugSymbols) + return false; + if (IsInterpreted (Assembly.GetIdentity (path))) + return false; + return true; + }); var grouped = Targets.SelectMany ((Target t) => t.Assemblies).GroupBy ((Assembly asm) => asm.Identity); foreach (var @group in grouped) { diff --git a/tools/mtouch/Assembly.cs b/tools/mtouch/Assembly.cs index e7618983ada0..ac37322c80c3 100644 --- a/tools/mtouch/Assembly.cs +++ b/tools/mtouch/Assembly.cs @@ -54,11 +54,7 @@ public HashSet DependencyMap { public bool IsAOTCompiled { get { - if (App.UseInterpreter) - /* interpreter only requires a few stubs that are attached - * to mscorlib.dll, other assemblies won't be AOT compiled */ - return FileName == "mscorlib.dll"; - return true; + return App.IsAOTCompiled (Identity); } } @@ -108,15 +104,18 @@ public void ComputeDependencyMap (List exceptions) ComputeDependencies (exceptions); } + public delegate bool StripAssembly (string path); + // returns false if the assembly was not copied (because it was already up-to-date). - public bool CopyAssembly (string source, string target, bool copy_debug_symbols = true, bool strip = false) + public bool CopyAssembly (string source, string target, bool copy_debug_symbols = true, StripAssembly strip = null) { var copied = false; try { - if (!Application.IsUptodate (source, target) && (strip || !Cache.CompareAssemblies (source, target))) { + var strip_assembly = strip != null && strip (source); + if (!Application.IsUptodate (source, target) && (strip_assembly || !Cache.CompareAssemblies (source, target))) { copied = true; - if (strip) { + if (strip_assembly) { Driver.FileDelete (target); Directory.CreateDirectory (Path.GetDirectoryName (target)); MonoTouch.Tuner.Stripper.Process (source, target); @@ -195,7 +194,7 @@ public void CopyConfigToDirectory (string directory) // Aot data is copied separately, because we might want to copy aot data // even if we don't want to copy the assembly (if 32/64-bit assemblies are identical, // only one is copied, but we still want the aotdata for both). - public void CopyToDirectory (string directory, bool reload = true, bool check_case = false, bool only_copy = false, bool copy_debug_symbols = true, bool strip = false) + public void CopyToDirectory (string directory, bool reload = true, bool check_case = false, bool only_copy = false, bool copy_debug_symbols = true, StripAssembly strip = null) { var target = Path.Combine (directory, FileName); diff --git a/tools/mtouch/Target.cs b/tools/mtouch/Target.cs index 75585bd4394d..fdd067bf1d80 100644 --- a/tools/mtouch/Target.cs +++ b/tools/mtouch/Target.cs @@ -283,6 +283,29 @@ public void Initialize (bool show_warnings) } linker_flags = new CompilerFlags (this); + + // Verify that there are no entries in our list of intepreted assemblies that doesn't match + // any of the assemblies we know about. + if (App.UseInterpreter) { + var exceptions = new List (); + foreach (var entry in App.InterpretedAssemblies) { + var assembly = entry; + if (string.IsNullOrEmpty (assembly)) + continue; + + if (assembly [0] == '-') + assembly = assembly.Substring (1); + + if (assembly == "all") + continue; + + if (Assemblies.ContainsKey (assembly)) + continue; + + exceptions.Add (ErrorHelper.CreateWarning (138, $"Cannot find the assembly '{assembly}', passed as an argument to --interpreter.")); + } + ErrorHelper.ThrowIfErrors (exceptions); + } } // This is to load the symbols for all assemblies, so that we can give better error messages @@ -1134,6 +1157,8 @@ void AOTCompile () } if (App.UseInterpreter) + /* TODO: not sure? we might have to continue here, depending on + * the set of assemblies are AOT'd? */ return; // Code in one assembly (either in a P/Invoke or a third-party library) can depend on a third-party library in another assembly. diff --git a/tools/mtouch/mtouch.cs b/tools/mtouch/mtouch.cs index 983bc6a64f5e..ed27746d98e8 100644 --- a/tools/mtouch/mtouch.cs +++ b/tools/mtouch/mtouch.cs @@ -455,7 +455,8 @@ public static string GetAotArguments (Application app, string filename, Abi abi, bool enable_debug = app.EnableDebug; bool enable_debug_symbols = app.PackageManagedDebugSymbols; bool llvm_only = app.EnableLLVMOnlyBitCode; - bool interp = app.UseInterpreter; + bool interp = app.IsInterpreted (Assembly.GetIdentity (filename)); + bool interp_full = !interp && app.UseInterpreter && fname == "mscorlib.dll"; bool is32bit = (abi & Abi.Arch32Mask) > 0; string arch = abi.AsArchString (); @@ -474,9 +475,15 @@ public static string GetAotArguments (Application app, string filename, Abi abi, args.Append (app.AotArguments); if (llvm_only) args.Append ("llvmonly,"); - else if (interp) + else if (interp) { + if (fname != "mscorlib.dll") + throw ErrorHelper.CreateError (99, $"Internal error: can only enable the interpreter for mscorlib.dll when AOT-compiling assemblies (tried to interpret {fname}). Please file an issue at https://github.com/xamarin/xamarin-macios/issues/new."); args.Append ("interp,"); - else + } else if (interp_full) { + if (fname != "mscorlib.dll") + throw ErrorHelper.CreateError (99, $"Internal error: can only enable the interpreter for mscorlib.dll when AOT-compiling assemblies (tried to interpret {fname}). Please file an issue at https://github.com/xamarin/xamarin-macios/issues/new."); + args.Append ("interp,full,"); + } else args.Append ("full,"); var aname = Path.GetFileNameWithoutExtension (fname); @@ -1247,7 +1254,14 @@ static Application ParseArguments (string [] args, out Action a) app.LLVMOptimizations [asm] = opt; } }, - { "interpreter", "Enable the *experimental* interpreter.", v => { app.UseInterpreter = true; }}, + { "interpreter:", "Enable the *experimental* interpreter. Optionally takes a comma-separated list of assemblies to interpret (if prefixed with a minus sign, the assembly will be AOT-compiled instead). 'all' can be used to specify all assemblies. This argument can be specified multiple times.", v => + { + app.UseInterpreter = true; + if (!string.IsNullOrEmpty (v)) { + app.InterpretedAssemblies.AddRange (v.Split (new char [] { ',' }, StringSplitOptions.RemoveEmptyEntries)); + } + } + }, { "http-message-handler=", "Specify the default HTTP message handler for HttpClient", v => { http_message_handler = v; }}, { "output-format=", "Specify the output format for some commands. Possible values: Default, XML", v => {