From bacd958be0f20f6993371d0068877058e7d16505 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Wed, 18 Mar 2020 13:53:55 +0100 Subject: [PATCH] [mtouch/mmp] Rework how we find developer tools. Partial fix for #4634 and fixes #8005. (#8121) (#8128) Partial fix for https://github.com/xamarin/xamarin-macios/issues/4634. Fixes https://github.com/xamarin/xamarin-macios/issues/8005. --- tests/mmptest/src/MMPTest.cs | 2 +- tests/mmptest/src/NativeReferencesTests.cs | 2 +- tools/common/Driver.cs | 196 +++++++++++++++++++-- tools/mmp/Application.cs | 1 + tools/mmp/driver.cs | 38 +--- tools/mmp/mmp.csproj | 1 + tools/mtouch/Application.cs | 4 +- tools/mtouch/Target.cs | 2 +- tools/mtouch/mtouch.cs | 104 ----------- 9 files changed, 197 insertions(+), 153 deletions(-) diff --git a/tests/mmptest/src/MMPTest.cs b/tests/mmptest/src/MMPTest.cs index 37c7ec8a669e..1dac8b2387ea 100644 --- a/tests/mmptest/src/MMPTest.cs +++ b/tests/mmptest/src/MMPTest.cs @@ -26,7 +26,7 @@ public static string [] GetUnifiedProjectClangInvocation (string tmpDir, string TI.UnifiedTestConfig test = new TI.UnifiedTestConfig (tmpDir) { CSProjConfig = projectConfig }; string buildOutput = TI.TestUnifiedExecutable (test).BuildOutput; string [] splitBuildOutput = TI.TestUnifiedExecutable (test).BuildOutput.Split (new string[] { Environment.NewLine }, StringSplitOptions.None); - string clangInvocation = splitBuildOutput.Single (x => x.Contains ("clang")); + string clangInvocation = splitBuildOutput.Single (x => x.Contains ("usr/bin/clang")); return clangInvocation.Split (new string[] { " " }, StringSplitOptions.None); } diff --git a/tests/mmptest/src/NativeReferencesTests.cs b/tests/mmptest/src/NativeReferencesTests.cs index df261088032a..a21db75a7de0 100644 --- a/tests/mmptest/src/NativeReferencesTests.cs +++ b/tests/mmptest/src/NativeReferencesTests.cs @@ -112,7 +112,7 @@ public void Unified_WithStaticNativeRef_ClangIncludesOurStaticLib () MMPTests.RunMMPTest (tmpDir => { TI.UnifiedTestConfig test = new TI.UnifiedTestConfig (tmpDir) { ItemGroup = CreateSingleNativeRef (SimpleStaticPath, "Static") }; NativeReferenceTestCore (tmpDir, test, "Unified_WithNativeReferences_InMainProjectWorks - Static", null, true, false, s => { - string clangLine = s.Split ('\n').First (x => x.Contains ("xcrun -sdk macosx clang")); + string clangLine = s.Split ('\n').First (x => x.Contains ("usr/bin/clang")); return clangLine.Contains ("SimpleClassStatic.a"); }); }); diff --git a/tools/common/Driver.cs b/tools/common/Driver.cs index 508a6ff7a67d..9d1cd79ad4ca 100644 --- a/tools/common/Driver.cs +++ b/tools/common/Driver.cs @@ -640,27 +640,195 @@ static void ValidateXcode (bool accept_any_xcode_version, bool warn_if_not_found Driver.Log (1, "Using Xcode {0} ({2}) found in {1}", XcodeVersion, sdk_root, XcodeProductVersion); } - public static int XcodeRun (string command, params string [] arguments) + internal static bool TryParseBool (string value, out bool result) { - return XcodeRun (command, (IList) arguments, null); + if (string.IsNullOrEmpty (value)) { + result = true; + return true; + } + + switch (value.ToLowerInvariant ()) { + case "1": + case "yes": + case "true": + case "enable": + result = true; + return true; + case "0": + case "no": + case "false": + case "disable": + result = false; + return true; + default: + return bool.TryParse (value, out result); + } } - public static int XcodeRun (string command, IList arguments, StringBuilder output = null) + internal static bool ParseBool (string value, string name, bool show_error = true) { - string [] env = DeveloperDirectory != String.Empty ? new string [] { "DEVELOPER_DIR", DeveloperDirectory } : null; + bool result; + if (!TryParseBool (value, out result)) + throw ErrorHelper.CreateError (26, "Could not parse the command line argument '-{0}': {1}", name, value); + return result; + } + + static readonly Dictionary tools = new Dictionary (); + static string FindTool (string tool) + { + string path; + + lock (tools) { + if (tools.TryGetValue (tool, out path)) + return path; + } + + path = LocateTool (tool); + static string LocateTool (string tool) + { + if (XcrunFind (tool, out var path)) + return path; + + // either /Developer (Xcode 4.2 and earlier), /Applications/Xcode.app/Contents/Developer (Xcode 4.3) or user override + path = Path.Combine (DeveloperDirectory, "usr", "bin", tool); + if (File.Exists (path)) + return path; + + // Xcode 4.3 (without command-line tools) also has a copy of 'strip' + path = Path.Combine (DeveloperDirectory, "Toolchains", "XcodeDefault.xctoolchain", "usr", "bin", tool); + if (File.Exists (path)) + return path; + + // Xcode "Command-Line Tools" install a copy in /usr/bin (and it can be there afterward) + path = Path.Combine ("/usr", "bin", tool); + if (File.Exists (path)) + return path; + + return null; + } + + // We can end up finding the same tool multiple times. + // That's not a problem. + lock (tools) + tools [tool] = path; + + if (path == null) + throw ErrorHelper.CreateError (5307, "Missing '{0}' tool. Please install Xcode 'Command-Line Tools' component", tool); + + return path; + } + + static bool XcrunFind (string tool, out string path) + { + return XcrunFind (ApplePlatform.None, false, tool, out path); + } + + static bool XcrunFind (ApplePlatform platform, bool is_simulator, string tool, out string path) + { + var env = new List (); + // Unset XCODE_DEVELOPER_DIR_PATH. See https://github.com/xamarin/xamarin-macios/issues/3931. + env.Add ("XCODE_DEVELOPER_DIR_PATH"); + env.Add (null); + // Set DEVELOPER_DIR if we have it + if (!string.IsNullOrEmpty (DeveloperDirectory)) { + env.Add ("DEVELOPER_DIR"); + env.Add (DeveloperDirectory); + } + + path = null; + var args = new List (); - args.Add ("-sdk"); - args.Add ("macosx"); - args.Add (command); - args.AddRange (arguments); - int ret = RunCommand ("xcrun", args, env, output); - if (ret != 0 && verbose > 1) { - StringBuilder debug = new StringBuilder (); - RunCommand ("xcrun", new [] { "--find", command }, env, debug); - Console.WriteLine ("failed using `{0}` from: {1}", command, debug); + if (platform != ApplePlatform.None) { + args.Add ("-sdk"); + switch (platform) { + case ApplePlatform.iOS: + args.Add (is_simulator ? "iphonesimulator" : "iphoneos"); + break; + case ApplePlatform.MacOSX: + args.Add ("macosx"); + break; + case ApplePlatform.TVOS: + args.Add (is_simulator ? "appletvsimulator" : "appletvos"); + break; + case ApplePlatform.WatchOS: + args.Add (is_simulator ? "watchsimulator" : "watchos"); + break; + default: + throw ErrorHelper.CreateError (71, "Unknown platform: {0}. This usually indicates a bug in {1}; please file a bug report at https://github.com/xamarin/xamarin-macios/issues/new with a test case.", platform.ToString (), PRODUCT); + } } - return ret; + args.Add ("-f"); + args.Add (tool); + + var output = new StringBuilder (); + int ret = RunCommand ("xcrun", args, env.ToArray (), output); + + if (ret == 0) { + path = output.ToString ().Trim (); + } else { + Log (1, "Failed to locate the developer tool '{0}', 'xcrun {1}' returned with the exit code {2}:\n{3}", tool, string.Join (" ", args), ret, output.ToString ()); + } + + return ret == 0; + } + + public static void RunXcodeTool (string tool, params string[] arguments) + { + RunXcodeTool (tool, (IList) arguments); + } + + public static void RunXcodeTool (string tool, IList arguments) + { + var executable = FindTool (tool); + var rv = RunCommand (executable, arguments); + if (rv != 0) + throw ErrorHelper.CreateError (5309, "Failed to execute the tool '{0}', it failed with an error code '{1}'. Please check the build log for details.", tool, rv); + } + + public static void RunClang (IList arguments) + { + RunXcodeTool ("clang", arguments); + } + + public static void RunInstallNameTool (IList arguments) + { + RunXcodeTool ("install_name_tool", arguments); } + public static void RunBitcodeStrip (IList arguments) + { + RunXcodeTool ("bitcode_strip", arguments); + } + + public static void RunLipo (string output, IEnumerable inputs) + { + var sb = new List (); + sb.AddRange (inputs); + sb.Add ("-create"); + sb.Add ("-output"); + sb.Add (output); + RunLipo (sb); + } + + public static void RunLipo (IList options) + { + RunXcodeTool ("lipo", options); + } + + public static void CreateDsym (string output_dir, string appname, string dsym_dir) + { + RunDsymUtil (Path.Combine (output_dir, appname), "-num-threads", "4", "-z", "-o", dsym_dir); + RunCommand ("/usr/bin/mdimport", dsym_dir); + } + + public static void RunDsymUtil (params string [] options) + { + RunXcodeTool ("dsymutil", options); + } + + public static void RunStrip (IList options) + { + RunXcodeTool ("strip", options); + } } } diff --git a/tools/mmp/Application.cs b/tools/mmp/Application.cs index e9a1b2871650..ed94cdd383a3 100644 --- a/tools/mmp/Application.cs +++ b/tools/mmp/Application.cs @@ -9,6 +9,7 @@ public partial class Application public bool Is32Build => false; public bool Is64Build => true; public bool IsDualBuild => false; + public bool IsSimulatorBuild => false; bool RequiresXcodeHeaders => Driver.Registrar == RegistrarMode.Static && LinkMode == LinkMode.None; diff --git a/tools/mmp/driver.cs b/tools/mmp/driver.cs index a141ac16ca85..1106ee7813fd 100644 --- a/tools/mmp/driver.cs +++ b/tools/mmp/driver.cs @@ -823,16 +823,8 @@ static void Pack (IList unprocessed) Watch ("Copy Dependencies", 1); // MDK check - var ret = Compile (); + Compile (); Watch ("Compile", 1); - if (ret != 0) { - if (ret == 1) - throw new MonoMacException (5109, true, "Native linking failed with error code 1. Check build log for details."); - if (ret == 69) - throw new MonoMacException (5308, true, "Xcode license agreement may not have been accepted. Please launch Xcode."); - // if not then the compilation really failed - throw new MonoMacException (5103, true, String.Format ("Failed to compile. Error code - {0}. Please file a bug report at https://github.com/xamarin/xamarin-macios/issues/new", ret)); - } if (generate_plist) GeneratePList (); @@ -1138,10 +1130,8 @@ static void HandleFramework (IList args, string framework, bool weak) LipoLibrary (framework, Path.Combine (name, Path.Combine (frameworks_dir, name + ".framework", name))); } - static int Compile () + static void Compile () { - int ret = 1; - string [] cflags = Array.Empty (); string libdir; StringBuilder cflagsb = new StringBuilder (); @@ -1402,13 +1392,10 @@ static int Compile () sourceFiles.Add (main); args.AddRange (sourceFiles); - - ret = XcodeRun ("clang", args, null); + RunClang (args); } catch (Win32Exception e) { throw new MonoMacException (5103, true, e, "Failed to compile the file '{0}'. Please file a bug report at https://github.com/xamarin/xamarin-macios/issues/new", "driver"); } - - return ret; } // check that we have a reference to Xamarin.Mac.dll and not to MonoMac.dll. @@ -1537,9 +1524,7 @@ static void CopyDependencies (IDictionary> librar string libName = Path.GetFileName (linkWith); string finalLibPath = Path.Combine (mmp_dir, libName); Application.UpdateFile (linkWith, finalLibPath); - int ret = XcodeRun ("install_name_tool", new [] { "-id", "@executable_path/../" + BundleName + "/" + libName, finalLibPath }); - if (ret != 0) - throw new MonoMacException (5310, true, "install_name_tool failed with an error code '{0}'. Check build log for details.", ret); + RunInstallNameTool (new [] { "-id", "@executable_path/../" + BundleName + "/" + libName, finalLibPath }); native_libraries_copied_in.Add (libName); } } @@ -1568,9 +1553,7 @@ static void CopyDependencies (IDictionary> librar // if required update the paths inside the .dylib that was copied if (sb.Count > 0) { sb.Add (library); - int ret = XcodeRun ("install_name_tool", sb); - if (ret != 0) - throw new MonoMacException (5310, true, "install_name_tool failed with an error code '{0}'. Check build log for details.", ret); + RunInstallNameTool (sb); sb.Clear (); } } @@ -1708,11 +1691,8 @@ static void ProcessNativeLibrary (HashSet processed, string library, Lis LipoLibrary (name, dest); if (native_references.Contains (real_src)) { - if (!isStaticLib) { - int ret = XcodeRun ("install_name_tool", new [] { "-id", "@executable_path/../" + BundleName + "/" + name, dest }); - if (ret != 0) - throw new MonoMacException (5310, true, "install_name_tool failed with an error code '{0}'. Check build log for details.", ret); - } + if (!isStaticLib) + RunInstallNameTool (new [] { "-id", "@executable_path/../" + BundleName + "/" + name, dest }); native_libraries_copied_in.Add (name); } @@ -1744,9 +1724,7 @@ static void LipoLibrary (string name, string dest) if (existingArchs.Count () < 2) return; - int ret = XcodeRun ("lipo", new [] { dest, "-thin", arch, "-output", dest }); - if (ret != 0) - throw new MonoMacException (5311, true, "lipo failed with an error code '{0}'. Check build log for details.", ret); + RunLipo (new [] { dest, "-thin", arch, "-output", dest }); if (name != "MonoPosixHelper" && name != "libmono-native-unified" && name != "libmono-native-compat") ErrorHelper.Warning (2108, $"{name} was stripped of architectures except {arch} to comply with App Store restrictions. This could break existing codesigning signatures. Consider stripping the library with lipo or disabling with --optimize=-trim-architectures"); } diff --git a/tools/mmp/mmp.csproj b/tools/mmp/mmp.csproj index 581be87d8a23..eca1417b9864 100644 --- a/tools/mmp/mmp.csproj +++ b/tools/mmp/mmp.csproj @@ -8,6 +8,7 @@ mmp mmp v4.6 + 8.0 True diff --git a/tools/mtouch/Application.cs b/tools/mtouch/Application.cs index c0894f86e2fa..6d27c525e098 100644 --- a/tools/mtouch/Application.cs +++ b/tools/mtouch/Application.cs @@ -1811,7 +1811,7 @@ void BuildBundle () Driver.RunLipo (targetPath, files); } if (LibMonoLinkMode == AssemblyBuildTarget.Framework) - Driver.XcodeRun ("install_name_tool", "-change", "@rpath/libmonosgen-2.0.dylib", "@rpath/Mono.framework/Mono", targetPath); + Driver.RunInstallNameTool (new [] { "-change", "@rpath/libmonosgen-2.0.dylib", "@rpath/Mono.framework/Mono", targetPath }); // Remove architectures we don't care about. if (IsDeviceBuild) @@ -1863,7 +1863,7 @@ public void StripBitcode (string macho_file) } sb.Add ("-o"); sb.Add (macho_file); - Driver.XcodeRun ("bitcode_strip", sb); + Driver.RunBitcodeStrip (sb); } // Returns true if is up-to-date diff --git a/tools/mtouch/Target.cs b/tools/mtouch/Target.cs index 5cf05924b91a..db93b9eab68f 100644 --- a/tools/mtouch/Target.cs +++ b/tools/mtouch/Target.cs @@ -1718,7 +1718,7 @@ public static void AdjustDylibs (string output) } if (sb.Count > 0) { sb.Add (output); - Driver.XcodeRun ("install_name_tool", sb); + Driver.RunInstallNameTool (sb); sb.Clear (); } } diff --git a/tools/mtouch/mtouch.cs b/tools/mtouch/mtouch.cs index 54fb58b388d6..47881ab7b45d 100644 --- a/tools/mtouch/mtouch.cs +++ b/tools/mtouch/mtouch.cs @@ -841,39 +841,6 @@ public static bool CanWeSymlinkTheApplication (Application app) return true; } - internal static bool TryParseBool (string value, out bool result) - { - if (string.IsNullOrEmpty (value)) { - result = true; - return true; - } - - switch (value.ToLowerInvariant ()) { - case "1": - case "yes": - case "true": - case "enable": - result = true; - return true; - case "0": - case "no": - case "false": - case "disable": - result = false; - return true; - default: - return bool.TryParse (value, out result); - } - } - - internal static bool ParseBool (string value, string name, bool show_error = true) - { - bool result; - if (!TryParseBool (value, out result)) - throw ErrorHelper.CreateError (26, "Could not parse the command line argument '-{0}:{1}'", name, value); - return result; - } - public static int Main (string [] args) { try { @@ -1507,77 +1474,6 @@ public static void CalculateCompilerPath (Application app) } } - // workaround issues like: - // * Xcode 4.x versus 4.3 (location of /Developer); and - // * the (optional) installation of "Command-Line Tools" by Xcode - public static void RunStrip (IList options) - { - // either /Developer (Xcode 4.2 and earlier), /Applications/Xcode.app/Contents/Developer (Xcode 4.3) or user override - string strip = FindTool ("strip"); - if (strip == null) - throw new MonoTouchException (5301, "Missing 'strip' tool. Please install Xcode 'Command-Line Tools' component"); - - if (RunCommand (strip, options) != 0) - throw new MonoTouchException (5304, true, "Failed to strip the final binary. Please review the build log."); - } - - static string FindTool (string tool) - { - // either /Developer (Xcode 4.2 and earlier), /Applications/Xcode.app/Contents/Developer (Xcode 4.3) or user override - var path = Path.Combine (DeveloperDirectory, "usr", "bin", tool); - if (File.Exists (path)) - return path; - - // Xcode "Command-Line Tools" install a copy in /usr/bin (and it can be there afterward) - path = Path.Combine ("/usr", "bin", tool); - if (File.Exists (path)) - return path; - - // Xcode 4.3 (without command-line tools) also has a copy of 'strip' - path = Path.Combine (DeveloperDirectory, "Toolchains", "XcodeDefault.xctoolchain", "usr", "bin", tool); - if (File.Exists (path)) - return path; - - return null; - } - - public static void CreateDsym (string output_dir, string appname, string dsym_dir) - { - RunDsymUtil (Path.Combine (output_dir, appname), "-num-threads", "4", "-z", "-o", dsym_dir); - RunCommand ("/usr/bin/mdimport", dsym_dir); - } - - public static void RunLipo (string output, IEnumerable inputs) - { - var sb = new List (); - sb.AddRange (inputs); - sb.Add ("-create"); - sb.Add ("-output"); - sb.Add (output); - RunLipo (sb); - } - - public static void RunLipo (IList options) - { - string lipo = FindTool ("lipo"); - if (lipo == null) - throw new MonoTouchException (5305, true, "Missing 'lipo' tool. Please install Xcode 'Command-Line Tools' component"); - if (RunCommand (lipo, options) != 0) - throw new MonoTouchException (5306, true, "Failed to create the a fat library. Please review the build log."); - } - - static void RunDsymUtil (params string[] options) - { - // either /Developer (Xcode 4.2 and earlier), /Applications/Xcode.app/Contents/Developer (Xcode 4.3) or user override - string dsymutil = FindTool ("dsymutil"); - if (dsymutil == null) { - ErrorHelper.Warning (5302, "Missing 'dsymutil' tool. Please install Xcode 'Command-Line Tools' component"); - return; - } - if (RunCommand (dsymutil, options) != 0) - throw new MonoTouchException (5303, true, "Failed to generate the debug symbols (dSYM directory). Please review the build log."); - } - static string GetFrameworkDir (string platform, Version iphone_sdk) { return Path.Combine (PlatformsDirectory, platform + ".platform", "Developer", "SDKs", platform + iphone_sdk.ToString () + ".sdk");