[mtouch/mmp] Rework how we find developer tools. Partial fix for #4634 and fixes #8005. (#8121)

tests/mmptest/src/MMPTest.cs

@@ -26,7 +26,7 @@ public static string [] GetUnifiedProjectClangInvocation (string tmpDir, string
 			TI.UnifiedTestConfig test = new TI.UnifiedTestConfig (tmpDir) { CSProjConfig = projectConfig };
 			string buildOutput = TI.TestUnifiedExecutable (test).BuildOutput;
 			string [] splitBuildOutput = TI.TestUnifiedExecutable (test).BuildOutput.Split (new string[] { Environment.NewLine }, StringSplitOptions.None);
-			string clangInvocation = splitBuildOutput.Single (x => x.Contains ("clang"));
+			string clangInvocation = splitBuildOutput.Single (x => x.Contains ("usr/bin/clang"));
 			return clangInvocation.Split (new string[] { " " }, StringSplitOptions.None);
 		}
 

tests/mmptest/src/NativeReferencesTests.cs

@@ -112,7 +112,7 @@ public void Unified_WithStaticNativeRef_ClangIncludesOurStaticLib ()
 			MMPTests.RunMMPTest (tmpDir => {
 				TI.UnifiedTestConfig test = new TI.UnifiedTestConfig (tmpDir) { ItemGroup = CreateSingleNativeRef (SimpleStaticPath, "Static") };
 				NativeReferenceTestCore (tmpDir, test, "Unified_WithNativeReferences_InMainProjectWorks - Static", null, true, false, s => {
-					string clangLine = s.Split ('\n').First (x => x.Contains ("xcrun -sdk macosx clang"));
+					string clangLine = s.Split ('\n').First (x => x.Contains ("usr/bin/clang"));
 					return clangLine.Contains ("SimpleClassStatic.a");
 				});
 			});

tools/common/Driver.cs

@@ -640,27 +640,195 @@ static void ValidateXcode (bool accept_any_xcode_version, bool warn_if_not_found
 			Driver.Log (1, "Using Xcode {0} ({2}) found in {1}", XcodeVersion, sdk_root, XcodeProductVersion);
 		}
 
-		public static int XcodeRun (string command, params string [] arguments)
+		internal static bool TryParseBool (string value, out bool result)
 		{
-			return XcodeRun (command, (IList<string>) arguments, null);
+			if (string.IsNullOrEmpty (value)) {
+				result = true;
+				return true;
+			}
+
+			switch (value.ToLowerInvariant ()) {
+			case "1":
+			case "yes":
+			case "true":
+			case "enable":
+				result = true;
+				return true;
+			case "0":
+			case "no":
+			case "false":
+			case "disable":
+				result = false;
+				return true;
+			default:
+				return bool.TryParse (value, out result);
+			}
 		}
 
-		public static int XcodeRun (string command, IList<string> arguments, StringBuilder output = null)
+		internal static bool ParseBool (string value, string name, bool show_error = true)
 		{
-			string [] env = DeveloperDirectory != String.Empty ? new string [] { "DEVELOPER_DIR", DeveloperDirectory } : null;
+			bool result;
+			if (!TryParseBool (value, out result))
+				throw ErrorHelper.CreateError (26, "Could not parse the command line argument '-{0}': {1}", name, value);
+			return result;
+		}
+
+		static readonly Dictionary<string, string> tools = new Dictionary<string, string> ();
+		static string FindTool (string tool)
+		{
+			string path;
+
+			lock (tools) {
+				if (tools.TryGetValue (tool, out path))
+					return path;
+			}
+
+			path = LocateTool (tool);
+			static string LocateTool (string tool)
+			{
+				if (XcrunFind (tool, out var path))
+					return path;
+
+				// either /Developer (Xcode 4.2 and earlier), /Applications/Xcode.app/Contents/Developer (Xcode 4.3) or user override
+				path = Path.Combine (DeveloperDirectory, "usr", "bin", tool);
+				if (File.Exists (path))
+					return path;
+
+				// Xcode 4.3 (without command-line tools) also has a copy of 'strip'
+				path = Path.Combine (DeveloperDirectory, "Toolchains", "XcodeDefault.xctoolchain", "usr", "bin", tool);
+				if (File.Exists (path))
+					return path;
+
+				// Xcode "Command-Line Tools" install a copy in /usr/bin (and it can be there afterward)
+				path = Path.Combine ("/usr", "bin", tool);
+				if (File.Exists (path))
+					return path;
+
+				return null;
+			}
+
+			// We can end up finding the same tool multiple times.
+			// That's not a problem.
+			lock (tools)
+				tools [tool] = path;
+
+			if (path == null)
+				throw ErrorHelper.CreateError (5307, "Missing '{0}' tool. Please install Xcode 'Command-Line Tools' component", tool);
+
+			return path;
+		}
+
+		static bool XcrunFind (string tool, out string path)
+		{
+			return XcrunFind (ApplePlatform.None, false, tool, out path);
+		}
+
+		static bool XcrunFind (ApplePlatform platform, bool is_simulator, string tool, out string path)
+		{
+			var env = new List<string> ();
+			// Unset XCODE_DEVELOPER_DIR_PATH. See https://github.com/xamarin/xamarin-macios/issues/3931.
+			env.Add ("XCODE_DEVELOPER_DIR_PATH");
+			env.Add (null);
+			// Set DEVELOPER_DIR if we have it
+			if (!string.IsNullOrEmpty (DeveloperDirectory)) {
+				env.Add ("DEVELOPER_DIR");
+				env.Add (DeveloperDirectory);
+			}
+
+			path = null;
+
 			var args = new List<string> ();
-			args.Add ("-sdk");
-			args.Add ("macosx");
-			args.Add (command);
-			args.AddRange (arguments);
-			int ret = RunCommand ("xcrun", args, env, output);
-			if (ret != 0 && verbose > 1) {
-				StringBuilder debug = new StringBuilder ();
-				RunCommand ("xcrun", new [] { "--find", command }, env, debug);
-				Console.WriteLine ("failed using `{0}` from: {1}", command, debug);
+			if (platform != ApplePlatform.None) {
+				args.Add ("-sdk");
+				switch (platform) {
+				case ApplePlatform.iOS:
+					args.Add (is_simulator ? "iphonesimulator" : "iphoneos");
+					break;
+				case ApplePlatform.MacOSX:
+					args.Add ("macosx");
+					break;
+				case ApplePlatform.TVOS:
+					args.Add (is_simulator ? "appletvsimulator" : "appletvos");
+					break;
+				case ApplePlatform.WatchOS:
+					args.Add (is_simulator ? "watchsimulator" : "watchos");
+					break;
+				default:
+					throw ErrorHelper.CreateError (71, "Unknown platform: {0}. This usually indicates a bug in {1}; please file a bug report at https://github.com/xamarin/xamarin-macios/issues/new with a test case.", platform.ToString (), PRODUCT);
+				}
 			}
-			return ret;
+			args.Add ("-f");
+			args.Add (tool);
+
+			var output = new StringBuilder ();
+			int ret = RunCommand ("xcrun", args, env.ToArray (), output);
+
+			if (ret == 0) {
+				path = output.ToString ().Trim ();
+			} else {
+				Log (1, "Failed to locate the developer tool '{0}', 'xcrun {1}' returned with the exit code {2}:\n{3}", tool, string.Join (" ", args), ret, output.ToString ());
+			}
+
+			return ret == 0;
+		}
+
+		public static void RunXcodeTool (string tool, params string[] arguments)
+		{
+			RunXcodeTool (tool, (IList<string>) arguments);
+		}
+
+		public static void RunXcodeTool (string tool, IList<string> arguments)
+		{
+			var executable = FindTool (tool);
+			var rv = RunCommand (executable, arguments);
+			if (rv != 0)
+				throw ErrorHelper.CreateError (5309, "Failed to execute the tool '{0}', it failed with an error code '{1}'. Please check the build log for details.", tool, rv);
+		}
+
+		public static void RunClang (IList<string> arguments)
+		{
+			RunXcodeTool ("clang", arguments);
+		}
+
+		public static void RunInstallNameTool (IList<string> arguments)
+		{
+			RunXcodeTool ("install_name_tool", arguments);
 		}
 
+		public static void RunBitcodeStrip (IList<string> arguments)
+		{
+			RunXcodeTool ("bitcode_strip", arguments);
+		}
+
+		public static void RunLipo (string output, IEnumerable<string> inputs)
+		{
+			var sb = new List<string> ();
+			sb.AddRange (inputs);
+			sb.Add ("-create");
+			sb.Add ("-output");
+			sb.Add (output);
+			RunLipo (sb);
+		}
+
+		public static void RunLipo (IList<string> options)
+		{
+			RunXcodeTool ("lipo", options);
+		}
+
+		public static void CreateDsym (string output_dir, string appname, string dsym_dir)
+		{
+			RunDsymUtil (Path.Combine (output_dir, appname), "-num-threads", "4", "-z", "-o", dsym_dir);
+			RunCommand ("/usr/bin/mdimport", dsym_dir);
+		}
+
+		public static void RunDsymUtil (params string [] options)
+		{
+			RunXcodeTool ("dsymutil", options);
+		}
+
+		public static void RunStrip (IList<string> options)
+		{
+			RunXcodeTool ("strip", options);
+		}
 	}
 }

tools/mmp/Application.cs

@@ -9,6 +9,7 @@ public partial class Application
 		public bool Is32Build => false;
 		public bool Is64Build => true;
 		public bool IsDualBuild => false;
+		public bool IsSimulatorBuild => false;
 
 		bool RequiresXcodeHeaders => Driver.Registrar == RegistrarMode.Static && LinkMode == LinkMode.None;
 

tools/mmp/driver.cs

@@ -823,16 +823,8 @@ static void Pack (IList<string> unprocessed)
 			Watch ("Copy Dependencies", 1);
 
 			// MDK check
-			var ret = Compile ();
+			Compile ();
 			Watch ("Compile", 1);
-			if (ret != 0) {
-				if (ret == 1)
-					throw new MonoMacException (5109, true, "Native linking failed with error code 1.  Check build log for details.");
-				if (ret == 69)
-					throw new MonoMacException (5308, true, "Xcode license agreement may not have been accepted.  Please launch Xcode.");
-				// if not then the compilation really failed
-				throw new MonoMacException (5103, true, String.Format ("Failed to compile. Error code - {0}. Please file a bug report at https://github.com/xamarin/xamarin-macios/issues/new", ret));
-			}
 
 			if (generate_plist)
 				GeneratePList ();
@@ -1138,10 +1130,8 @@ static void HandleFramework (IList<string> args, string framework, bool weak)
 				LipoLibrary (framework, Path.Combine (name, Path.Combine (frameworks_dir, name + ".framework", name)));
 		}
 
-		static int Compile ()
+		static void Compile ()
 		{
-			int ret = 1;
-
 			string [] cflags = Array.Empty<string> ();
 			string libdir;
 			StringBuilder cflagsb = new StringBuilder ();
@@ -1402,13 +1392,10 @@ static int Compile ()
 				sourceFiles.Add (main);
 				args.AddRange (sourceFiles);
 
-
-				ret = XcodeRun ("clang", args, null);
+				RunClang (args);
 			} catch (Win32Exception e) {
 				throw new MonoMacException (5103, true, e, "Failed to compile the file '{0}'. Please file a bug report at https://github.com/xamarin/xamarin-macios/issues/new", "driver");
 			}
-
-			return ret;
 		}
 
 		// check that we have a reference to Xamarin.Mac.dll and not to MonoMac.dll.
@@ -1537,9 +1524,7 @@ static void CopyDependencies (IDictionary<string, List<MethodDefinition>> librar
 					string libName = Path.GetFileName (linkWith);
 					string finalLibPath = Path.Combine (mmp_dir, libName);
 					Application.UpdateFile (linkWith, finalLibPath);
-					int ret = XcodeRun ("install_name_tool", new [] { "-id", "@executable_path/../" + BundleName + "/" + libName, finalLibPath });
-					if (ret != 0)
-						throw new MonoMacException (5310, true, "install_name_tool failed with an error code '{0}'. Check build log for details.", ret);
+					RunInstallNameTool (new [] { "-id", "@executable_path/../" + BundleName + "/" + libName, finalLibPath });
 					native_libraries_copied_in.Add (libName);
 				}
 			}
@@ -1568,9 +1553,7 @@ static void CopyDependencies (IDictionary<string, List<MethodDefinition>> librar
 				// if required update the paths inside the .dylib that was copied
 				if (sb.Count > 0) {
 					sb.Add (library);
-					int ret = XcodeRun ("install_name_tool", sb);
-					if (ret != 0)
-						throw new MonoMacException (5310, true, "install_name_tool failed with an error code '{0}'. Check build log for details.", ret);
+					RunInstallNameTool (sb);
 					sb.Clear ();
 				}
 			}
@@ -1708,11 +1691,8 @@ static void ProcessNativeLibrary (HashSet<string> processed, string library, Lis
 				LipoLibrary (name, dest);
 
 			if (native_references.Contains (real_src)) {
-				if (!isStaticLib) {
-					int ret = XcodeRun ("install_name_tool", new [] { "-id", "@executable_path/../" + BundleName + "/" + name, dest });
-					if (ret != 0)
-						throw new MonoMacException (5310, true, "install_name_tool failed with an error code '{0}'. Check build log for details.", ret);
-				}
+				if (!isStaticLib)
+					RunInstallNameTool (new [] { "-id", "@executable_path/../" + BundleName + "/" + name, dest });
 				native_libraries_copied_in.Add (name);
 			}
 
@@ -1744,9 +1724,7 @@ static void LipoLibrary (string name, string dest)
 			if (existingArchs.Count () < 2)
 				return;
 
-			int ret = XcodeRun ("lipo", new [] { dest, "-thin", arch, "-output", dest });
-			if (ret != 0)
-				throw new MonoMacException (5311, true, "lipo failed with an error code '{0}'. Check build log for details.", ret);
+			RunLipo (new [] { dest, "-thin", arch, "-output", dest });
 			if (name != "MonoPosixHelper" && name != "libmono-native-unified" && name != "libmono-native-compat")
 				ErrorHelper.Warning (2108, $"{name} was stripped of architectures except {arch} to comply with App Store restrictions. This could break existing codesigning signatures. Consider stripping the library with lipo or disabling with --optimize=-trim-architectures");
 		}

tools/mmp/mmp.csproj

@@ -8,6 +8,7 @@
     <AssemblyName>mmp</AssemblyName>
     <RootNamespace>mmp</RootNamespace>
     <TargetFrameworkVersion>v4.6</TargetFrameworkVersion>
+    <LangVersion>8.0</LangVersion>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
     <DebugSymbols>True</DebugSymbols>

tools/mtouch/Application.cs

@@ -1811,7 +1811,7 @@ void BuildBundle ()
 							Driver.RunLipo (targetPath, files);
 						}
 						if (LibMonoLinkMode == AssemblyBuildTarget.Framework)
-							Driver.XcodeRun ("install_name_tool", "-change", "@rpath/libmonosgen-2.0.dylib", "@rpath/Mono.framework/Mono", targetPath);
+							Driver.RunInstallNameTool (new [] { "-change", "@rpath/libmonosgen-2.0.dylib", "@rpath/Mono.framework/Mono", targetPath });
 
 						// Remove architectures we don't care about.
 						if (IsDeviceBuild)
@@ -1863,7 +1863,7 @@ public void StripBitcode (string macho_file)
 			}
 			sb.Add ("-o");
 			sb.Add (macho_file);
-			Driver.XcodeRun ("bitcode_strip", sb);
+			Driver.RunBitcodeStrip (sb);
 		}
 
 		// Returns true if is up-to-date

tools/mtouch/Target.cs

@@ -1718,7 +1718,7 @@ public static void AdjustDylibs (string output)
 			}
 			if (sb.Count > 0) {
 				sb.Add (output);
-				Driver.XcodeRun ("install_name_tool", sb);
+				Driver.RunInstallNameTool (sb);
 				sb.Clear ();
 			}
 		}