Implement MonoVM toolchain for net6.0 and net7.0 monikers (#2142) fixes

#2064

Co-authored-by: Adam Sitnik <[email protected]>

src/BenchmarkDotNet.Annotations/Jobs/RuntimeMoniker.cs

@@ -143,6 +143,16 @@ public enum RuntimeMoniker
         /// <summary>
         /// Mono with the Ahead of Time LLVM Compiler backend and .net7.0
         /// </summary>
-        MonoAOTLLVMNet70
+        MonoAOTLLVMNet70,
+
+        /// <summary>
+        /// .NET 6 using MonoVM (not CLR which is the default)
+        /// </summary>
+        Mono60,
+
+        /// <summary>
+        /// .NET 7 using MonoVM (not CLR which is the default)
+        /// </summary>
+        Mono70,
     }
 }

src/BenchmarkDotNet/Attributes/Jobs/MonoJobAttribute.cs

@@ -1,5 +1,6 @@
 using System;
 using BenchmarkDotNet.Environments;
+using BenchmarkDotNet.Extensions;
 using BenchmarkDotNet.Jobs;
 
 namespace BenchmarkDotNet.Attributes
@@ -11,6 +12,10 @@ public MonoJobAttribute(bool baseline = false) : base(Job.Default.WithRuntime(Mo
         {
         }
 
+        public MonoJobAttribute(RuntimeMoniker runtimeMoniker, bool baseline = false) : base(Job.Default.WithRuntime(runtimeMoniker.GetRuntime()).WithBaseline(baseline))
+        {
+        }
+
         public MonoJobAttribute(string name, string path, bool baseline = false)
             : base(new Job(name, new EnvironmentMode(new MonoRuntime(name, path)).Freeze()).WithBaseline(baseline).Freeze())
         {

src/BenchmarkDotNet/BenchmarkDotNet.csproj

@@ -33,7 +33,7 @@
     <PackageReference Include="System.Numerics.Vectors" Version="4.5.0" />
     <PackageReference Include="Microsoft.Win32.Registry" Version="5.0.0" />
   </ItemGroup>
-  <ItemGroup Condition="'$(OS)' == 'Windows_NT'">
+  <ItemGroup Condition="'$(OS)' == 'Windows_NT' AND '$(UseMonoRuntime)' != 'true' ">
     <ProjectReference Include="..\BenchmarkDotNet.Disassembler.x64\BenchmarkDotNet.Disassembler.x64.csproj">
       <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
     </ProjectReference>

src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs

@@ -29,6 +29,7 @@
 using Perfolizer.Mathematics.OutlierDetection;
 using Perfolizer.Mathematics.SignificanceTesting;
 using Perfolizer.Mathematics.Thresholds;
+using BenchmarkDotNet.Toolchains.Mono;
 
 namespace BenchmarkDotNet.ConsoleArguments
 {
@@ -425,6 +426,12 @@ private static Job CreateJobForGivenRuntime(Job baseJob, string runtimeId, Comma
                 case RuntimeMoniker.MonoAOTLLVMNet70:
                     return MakeMonoAOTLLVMJob(baseJob, options, "net7.0");
 
+                case RuntimeMoniker.Mono60:
+                    return MakeMonoJob(baseJob, options, MonoRuntime.Mono60);
+
+                case RuntimeMoniker.Mono70:
+                    return MakeMonoJob(baseJob, options, MonoRuntime.Mono70);
+
                 default:
                     throw new NotSupportedException($"Runtime {runtimeId} is not supported");
             }
@@ -452,6 +459,19 @@ private static Job CreateAotJob(Job baseJob, CommandLineOptions options, Runtime
             return baseJob.WithRuntime(runtime).WithToolchain(builder.ToToolchain());
         }
 
+        private static Job MakeMonoJob(Job baseJob, CommandLineOptions options, MonoRuntime runtime)
+        {
+            return baseJob
+                .WithRuntime(runtime)
+                .WithToolchain(MonoToolchain.From(
+                    new NetCoreAppSettings(
+                        targetFrameworkMoniker: runtime.MsBuildMoniker,
+                        runtimeFrameworkVersion: null,
+                        name: runtime.Name,
+                        customDotNetCliPath: options.CliPath?.FullName,
+                        packagesPath: options.RestorePath?.FullName)));
+        }
+
         private static Job MakeMonoAOTLLVMJob(Job baseJob, CommandLineOptions options, string msBuildMoniker)
         {
             var monoAotLLVMRuntime = new MonoAotLLVMRuntime(aotCompilerPath: options.AOTCompilerPath, aotCompilerMode: options.AOTCompilerMode, msBuildMoniker: msBuildMoniker);

src/BenchmarkDotNet/Environments/Runtimes/MonoRuntime.cs

@@ -6,6 +6,8 @@ namespace BenchmarkDotNet.Environments
     public class MonoRuntime : Runtime, IEquatable<MonoRuntime>
     {
         public static readonly MonoRuntime Default = new MonoRuntime("Mono");
+        public static readonly MonoRuntime Mono60 = new MonoRuntime("Mono with .NET 6.0", RuntimeMoniker.Mono60, "net6.0", isDotNetBuiltIn: true);
+        public static readonly MonoRuntime Mono70 = new MonoRuntime("Mono with .NET 7.0", RuntimeMoniker.Mono70, "net7.0", isDotNetBuiltIn: true);
 
         public string CustomPath { get; }
 
@@ -15,8 +17,15 @@ public class MonoRuntime : Runtime, IEquatable<MonoRuntime>
 
         public string MonoBclPath { get; }
 
+        internal bool IsDotNetBuiltIn { get; }
+
         private MonoRuntime(string name) : base(RuntimeMoniker.Mono, "mono", name) { }
 
+        private MonoRuntime(string name, RuntimeMoniker runtimeMoniker, string msBuildMoniker, bool isDotNetBuiltIn) : base(runtimeMoniker, msBuildMoniker, name)
+        {
+            IsDotNetBuiltIn = isDotNetBuiltIn;
+        }
+
         public MonoRuntime(string name, string customPath) : this(name) => CustomPath = customPath;
 
         public MonoRuntime(string name, string customPath, string aotArgs, string monoBclPath) : this(name)

src/BenchmarkDotNet/Extensions/RuntimeMonikerExtensions.cs

@@ -49,6 +49,10 @@ internal static Runtime GetRuntime(this RuntimeMoniker runtimeMoniker)
                     return NativeAotRuntime.Net60;
                 case RuntimeMoniker.NativeAot70:
                     return NativeAotRuntime.Net70;
+                case RuntimeMoniker.Mono60:
+                    return MonoRuntime.Mono60;
+                case RuntimeMoniker.Mono70:
+                    return MonoRuntime.Mono70;
                 default:
                     throw new ArgumentOutOfRangeException(nameof(runtimeMoniker), runtimeMoniker, "Runtime Moniker not supported");
             }

src/BenchmarkDotNet/Portability/RuntimeInformation.cs

@@ -25,7 +25,7 @@ internal static class RuntimeInformation
         internal const string ReleaseConfigurationName = "RELEASE";
         internal const string Unknown = "?";
 
-        public static bool IsMono { get; } = Type.GetType("Mono.Runtime") != null; // it allocates a lot of memory, we need to check it once in order to keep Engine non-allocating!
+        public static bool IsMono { get; } = Type.GetType("Mono.RuntimeStructs") != null; // it allocates a lot of memory, we need to check it once in order to keep Engine non-allocating!
 
         public static bool IsFullFramework =>
 #if NET6_0_OR_GREATER
@@ -184,6 +184,10 @@ internal static string GetRuntimeVersion()
 
                     return "Mono " + version;
                 }
+                else
+                {
+                    return $"{GetNetCoreVersion()} using MonoVM";
+                }
             }
             else if (IsFullFramework)
             {
@@ -208,22 +212,7 @@ internal static string GetRuntimeVersion()
             }
             else if (IsNetCore)
             {
-                var coreclrAssemblyInfo = FileVersionInfo.GetVersionInfo(typeof(object).GetTypeInfo().Assembly.Location);
-                var corefxAssemblyInfo = FileVersionInfo.GetVersionInfo(typeof(Regex).GetTypeInfo().Assembly.Location);
-
-                if (CoreRuntime.TryGetVersion(out var version) && version >= new Version(5, 0))
-                {
-                    // after the merge of dotnet/corefx and dotnet/coreclr into dotnet/runtime the version should always be the same
-                    Debug.Assert(coreclrAssemblyInfo.FileVersion == corefxAssemblyInfo.FileVersion);
-
-                    return $".NET {version} ({coreclrAssemblyInfo.FileVersion})";
-                }
-                else
-                {
-                    string runtimeVersion = version != default ? version.ToString() : "?";
-
-                    return $".NET Core {runtimeVersion} (CoreCLR {coreclrAssemblyInfo.FileVersion}, CoreFX {corefxAssemblyInfo.FileVersion})";
-                }
+                return GetNetCoreVersion();
             }
             else if (IsNativeAOT)
             {
@@ -233,6 +222,26 @@ internal static string GetRuntimeVersion()
             return Unknown;
         }
 
+        private static string GetNetCoreVersion()
+        {
+            var coreclrAssemblyInfo = FileVersionInfo.GetVersionInfo(typeof(object).GetTypeInfo().Assembly.Location);
+            var corefxAssemblyInfo = FileVersionInfo.GetVersionInfo(typeof(Regex).GetTypeInfo().Assembly.Location);
+
+            if (CoreRuntime.TryGetVersion(out var version) && version >= new Version(5, 0))
+            {
+                // after the merge of dotnet/corefx and dotnet/coreclr into dotnet/runtime the version should always be the same
+                Debug.Assert(coreclrAssemblyInfo.FileVersion == corefxAssemblyInfo.FileVersion);
+
+                return $".NET {version} ({coreclrAssemblyInfo.FileVersion})";
+            }
+            else
+            {
+                string runtimeVersion = version != default ? version.ToString() : "?";
+
+                return $".NET Core {runtimeVersion} (CoreCLR {coreclrAssemblyInfo.FileVersion}, CoreFX {corefxAssemblyInfo.FileVersion})";
+            }
+        }
+
         internal static Runtime GetCurrentRuntime()
         {
             //do not change the order of conditions because it may cause incorrect determination of runtime

src/BenchmarkDotNet/Toolchains/CsProj/CsProjCoreToolchain.cs

@@ -24,7 +24,7 @@ public class CsProjCoreToolchain : Toolchain, IEquatable<CsProjCoreToolchain>
         [PublicAPI] public static readonly IToolchain NetCoreApp60 = From(NetCoreAppSettings.NetCoreApp60);
         [PublicAPI] public static readonly IToolchain NetCoreApp70 = From(NetCoreAppSettings.NetCoreApp70);
 
-        private CsProjCoreToolchain(string name, IGenerator generator, IBuilder builder, IExecutor executor, string customDotNetCliPath)
+        internal CsProjCoreToolchain(string name, IGenerator generator, IBuilder builder, IExecutor executor, string customDotNetCliPath)
             : base(name, generator, builder, executor)
         {
             CustomDotNetCliPath = customDotNetCliPath;

src/BenchmarkDotNet/Toolchains/Mono/MonoGenerator.cs

@@ -0,0 +1,19 @@
+using BenchmarkDotNet.Characteristics;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Toolchains.CsProj;
+
+namespace BenchmarkDotNet.Toolchains.Mono
+{
+    public class MonoGenerator : CsProjGenerator
+    {
+        public MonoGenerator(string targetFrameworkMoniker, string cliPath, string packagesPath, string runtimeFrameworkVersion) : base(targetFrameworkMoniker, cliPath, packagesPath, runtimeFrameworkVersion, true)
+        {
+        }
+
+        protected override string GetRuntimeSettings(GcMode gcMode, IResolver resolver)
+        {
+            // Workaround for 'Found multiple publish output files with the same relative path' error
+            return base.GetRuntimeSettings(gcMode, resolver) + "<PropertyGroup><ErrorOnDuplicatePublishOutputFiles>false</ErrorOnDuplicatePublishOutputFiles></PropertyGroup>";
+        }
+    }
+}

src/BenchmarkDotNet/Toolchains/Mono/MonoPublisher.cs

@@ -0,0 +1,38 @@
+using System.Collections.Generic;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Loggers;
+using BenchmarkDotNet.Running;
+using BenchmarkDotNet.Toolchains.DotNetCli;
+using BenchmarkDotNet.Toolchains.Results;
+
+namespace BenchmarkDotNet.Toolchains.Mono
+{
+    public class MonoPublisher : IBuilder
+    {
+        public MonoPublisher(string customDotNetCliPath)
+        {
+            CustomDotNetCliPath = customDotNetCliPath;
+            var runtimeIdentifier = CustomDotNetCliToolchainBuilder.GetPortableRuntimeIdentifier();
+
+            // /p:RuntimeIdentifiers is set explicitly here because --self-contained requires it, see https://github.com/dotnet/sdk/issues/10566
+            ExtraArguments = $"--self-contained -r {runtimeIdentifier} /p:UseMonoRuntime=true /p:RuntimeIdentifiers={runtimeIdentifier}";
+        }
+
+        private string CustomDotNetCliPath { get; }
+
+        private string ExtraArguments { get; }
+
+        private IReadOnlyList<EnvironmentVariable> EnvironmentVariables { get; }
+
+        public BuildResult Build(GenerateResult generateResult, BuildPartition buildPartition, ILogger logger)
+            => new DotNetCliCommand(
+                    CustomDotNetCliPath,
+                    ExtraArguments,
+                    generateResult,
+                    logger,
+                    buildPartition,
+                    EnvironmentVariables,
+                    buildPartition.Timeout)
+                .Publish().ToBuildResult(generateResult);
+    }
+}

src/BenchmarkDotNet/Toolchains/Mono/MonoToolchain.cs

@@ -0,0 +1,35 @@
+using BenchmarkDotNet.Toolchains.CsProj;
+using BenchmarkDotNet.Toolchains.DotNetCli;
+using JetBrains.Annotations;
+using System;
+
+namespace BenchmarkDotNet.Toolchains.Mono
+{
+    [PublicAPI]
+    public class MonoToolchain : CsProjCoreToolchain, IEquatable<MonoToolchain>
+    {
+        [PublicAPI] public static readonly IToolchain Mono60 = From(new NetCoreAppSettings("net6.0", null, "mono60"));
+        [PublicAPI] public static readonly IToolchain Mono70 = From(new NetCoreAppSettings("net7.0", null, "mono70"));
+
+        private MonoToolchain(string name, IGenerator generator, IBuilder builder, IExecutor executor, string customDotNetCliPath)
+            : base(name, generator, builder, executor, customDotNetCliPath)
+        {
+        }
+
+        [PublicAPI]
+        public static new IToolchain From(NetCoreAppSettings settings)
+        {
+            return new MonoToolchain(settings.Name,
+                        new MonoGenerator(settings.TargetFrameworkMoniker, settings.CustomDotNetCliPath, settings.PackagesPath, settings.RuntimeFrameworkVersion),
+                        new MonoPublisher(settings.CustomDotNetCliPath),
+                        new DotNetCliExecutor(settings.CustomDotNetCliPath),
+                        settings.CustomDotNetCliPath);
+        }
+
+        public override bool Equals(object obj) => obj is MonoToolchain typed && Equals(typed);
+
+        public bool Equals(MonoToolchain other) => Generator.Equals(other.Generator);
+
+        public override int GetHashCode() => Generator.GetHashCode();
+    }
+}

src/BenchmarkDotNet/Toolchains/Toolchain.cs

@@ -39,7 +39,7 @@ public virtual IEnumerable<ValidationError> Validate(BenchmarkCase benchmarkCase
                     benchmarkCase);
             }
 
-            if (runtime is MonoRuntime mono && !benchmarkCase.GetToolchain().IsInProcess)
+            if (runtime is MonoRuntime mono && !mono.IsDotNetBuiltIn && !benchmarkCase.GetToolchain().IsInProcess)
             {
                 if (string.IsNullOrEmpty(mono.CustomPath) && !HostEnvironmentInfo.GetCurrent().IsMonoInstalled.Value)
                 {

src/BenchmarkDotNet/Toolchains/ToolchainExtensions.cs

@@ -48,6 +48,8 @@ internal static IToolchain GetToolchain(this Runtime runtime, Descriptor descrip
                         return InProcessNoEmitToolchain.Instance;
                     if (!string.IsNullOrEmpty(mono.AotArgs))
                         return MonoAotToolchain.Instance;
+                    if (mono.IsDotNetBuiltIn)
+                        return MonoToolchain.From(new NetCoreAppSettings(targetFrameworkMoniker: mono.MsBuildMoniker, runtimeFrameworkVersion: null, name: mono.Name));
 
                     return RoslynToolchain.Instance;
 
@@ -129,6 +131,12 @@ private static IToolchain GetToolchain(RuntimeMoniker runtimeMoniker)
                 case RuntimeMoniker.NativeAot70:
                     return NativeAotToolchain.Net70;
 
+                case RuntimeMoniker.Mono60:
+                    return MonoToolchain.Mono60;
+
+                case RuntimeMoniker.Mono70:
+                    return MonoToolchain.Mono70;
+
                 default:
                     throw new ArgumentOutOfRangeException(nameof(runtimeMoniker), runtimeMoniker, "RuntimeMoniker not supported");
             }

tests/BenchmarkDotNet.IntegrationTests/MonoTests.cs

@@ -0,0 +1,31 @@
+using System;
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Environments;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Tests.XUnit;
+
+namespace BenchmarkDotNet.IntegrationTests
+{
+    public class MonoTests : BenchmarkTestExecutor
+    {
+        [FactDotNetCoreOnly("UseMonoRuntime option is available in .NET Core only starting from .NET 6")]
+        public void Mono60IsSupported()
+        {
+            var config = ManualConfig.CreateEmpty().AddJob(Job.Dry.WithRuntime(MonoRuntime.Mono60));
+            CanExecute<MonoBenchmark>(config);
+        }
+
+        public class MonoBenchmark
+        {
+            [Benchmark]
+            public void Check()
+            {
+                if (Type.GetType("Mono.RuntimeStructs") == null)
+                {
+                    throw new Exception("This is not Mono runtime");
+                }
+            }
+        }
+    }
+}