Explain our Categories policy and enforce it using a Validator (#196)

* introduce the concept of ThirdParty category * intorduce new validator that enforces mandatory category policy * add .NET Framework benchmarks to CoreCLR and CoreFX categories * add 3rd party libs benchmarks to ThirdParty category, 0 errors! * add explanation to the docs * code review fixes
dotnet · Dec 15, 2018 · 0c626c0 · 0c626c0
1 parent 29b47a6
commit 0c626c0
Show file tree

Hide file tree

Showing 34 changed files with 135 additions and 14 deletions.
diff --git a/src/benchmarks/micro/Categories.cs b/src/benchmarks/micro/Categories.cs
@@ -6,6 +6,20 @@ namespace MicroBenchmarks
 {
     public static class Categories
     {
+        /// <summary>
+        /// benchmarks belonging to this category are not going to be executed as part of our daily CI runs
+        /// we are going to run them periodically to make sure we don't regress any of the most popular 3rd party libraries.
+        /// </summary>
+        public const string ThirdParty = "ThirdParty";
+
+        /// <summary>
+        /// benchmarks belonging to this category are executed for CoreFX CI jobs
+        /// </summary>
+        public const string CoreFX = "CoreFX";
+
+        /// <summary>
+        /// benchmarks belonging to this category are executed for CoreCLR CI jobs
+        /// </summary>
         public const string CoreCLR = "CoreCLR";
             public const string BenchmarksGame = "BenchmarksGame";
             public const string Benchstones = "Benchstones";
@@ -15,8 +29,6 @@ public static class Categories
             public const string V8 = "V8";
             public const string Perflab = "Perflab";
             public const string Virtual = "Virtual";
-
-        public const string CoreFX = "CoreFX";
 
         public const string LINQ = "LINQ";
         public const string Reflection = "Reflection";

diff --git a/src/benchmarks/micro/Harness/MandatoryCategoryValidator.cs b/src/benchmarks/micro/Harness/MandatoryCategoryValidator.cs
@@ -0,0 +1,33 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Generic;
+using System.Linq;
+using BenchmarkDotNet.Validators;
+
+namespace MicroBenchmarks
+{
+    /// <summary>
+    /// this class makes sure that every benchmark belongs to either a CoreFX, CoreCLR or ThirdParty category
+    /// for CoreCLR CI jobs we want to run only benchmarks form CoreCLR category
+    /// the same goes for CoreFX
+    /// </summary>
+    public class MandatoryCategoryValidator : IValidator
+    {
+        public static readonly IValidator FailOnError = new MandatoryCategoryValidator();
+
+        public bool TreatsWarningsAsErrors => true;
+
+        public IEnumerable<ValidationError> Validate(ValidationParameters validationParameters)
+            => validationParameters.Benchmarks
+                .Where(benchmark => !benchmark.Descriptor.Categories.Any(category => category == Categories.CoreFX || category == Categories.CoreCLR || category == Categories.ThirdParty))
+                .Select(benchmark => benchmark.Descriptor.GetFilterName())
+                .Distinct()
+                .Select(benchmarkId =>
+                    new ValidationError(
+                        isCritical: TreatsWarningsAsErrors,
+                        $"{benchmarkId} does not belong to one of the mandatory categories: {Categories.CoreCLR}, {Categories.CoreFX}, {Categories.ThirdParty}. Use [BenchmarkCategory(Categories.$)]")
+                );
+    }
+}
diff --git a/src/benchmarks/micro/Program.cs b/src/benchmarks/micro/Program.cs
@@ -35,6 +35,7 @@ private static IConfig GetConfig()
                 .With(new OperatingSystemFilter())
                 .With(JsonExporter.Full) // make sure we export to Json (for BenchView integration purpose)
                 .With(StatisticColumn.Median, StatisticColumn.Min, StatisticColumn.Max)
-                .With(TooManyTestCasesValidator.FailOnError);
+                .With(TooManyTestCasesValidator.FailOnError)
+                .With(MandatoryCategoryValidator.FailOnError);
     }
 }
diff --git a/src/benchmarks/micro/README.md b/src/benchmarks/micro/README.md
@@ -128,6 +128,21 @@ System
  │     └─ReadOnlySpan
 ```
 
+## Categories
+
+Every benchmark should belong to either CoreCLR, CoreFX or ThirdParty library. It allows for proper filtering for CI runs:
+
+* CoreCLR - benchmarks belonging to this category are executed for CoreCLR CI jobs
+* CoreFX - benchmarks belonging to this category are executed for CoreFX CI jobs
+* ThirdParty - benchmarks belonging to this category are not going to be executed as part of our daily CI runs. We are going to run them periodically to make sure we don't regress any of the most popular 3rd party libraries.
+
+Adding given type/method to particular category requires using a `[BenchmarkCategory]` attribute:
+
+```cs
+[BenchmarkCategory(Categories.CoreFX)]
+public class SomeType
+```
+
 ## All Statistics
 
 By default BenchmarkDotNet displays only `Mean`, `Error` and `StdDev` in the results. If you want to see more statistics, please pass `--allStats` as an extra argument to the app: `dotnet run -c Release -f netcoreapp2.1 -- --allStats`. If you build your own config, please use `config.With(StatisticColumn.AllStatistics)`.

diff --git a/src/benchmarks/micro/Serializers/Binary_FromStream.cs b/src/benchmarks/micro/Serializers/Binary_FromStream.cs
@@ -60,20 +60,23 @@ public void SetupMessagePack()
             MessagePack.MessagePackSerializer.Serialize<T>(memoryStream, value);
         }
 
+        [BenchmarkCategory(Categories.CoreFX)]
         [Benchmark(Description = nameof(BinaryFormatter))]
         public T BinaryFormatter_()
         {
             memoryStream.Position = 0;
             return (T)binaryFormatter.Deserialize(memoryStream);
         }
 
+        [BenchmarkCategory(Categories.ThirdParty)]
         [Benchmark(Description = "protobuf-net")]
         public T ProtoBuffNet()
         {
             memoryStream.Position = 0;
             return ProtoBuf.Serializer.Deserialize<T>(memoryStream);
         }
 
+        [BenchmarkCategory(Categories.ThirdParty)]
         [Benchmark(Description = "ZeroFormatter_Naive")]
         public T ZeroFormatter_Naive()
         {
@@ -86,6 +89,7 @@ public T ZeroFormatter_Naive()
         /// they are deserialized for real when they are used for the first time
         /// if we don't touch the properites, they are not being deserialized and the result is skewed
         /// </summary>
+        [BenchmarkCategory(Categories.ThirdParty)]
         [Benchmark(Description = "ZeroFormatter_Real")]
         public long ZeroFormatter_Real()
         {
@@ -96,6 +100,7 @@ public long ZeroFormatter_Real()
             return deserialized.TouchEveryProperty();
         }
 
+        [BenchmarkCategory(Categories.ThirdParty)]
         [Benchmark(Description = "MessagePack")]
         public T MessagePack_()
         {

diff --git a/src/benchmarks/micro/Serializers/Binary_ToStream.cs b/src/benchmarks/micro/Serializers/Binary_ToStream.cs
@@ -32,27 +32,31 @@ public Binary_ToStream()
             ProtoBuf.Meta.RuntimeTypeModel.Default.Add(typeof(DateTimeOffset), false).SetSurrogate(typeof(DateTimeOffsetSurrogate)); // https://stackoverflow.com/a/7046868
         }
 
+        [BenchmarkCategory(Categories.CoreFX)]
         [Benchmark(Description = nameof(BinaryFormatter))]
         public void BinaryFormatter_()
         {
             memoryStream.Position = 0;
             binaryFormatter.Serialize(memoryStream, value);
         }
 
+        [BenchmarkCategory(Categories.ThirdParty)]
         [Benchmark(Description = "protobuf-net")]
         public void ProtoBuffNet()
         {
             memoryStream.Position = 0;
             ProtoBuf.Serializer.Serialize(memoryStream, value);
         }
 
+        [BenchmarkCategory(Categories.ThirdParty)]
         [Benchmark(Description = "ZeroFormatter")]
         public void ZeroFormatter_()
         {
             memoryStream.Position = 0;
             ZeroFormatter.ZeroFormatterSerializer.Serialize<T>(memoryStream, value);
         }
 
+        [BenchmarkCategory(Categories.ThirdParty)]
         [Benchmark(Description = "MessagePack")]
         public void MessagePack_()
         {

diff --git a/src/benchmarks/micro/Serializers/Json_FromStream.cs b/src/benchmarks/micro/Serializers/Json_FromStream.cs
@@ -72,6 +72,7 @@ public void SetupDataContractJsonSerializer_()
             dataContractJsonSerializer.WriteObject(memoryStream, value);
         }
 
+        [BenchmarkCategory(Categories.ThirdParty)]
         [Benchmark(Description = "Jil")]
         public T Jil_()
         {
@@ -81,7 +82,7 @@ public T Jil_()
                 return Jil.JSON.Deserialize<T>(reader);
         }
 
-        [BenchmarkCategory(Categories.CoreCLR, Categories.CoreFX)]
+        [BenchmarkCategory(Categories.CoreCLR, Categories.CoreFX, Categories.ThirdParty)] // JSON.NET is so popular that despite being 3rd Party lib we run the benchmarks for CoreFX and CoreCLR CI
         [Benchmark(Description = "JSON.NET")]
         public T JsonNet_()
         {
@@ -91,6 +92,7 @@ public T JsonNet_()
                 return (T)newtonSoftJsonSerializer.Deserialize(reader, typeof(T));
         }
 
+        [BenchmarkCategory(Categories.ThirdParty)]
         [Benchmark(Description = "Utf8Json")]
         public T Utf8Json_()
         {

diff --git a/src/benchmarks/micro/Serializers/Json_FromString.cs b/src/benchmarks/micro/Serializers/Json_FromString.cs
@@ -27,13 +27,15 @@ public class Json_FromString<T>
         [GlobalSetup(Target = nameof(Utf8Json_))]
         public void SerializeUtf8Json_() => serialized = Utf8Json.JsonSerializer.ToJsonString(value);
 
+        [BenchmarkCategory(Categories.ThirdParty)]
         [Benchmark(Description = "Jil")]
         public T Jil_() => Jil.JSON.Deserialize<T>(serialized);
 
-        [BenchmarkCategory(Categories.CoreCLR, Categories.CoreFX)]
+        [BenchmarkCategory(Categories.CoreCLR, Categories.CoreFX, Categories.ThirdParty)]
         [Benchmark(Description = "JSON.NET")]
         public T JsonNet_() => Newtonsoft.Json.JsonConvert.DeserializeObject<T>(serialized);
 
+        [BenchmarkCategory(Categories.ThirdParty)]
         [Benchmark(Description = "Utf8Json")]
         public T Utf8Json_() => Utf8Json.JsonSerializer.Deserialize<T>(serialized);
     }

diff --git a/src/benchmarks/micro/Serializers/Json_ToStream.cs b/src/benchmarks/micro/Serializers/Json_ToStream.cs
@@ -36,21 +36,23 @@ public Json_ToStream()
             newtonSoftJsonSerializer = new Newtonsoft.Json.JsonSerializer();
         }
 
+        [BenchmarkCategory(Categories.ThirdParty)]
         [Benchmark(Description = "Jil")]
         public void Jil_()
         {
             memoryStream.Position = 0;
             Jil.JSON.Serialize<T>(value, streamWriter);
         }
 
-        [BenchmarkCategory(Categories.CoreCLR, Categories.CoreFX)]
+        [BenchmarkCategory(Categories.CoreCLR, Categories.CoreFX, Categories.ThirdParty)]
         [Benchmark(Description = "JSON.NET")]
         public void JsonNet_()
         {
             memoryStream.Position = 0;
             newtonSoftJsonSerializer.Serialize(streamWriter, value);
         }
 
+        [BenchmarkCategory(Categories.ThirdParty)]
         [Benchmark(Description = "Utf8Json")]
         public void Utf8Json_()
         {

diff --git a/src/benchmarks/micro/Serializers/Json_ToString.cs b/src/benchmarks/micro/Serializers/Json_ToString.cs
@@ -17,13 +17,15 @@ public class Json_ToString<T>
 
         public Json_ToString() => value = DataGenerator.Generate<T>();
 
+        [BenchmarkCategory(Categories.ThirdParty)]
         [Benchmark(Description = "Jil")]
         public string Jil_() => Jil.JSON.Serialize<T>(value);
 
-        [BenchmarkCategory(Categories.CoreCLR, Categories.CoreFX)]
+        [BenchmarkCategory(Categories.CoreCLR, Categories.CoreFX, Categories.ThirdParty)]
         [Benchmark(Description = "JSON.NET")]
         public string JsonNet_() => Newtonsoft.Json.JsonConvert.SerializeObject(value);
 
+        [BenchmarkCategory(Categories.ThirdParty)]
         [Benchmark(Description = "Utf8Json")]
         public string Utf8Json_() => Utf8Json.JsonSerializer.ToJsonString(value);
 

diff --git a/src/benchmarks/micro/Serializers/Xml_FromStream.cs b/src/benchmarks/micro/Serializers/Xml_FromStream.cs
@@ -18,7 +18,6 @@ namespace MicroBenchmarks.Serializers
     [GenericTypeArguments(typeof(XmlElement))]
     [GenericTypeArguments(typeof(SimpleStructWithProperties))]
     [GenericTypeArguments(typeof(ClassImplementingIXmlSerialiable))]
-    [BenchmarkCategory(Categories.CoreFX)]
     public class Xml_FromStream<T>
     {
         private readonly T value;
@@ -48,13 +47,15 @@ public void SetupDataContractSerializer()
             dataContractSerializer.WriteObject(memoryStream, value);
         }
 
+        [BenchmarkCategory(Categories.CoreFX, Categories.CoreCLR)]
         [Benchmark(Description = nameof(XmlSerializer))]
         public T XmlSerializer_()
         {
             memoryStream.Position = 0;
             return (T)xmlSerializer.Deserialize(memoryStream);
         }
 
+        [BenchmarkCategory(Categories.CoreFX)]
         [Benchmark(Description = nameof(DataContractSerializer))]
         public T DataContractSerializer_()
         {

diff --git a/src/benchmarks/micro/Serializers/Xml_ToStream.cs b/src/benchmarks/micro/Serializers/Xml_ToStream.cs
@@ -18,7 +18,6 @@ namespace MicroBenchmarks.Serializers
     [GenericTypeArguments(typeof(XmlElement))]
     [GenericTypeArguments(typeof(SimpleStructWithProperties))]
     [GenericTypeArguments(typeof(ClassImplementingIXmlSerialiable))]
-    [BenchmarkCategory(Categories.CoreFX)]
     public class Xml_ToStream<T>
     {
         private readonly T value;
@@ -34,13 +33,15 @@ public Xml_ToStream()
             memoryStream = new MemoryStream(capacity: short.MaxValue);
         }
 
+        [BenchmarkCategory(Categories.CoreFX, Categories.CoreCLR)]
         [Benchmark(Description = nameof(XmlSerializer))]
         public void XmlSerializer_()
         {
             memoryStream.Position = 0;
             xmlSerializer.Serialize(memoryStream, value);
         }
 
+        [BenchmarkCategory(Categories.CoreFX)]
         [Benchmark(Description = nameof(DataContractSerializer))]
         public void DataContractSerializer_()
         {

diff --git a/src/benchmarks/micro/corefx/System.ComponentModel.TypeConverter/Perf.TypeDescriptorTests.cs b/src/benchmarks/micro/corefx/System.ComponentModel.TypeConverter/Perf.TypeDescriptorTests.cs
@@ -3,9 +3,11 @@
 // See the LICENSE file in the project root for more information.
 
 using BenchmarkDotNet.Attributes;
+using MicroBenchmarks;
 
 namespace System.ComponentModel.Tests
 {
+    [BenchmarkCategory(Categories.CoreFX)]
     public class Perf_TypeDescriptorTests
     {
         [Benchmark]

diff --git a/src/benchmarks/micro/corefx/System.Console/Perf.Console.cs b/src/benchmarks/micro/corefx/System.Console/Perf.Console.cs
@@ -2,9 +2,9 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using System.IO;
 using BenchmarkDotNet.Attributes;
 using BenchmarkDotNet.Engines;
+using MicroBenchmarks;
 
 namespace System.ConsoleTests
 {
@@ -14,6 +14,7 @@ namespace System.ConsoleTests
     /// - OpenStandardInput, OpenStandardOutput, OpenStandardError
     /// - ForegroundColor, BackgroundColor, ResetColor
     /// </summary>
+    [BenchmarkCategory(Categories.CoreFX)]
     public class Perf_Console
     {
         private readonly Consumer consumer = new Consumer();

diff --git a/src/benchmarks/micro/corefx/System.Diagnostics/Perf_Process.cs b/src/benchmarks/micro/corefx/System.Diagnostics/Perf_Process.cs
@@ -3,9 +3,11 @@
 // See the LICENSE file in the project root for more information.
 
 using BenchmarkDotNet.Attributes;
+using MicroBenchmarks;
 
 namespace System.Diagnostics
 {
+    [BenchmarkCategory(Categories.CoreFX)]
     public class Perf_Process
     {
         private readonly string _nonExistingName = Guid.NewGuid().ToString();

diff --git a/src/benchmarks/micro/corefx/System.Globalization/Perf.CompareInfo.cs b/src/benchmarks/micro/corefx/System.Globalization/Perf.CompareInfo.cs
@@ -4,9 +4,11 @@
 
 using System.Collections.Generic;
 using BenchmarkDotNet.Attributes;
+using MicroBenchmarks;
 
 namespace System.Globalization.Tests
 {
+    [BenchmarkCategory(Categories.CoreFX)]
     public class Perf_CompareInfo
     {
         private static string GenerateInputString(char source, int count, char replaceChar, int replacePos)

diff --git a/src/benchmarks/micro/corefx/System.Globalization/Perf.CultureInfo.cs b/src/benchmarks/micro/corefx/System.Globalization/Perf.CultureInfo.cs
@@ -3,9 +3,11 @@
 // See the LICENSE file in the project root for more information.
 
 using BenchmarkDotNet.Attributes;
+using MicroBenchmarks;
 
 namespace System.Globalization.Tests
 {
+    [BenchmarkCategory(Categories.CoreFX)]
     public class Perf_CultureInfo
     {
         [Benchmark]

diff --git a/src/benchmarks/micro/corefx/System.Globalization/Perf.DateTimeCultureInfo.cs b/src/benchmarks/micro/corefx/System.Globalization/Perf.DateTimeCultureInfo.cs
@@ -4,6 +4,7 @@
 
 using System.Collections.Generic;
 using BenchmarkDotNet.Attributes;
+using MicroBenchmarks;
 
 namespace System.Globalization.Tests
 {
@@ -12,6 +13,7 @@ namespace System.Globalization.Tests
     /// 
     /// Primary methods affected: Parse, ToString
     /// </summary>
+    [BenchmarkCategory(Categories.CoreFX)]
     public class Perf_DateTimeCultureInfo
     {
         private readonly DateTime _time = DateTime.Now;

diff --git a/src/benchmarks/micro/corefx/System.Globalization/Perf.NumberCultureInfo.cs b/src/benchmarks/micro/corefx/System.Globalization/Perf.NumberCultureInfo.cs
@@ -4,12 +4,14 @@
 
 using System.Collections.Generic;
 using BenchmarkDotNet.Attributes;
+using MicroBenchmarks;
 
 namespace System.Globalization.Tests
 {
     /// <summary>
     /// Performance tests for converting numbers to different CultureInfos
     /// </summary>
+    [BenchmarkCategory(Categories.CoreFX)]
     public class Perf_NumberCultureInfo
     {
         public IEnumerable<object> Cultures()