From 7850312340f5c3dd8e2903270083742a5d75752c Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 8 Apr 2024 16:49:57 +0200 Subject: [PATCH] Do not disable BackgroundCompilation in benchmarks (#9646) * Do not disable BackgroundCompilation in benchmarks * Increase warmup for Table sorting benchmarks. #9470 --- .../org/enso/compiler/benchmarks/Utils.java | 8 ++++++- .../semantic/ArrayProxyBenchmarks.java | 3 ++- .../benchmarks/semantic/AtomBenchmarks.java | 3 ++- .../semantic/CallableBenchmarks.java | 2 +- .../benchmarks/semantic/EqualsBenchmarks.java | 3 ++- .../semantic/IfVsCaseBenchmarks.java | 3 ++- .../benchmarks/semantic/ListBenchmarks.java | 3 ++- .../NamedDefaultedArgumentBenchmarks.java | 2 +- .../NestedPatternCompilationBenchmarks.java | 3 ++- .../semantic/RecursionBenchmarks.java | 2 +- .../bench/benchmarks/semantic/SrcUtil.java | 24 ------------------- .../benchmarks/semantic/StringBenchmarks.java | 3 ++- .../semantic/TypePatternBenchmarks.java | 3 ++- .../benchmarks/semantic/VectorBenchmarks.java | 3 ++- .../semantic/WarningBenchmarks.java | 3 ++- test/Benchmarks/src/Table/Sorting.enso | 7 +++--- 16 files changed, 34 insertions(+), 41 deletions(-) diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/Utils.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/Utils.java index 4b5f31946002..5fa97f05f2eb 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/Utils.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/Utils.java @@ -18,7 +18,9 @@ import org.graalvm.polyglot.Value; import org.graalvm.polyglot.io.IOAccess; -public class Utils { +public final class Utils { + private Utils() {} + public static Context.Builder createDefaultContextBuilder() { return Context.newBuilder() .allowExperimentalOptions(true) @@ -28,6 +30,10 @@ public static Context.Builder createDefaultContextBuilder() { .option(RuntimeOptions.LOG_LEVEL, Level.WARNING.getName()) .option(RuntimeOptions.DISABLE_IR_CACHES, "true") .option(RuntimeOptions.STRICT_ERRORS, "true") + .option("engine.CompilationFailureAction", "Print") + // MultiTier = false does not affect the peak performance, and the graphs generated from + // that are easier to inspect in IGV. + .option("engine.MultiTier", "false") .logHandler(System.err) .allowIO(IOAccess.ALL) .allowAllAccess(true); diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/ArrayProxyBenchmarks.java b/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/ArrayProxyBenchmarks.java index c8d86cecc483..1327a37cee10 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/ArrayProxyBenchmarks.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/ArrayProxyBenchmarks.java @@ -2,6 +2,7 @@ import java.util.concurrent.TimeUnit; import java.util.function.Function; +import org.enso.compiler.benchmarks.Utils; import org.graalvm.polyglot.Value; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; @@ -36,7 +37,7 @@ public class ArrayProxyBenchmarks { @Setup public void initializeBenchmark(BenchmarkParams params) throws Exception { - var ctx = SrcUtil.newContextBuilder().build(); + var ctx = Utils.createDefaultContextBuilder().build(); var code = """ import Standard.Base.Data.Vector.Vector diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/AtomBenchmarks.java b/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/AtomBenchmarks.java index 3f822fbc1947..55c5520c6668 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/AtomBenchmarks.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/AtomBenchmarks.java @@ -2,6 +2,7 @@ import java.io.IOException; import java.util.concurrent.TimeUnit; +import org.enso.compiler.benchmarks.Utils; import org.graalvm.polyglot.Context; import org.graalvm.polyglot.Value; import org.openjdk.jmh.annotations.Benchmark; @@ -201,7 +202,7 @@ public class AtomBenchmarks { @Setup public void initializeBenchmarks(BenchmarkParams params) throws IOException { - this.context = SrcUtil.newContextBuilder().build(); + this.context = Utils.createDefaultContextBuilder().build(); var millionElemListMethod = mainMethod(context, "millionElementList", MILLION_ELEMENT_LIST); this.millionElementsList = millionElemListMethod.execute(); diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/CallableBenchmarks.java b/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/CallableBenchmarks.java index 92a145b74604..e165afa9ed94 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/CallableBenchmarks.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/CallableBenchmarks.java @@ -77,7 +77,7 @@ public class CallableBenchmarks { @Setup public void initializeBenchmarks(BenchmarkParams params) { - this.context = SrcUtil.newContextBuilder().build(); + this.context = org.enso.compiler.benchmarks.Utils.createDefaultContextBuilder().build(); this.sumTCOfromCall = Utils.getMainMethod(context, SUM_TCO_FROM_CALL_CODE); this.sumTCOmethodCall = Utils.getMainMethod(context, SUM_TCO_METHOD_CALL_CODE); diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/EqualsBenchmarks.java b/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/EqualsBenchmarks.java index 2247571dc2f6..b6806a700ee2 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/EqualsBenchmarks.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/EqualsBenchmarks.java @@ -7,6 +7,7 @@ import java.util.Random; import java.util.Set; import java.util.concurrent.TimeUnit; +import org.enso.compiler.benchmarks.Utils; import org.enso.polyglot.MethodNames.Module; import org.graalvm.polyglot.Value; import org.openjdk.jmh.annotations.Benchmark; @@ -53,7 +54,7 @@ public class EqualsBenchmarks { public void initializeBenchmark(BenchmarkParams params) throws Exception { var random = new Random(42); - var ctx = SrcUtil.newContextBuilder().build(); + var ctx = Utils.createDefaultContextBuilder().build(); var benchmarkName = SrcUtil.findName(params); var codeBuilder = diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/IfVsCaseBenchmarks.java b/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/IfVsCaseBenchmarks.java index 097786e2d038..25d22dbdb9d6 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/IfVsCaseBenchmarks.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/IfVsCaseBenchmarks.java @@ -7,6 +7,7 @@ import java.util.List; import java.util.Objects; import java.util.concurrent.TimeUnit; +import org.enso.compiler.benchmarks.Utils; import org.enso.polyglot.MethodNames.Module; import org.graalvm.polyglot.Context; import org.graalvm.polyglot.Value; @@ -43,7 +44,7 @@ public class IfVsCaseBenchmarks { @Setup public void initializeBench(BenchmarkParams params) throws IOException { OutputStream out = new ByteArrayOutputStream(); - ctx = SrcUtil.newContextBuilder().out(out).err(out).build(); + ctx = Utils.createDefaultContextBuilder().out(out).err(out).build(); var code = """ diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/ListBenchmarks.java b/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/ListBenchmarks.java index cdadda6bcc23..da3dfbd6a321 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/ListBenchmarks.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/ListBenchmarks.java @@ -2,6 +2,7 @@ import java.util.concurrent.TimeUnit; import java.util.function.Function; +import org.enso.compiler.benchmarks.Utils; import org.graalvm.polyglot.Value; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; @@ -33,7 +34,7 @@ public class ListBenchmarks { @Setup public void initializeBenchmark(BenchmarkParams params) throws Exception { - var ctx = SrcUtil.newContextBuilder().build(); + var ctx = Utils.createDefaultContextBuilder().build(); var benchmarkName = SrcUtil.findName(params); var code = diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/NamedDefaultedArgumentBenchmarks.java b/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/NamedDefaultedArgumentBenchmarks.java index bcbba410f2a9..8d2abc340672 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/NamedDefaultedArgumentBenchmarks.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/NamedDefaultedArgumentBenchmarks.java @@ -53,7 +53,7 @@ public class NamedDefaultedArgumentBenchmarks { @Setup public void initializeBenchmarks(BenchmarkParams params) { - this.context = SrcUtil.newContextBuilder().build(); + this.context = org.enso.compiler.benchmarks.Utils.createDefaultContextBuilder().build(); this.sumTCOWithNamedArguments = Utils.getMainMethod(context, SUM_TCO_WITH_NAMED_ARGUMENTS_CODE); this.sumTCOWithDefaultedArguments = Utils.getMainMethod(context, SUM_TCO_WITH_DEFAULTED_ARGUMENTS_CODE); diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/NestedPatternCompilationBenchmarks.java b/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/NestedPatternCompilationBenchmarks.java index ac6cab385018..0901c959051d 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/NestedPatternCompilationBenchmarks.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/NestedPatternCompilationBenchmarks.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.util.concurrent.TimeUnit; import java.util.function.Function; +import org.enso.compiler.benchmarks.Utils; import org.graalvm.polyglot.Context; import org.graalvm.polyglot.Value; import org.openjdk.jmh.annotations.Benchmark; @@ -32,7 +33,7 @@ public class NestedPatternCompilationBenchmarks { @Setup public void initializeBenchmark(BenchmarkParams params) throws Exception { - ctx = SrcUtil.newContextBuilder().build(); + ctx = Utils.createDefaultContextBuilder().build(); benchmarkName = SrcUtil.findName(params); code = """ diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/RecursionBenchmarks.java b/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/RecursionBenchmarks.java index 860fd27ed19a..9fe7ae847e7f 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/RecursionBenchmarks.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/RecursionBenchmarks.java @@ -119,7 +119,7 @@ State.put Number (acc + n) @Setup public void initializeBenchmarks(BenchmarkParams params) { - this.context = SrcUtil.newContextBuilder().build(); + this.context = org.enso.compiler.benchmarks.Utils.createDefaultContextBuilder().build(); this.sumTCO = Utils.getMainMethod(context, SUM_TCO_CODE); this.sumTCOWithEval = Utils.getMainMethod(context, SUM_TCO_WITH_EVAL_CODE); diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/SrcUtil.java b/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/SrcUtil.java index 5cc93871214d..bbb33261d5bd 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/SrcUtil.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/SrcUtil.java @@ -3,15 +3,11 @@ import java.io.File; import java.io.FileWriter; import java.io.IOException; -import java.nio.file.Paths; import java.util.Objects; -import java.util.logging.Level; import org.enso.polyglot.MethodNames.Module; -import org.enso.polyglot.RuntimeOptions; import org.graalvm.polyglot.Context; import org.graalvm.polyglot.Source; import org.graalvm.polyglot.Value; -import org.graalvm.polyglot.io.IOAccess; import org.openjdk.jmh.infra.BenchmarkParams; final class SrcUtil { @@ -49,24 +45,4 @@ static Value getMainMethod(Context context, String benchmarkName, String code) } return main; } - - /** - * Typical builder suitable for benchmarking. - * - * @return preconfigured builder to use as a base for benchmarking - */ - static Context.Builder newContextBuilder() { - return Context.newBuilder() - .allowExperimentalOptions(true) - .option(RuntimeOptions.LOG_LEVEL, Level.WARNING.getName()) - .logHandler(System.err) - .allowIO(IOAccess.ALL) - .allowAllAccess(true) - .option("engine.MultiTier", "false") - .option("engine.BackgroundCompilation", "false") - .option("engine.CompilationFailureAction", "Print") - .option( - RuntimeOptions.LANGUAGE_HOME_OVERRIDE, - Paths.get("../../distribution/component").toFile().getAbsolutePath()); - } } diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/StringBenchmarks.java b/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/StringBenchmarks.java index 32286fca8d70..81fbe31c8e0a 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/StringBenchmarks.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/StringBenchmarks.java @@ -2,6 +2,7 @@ import java.util.concurrent.TimeUnit; import java.util.function.Function; +import org.enso.compiler.benchmarks.Utils; import org.graalvm.polyglot.Value; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; @@ -29,7 +30,7 @@ public class StringBenchmarks { @Setup public void initializeBenchmark(BenchmarkParams params) throws Exception { - var ctx = SrcUtil.newContextBuilder().build(); + var ctx = Utils.createDefaultContextBuilder().build(); var code = """ from Standard.Base import all diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/TypePatternBenchmarks.java b/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/TypePatternBenchmarks.java index c1970ad3eb7f..3000527826a2 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/TypePatternBenchmarks.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/TypePatternBenchmarks.java @@ -2,6 +2,7 @@ import java.util.concurrent.TimeUnit; import java.util.function.Function; +import org.enso.compiler.benchmarks.Utils; import org.enso.polyglot.MethodNames.Module; import org.graalvm.polyglot.Value; import org.openjdk.jmh.annotations.Benchmark; @@ -30,7 +31,7 @@ public class TypePatternBenchmarks { @Setup public void initializeBenchmark(BenchmarkParams params) throws Exception { - var ctx = SrcUtil.newContextBuilder().build(); + var ctx = Utils.createDefaultContextBuilder().build(); var code = """ from Standard.Base import Integer, Vector, Any, Float diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/VectorBenchmarks.java b/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/VectorBenchmarks.java index 4a16cda4c840..679ec8f6ee5f 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/VectorBenchmarks.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/VectorBenchmarks.java @@ -3,6 +3,7 @@ import java.util.AbstractList; import java.util.concurrent.TimeUnit; import java.util.function.Function; +import org.enso.compiler.benchmarks.Utils; import org.graalvm.polyglot.Value; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; @@ -30,7 +31,7 @@ public class VectorBenchmarks { @Setup public void initializeBenchmark(BenchmarkParams params) throws Exception { - var ctx = SrcUtil.newContextBuilder().build(); + var ctx = Utils.createDefaultContextBuilder().build(); var benchmarkName = SrcUtil.findName(params); var code = """ diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/WarningBenchmarks.java b/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/WarningBenchmarks.java index 820b01d4ae58..b7d81e156c28 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/WarningBenchmarks.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/WarningBenchmarks.java @@ -6,6 +6,7 @@ import java.util.Objects; import java.util.Random; import java.util.concurrent.TimeUnit; +import org.enso.compiler.benchmarks.Utils; import org.enso.polyglot.MethodNames; import org.graalvm.polyglot.Context; import org.graalvm.polyglot.Value; @@ -67,7 +68,7 @@ private GeneratedVector generateRandomVector( @Setup public void initializeBench(BenchmarkParams params) throws IOException { - this.ctx = SrcUtil.newContextBuilder().build(); + this.ctx = Utils.createDefaultContextBuilder().build(); var random = new Random(42); benchmarkName = SrcUtil.findName(params); diff --git a/test/Benchmarks/src/Table/Sorting.enso b/test/Benchmarks/src/Table/Sorting.enso index def0978e2ed2..f821337db595 100644 --- a/test/Benchmarks/src/Table/Sorting.enso +++ b/test/Benchmarks/src/Table/Sorting.enso @@ -42,13 +42,14 @@ type Data Data.Value create_ints create_dates create_objects create_ints_table create_dates_table create_objects_table -options = Bench.options . set_warmup (Bench.phase_conf 1 7) . set_measure (Bench.phase_conf 1 3) +table_sort_opts = Bench.options . set_warmup (Bench.phase_conf 5 8) . set_measure (Bench.phase_conf 5 3) +vec_sort_opts = Bench.options . set_warmup (Bench.phase_conf 1 7) . set_measure (Bench.phase_conf 1 3) collect_benches = Bench.build builder-> data = Data.create - builder.group "Table_Sorting" options group_builder-> + builder.group "Table_Sorting" table_sort_opts group_builder-> group_builder.specify "table_order_by_ints" <| data.ints_table.order_by [Sort_Column.Index 0] @@ -58,7 +59,7 @@ collect_benches = Bench.build builder-> group_builder.specify "table_order_by_objects" <| data.objects_table.order_by [Sort_Column.Index 0] - builder.group "Vector_Sorting" options group_builder-> + builder.group "Vector_Sorting" vec_sort_opts group_builder-> group_builder.specify "vector_sort_ints" <| data.ints.sort