From 8a7aa24a94f6e983c05e75d0064884221978e8f7 Mon Sep 17 00:00:00 2001 From: Michael Mior Date: Thu, 22 Sep 2022 07:44:48 -0400 Subject: [PATCH 1/4] Add support for Java via JMH --- .github/workflows/ci.yml | 37 ++++ .github/workflows/java.yml | 56 ++++++ .gitignore | 2 + README.md | 7 +- examples/java/pom.xml | 172 ++++++++++++++++++ .../jmh/samples/JMHSample_01_HelloWorld.java | 92 ++++++++++ src/config.ts | 1 + src/default_index_html.ts | 1 + src/extract.ts | 58 ++++++ src/write.ts | 2 + test/data/extract/jmh_output.json | 53 ++++++ test/extract.spec.ts | 14 +- 12 files changed, 493 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/java.yml create mode 100644 examples/java/pom.xml create mode 100644 examples/java/src/main/java/org/openjdk/jmh/samples/JMHSample_01_HelloWorld.java create mode 100644 test/data/extract/jmh_output.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d04c3021d..accbc8be2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -293,6 +293,43 @@ jobs: fail-on-alert: true - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.Net Benchmark' + jmh: + name: Run JMH Java Benchmark Framework example + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v1 + - uses: actions/setup-java@v2 + with: + distribution: 'adopt' + java-version: '11' + - uses: actions/cache@v1 + with: + path: ~/.npm + key: ${{ runner.os }}-node-${{ hashFiles('package-lock.json') }} + - run: npm ci + - run: npm run build + - name: Save previous data.js + run: | + git fetch origin gh-pages + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - + - name: Run benchmark + run: | + cd examples/java + mvn clean verify + java -jar target/benchmarks.jar -wi 1 -i 3 -f 1 -rf json + - name: Store benchmark result + uses: ./ + with: + name: JMH Benchmark + tool: "jmh" + output-file-path: examples/java/jmh-result.json + skip-fetch-gh-pages: true + fail-on-alert: true + - run: node ./scripts/ci_validate_modification.js before_data.js 'JMH Benchmark' + only-alert-with-cache: name: Run alert check with actions/cache runs-on: ubuntu-latest diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml new file mode 100644 index 000000000..fa7763cc9 --- /dev/null +++ b/.github/workflows/java.yml @@ -0,0 +1,56 @@ +name: JMH Example +on: + push: + branches: + - master + +permissions: + contents: write + deployments: write + +jobs: + benchmark: + name: Run JMH benchmark example + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v2 + with: + distribution: 'adopt' + java-version: '11' + - name: Run benchmark + run: | + cd examples/java + mvn clean verify + java -jar target/benchmarks.jar -wi 1 -i 3 -f 1 -rf json + + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: JMH Benchmark + tool: 'jmh' + output-file-path: examples/java/jmh-result.json + # Use personal access token instead of GITHUB_TOKEN due to https://github.community/t/github-action-not-triggering-gh-pages-upon-push/16096 + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + # Show alert with commit comment on detecting possible performance regression + alert-threshold: '200%' + comment-on-alert: true + fail-on-alert: true + alert-comment-cc-users: '@michaelmior' + + - name: Store benchmark result - separate results repo + uses: benchmark-action/github-action-benchmark@v1 + with: + name: JMH Benchmark + tool: 'jmh' + output-file-path: examples/java/jmh-result.json + # Use personal access token instead of GITHUB_TOKEN due to https://github.community/t/github-action-not-triggering-gh-pages-upon-push/16096 + github-token: ${{ secrets.BENCHMARK_ACTION_BOT_TOKEN }} + auto-push: true + # Show alert with commit comment on detecting possible performance regression + alert-threshold: '200%' + comment-on-alert: true + fail-on-alert: true + alert-comment-cc-users: '@michaelmior' + gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' diff --git a/.gitignore b/.gitignore index 22f48fa89..c6a69e261 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,5 @@ /coverage /dist /.idea +target/ +jmh-result.json diff --git a/README.md b/README.md index 2e6d8437d..d2cd4c77b 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ This action currently supports the following tools: - [BenchmarkTools.jl][] for Julia packages - [Benchmark.Net][benchmarkdotnet] for .Net projects - [benchmarkluau](https://github.com/Roblox/luau/tree/master/bench) for Luau projects +- [JMH][jmh] for Java projects - Custom benchmarks where either 'biggerIsBetter' or 'smallerIsBetter' Multiple languages in the same repository are supported for polyglot projects. @@ -48,6 +49,7 @@ definitions are in [.github/workflows/](./.github/workflows) directory. Live wor | C++ (Catch2) | [![C++ Catch2 Example Workflow][catch2-badge]][catch2-workflow-example] | [examples/catch2](./examples/catch2) | | Julia | [![Julia Example][julia-badge]][julia-workflow-example] | [examples/julia](./examples/julia) | | .Net | [![C# Benchmark.Net Example Workflow][benchmarkdotnet-badge]][benchmarkdotnet-workflow-example] | [examples/benchmarkdotnet](./examples/benchmarkdotnet) | +| Java | [![Java Example Workflow][java-badge]][java-workflow-example] | [examples/java](./examples/java) | | Luau | Coming soon | Coming soon | All benchmark charts from above workflows are gathered in GitHub pages: @@ -354,7 +356,7 @@ Name of the benchmark. This value must be identical across all benchmarks in you - Default: N/A Tool for running benchmark. The value must be one of `"cargo"`, `"go"`, `"benchmarkjs"`, `"pytest"`, -`"googlecpp"`, `"catch2"`, `"julia"`, `"benchmarkdotnet"`,`"benchmarkluau"`, `"customBiggerIsBetter"`, `"customSmallerIsBetter"`. +`"googlecpp"`, `"catch2"`, `"julia"`, "jmh", `"benchmarkdotnet"`,`"benchmarkluau"`, `"customBiggerIsBetter"`, `"customSmallerIsBetter"`. #### `output-file-path` (Required) @@ -600,6 +602,7 @@ Every release will appear on your GitHub notifications page. [cpp-badge]: https://github.com/benchmark-action/github-action-benchmark/workflows/C%2B%2B%20Example/badge.svg [catch2-badge]: https://github.com/benchmark-action/github-action-benchmark/workflows/Catch2%20C%2B%2B%20Example/badge.svg [julia-badge]: https://github.com/benchmark-action/github-action-benchmark/workflows/Julia%20Example%20with%20BenchmarkTools.jl/badge.svg +[java-badge]: https://github.com/benchmark-action/github-action-benchmark/workflows/JMH%20Example/badge.svg [github-action]: https://github.com/features/actions [cargo-bench]: https://doc.rust-lang.org/cargo/commands/cargo-bench.html [benchmarkjs]: https://benchmarkjs.com/ @@ -615,12 +618,14 @@ Every release will appear on your GitHub notifications page. [cpp-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22C%2B%2B+Example%22 [catch2-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Catch2+C%2B%2B+Example%22 [julia-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Julia+Example+with+BenchmarkTools.jl%22 +[java-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22JMH+Example%22 [help-watch-release]: https://docs.github.com/en/github/receiving-notifications-about-activity-on-github/watching-and-unwatching-releases-for-a-repository [help-github-token]: https://docs.github.com/en/actions/security-guides/automatic-token-authentication [minimal-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Example+for+minimal+setup [commit-comment-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Example+for+alert+with+commit+comment [google-benchmark]: https://github.com/google/benchmark [catch2]: https://github.com/catchorg/Catch2 +[jmh]: https://openjdk.java.net/projects/code-tools/jmh/ [lighthouse-ci-action]: https://github.com/treosh/lighthouse-ci-action [lighthouse-ci]: https://github.com/GoogleChrome/lighthouse-ci [BenchmarkTools.jl]: https://github.com/JuliaCI/BaseBenchmarks.jl diff --git a/examples/java/pom.xml b/examples/java/pom.xml new file mode 100644 index 000000000..fd6f342d2 --- /dev/null +++ b/examples/java/pom.xml @@ -0,0 +1,172 @@ + + + + 4.0.0 + + org.openjdk.jmh.samples + jmh-sample + 1.0 + jar + + JMH benchmark sample: Java + + + + + + org.openjdk.jmh + jmh-core + ${jmh.version} + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + provided + + + + + UTF-8 + + + 1.29 + + + 1.8 + + + benchmarks + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.0 + + ${javac.target} + ${javac.target} + ${javac.target} + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.1 + + + package + + shade + + + ${uberjar.name} + + + org.openjdk.jmh.Main + + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + + + maven-clean-plugin + 2.5 + + + maven-deploy-plugin + 2.8.1 + + + maven-install-plugin + 2.5.1 + + + maven-jar-plugin + 2.4 + + + maven-javadoc-plugin + 2.9.1 + + + maven-resources-plugin + 2.6 + + + maven-site-plugin + 3.3 + + + maven-source-plugin + 2.2.1 + + + maven-surefire-plugin + 2.17 + + + + + + diff --git a/examples/java/src/main/java/org/openjdk/jmh/samples/JMHSample_01_HelloWorld.java b/examples/java/src/main/java/org/openjdk/jmh/samples/JMHSample_01_HelloWorld.java new file mode 100644 index 000000000..cd7af888d --- /dev/null +++ b/examples/java/src/main/java/org/openjdk/jmh/samples/JMHSample_01_HelloWorld.java @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.jmh.samples; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +public class JMHSample_01_HelloWorld { + + /* + * This is our first benchmark method. + * + * The contract for the benchmark methods is very simple: + * annotate it with @Benchmark, and you are set to go. + * JMH will run the test by continuously calling this method, and measuring + * the performance metrics for its execution. + * + * The method names are non-essential, it matters they are marked with + * @Benchmark. You can have multiple benchmark methods + * within the same class. + * + * Note: if the benchmark method never finishes, then JMH run never + * finishes as well. If you throw the exception from the method body, + * the JMH run ends abruptly for this benchmark, and JMH will run + * the next benchmark down the list. + * + * Although this benchmark measures "nothing", it is the good showcase + * for the overheads the infrastructure bear on the code you measure + * in the method. There are no magical infrastructures which incur no + * overhead, and it's important to know what are the infra overheads + * you are dealing with. You might find this thought unfolded in future + * examples by having the "baseline" measurements to compare against. + */ + + @Benchmark + public void wellHelloThere() { + // this method was intentionally left blank. + } + + /* + * ============================== HOW TO RUN THIS TEST: ==================================== + * + * You are expected to see the run with large number of iterations, and + * very large throughput numbers. You can see that as the estimate of the + * harness overheads per method call. In most of our measurements, it is + * down to several cycles per call. + * + * a) Via command-line: + * $ mvn clean install + * $ java -jar target/benchmarks.jar ".*JMHSample_01.*" + * + * JMH generates self-contained JARs, bundling JMH together with it. + * The runtime options for the JMH are available with "-h": + * $ java -jar target/benchmarks.jar -h + * + * b) Via Java API: + */ + + public static void main(String[] args) throws RunnerException { + Options opt = new OptionsBuilder() + .include(".*" + JMHSample_01_HelloWorld.class.getSimpleName() + ".*") + .forks(1) + .build(); + + new Runner(opt).run(); + } +} diff --git a/src/config.ts b/src/config.ts index d6cf537d9..d1a4919b8 100644 --- a/src/config.ts +++ b/src/config.ts @@ -34,6 +34,7 @@ export const VALID_TOOLS = [ 'googlecpp', 'catch2', 'julia', + 'jmh', 'benchmarkdotnet', 'customBiggerIsBetter', 'customSmallerIsBetter', diff --git a/src/default_index_html.ts b/src/default_index_html.ts index 142f135d5..a6246d51b 100644 --- a/src/default_index_html.ts +++ b/src/default_index_html.ts @@ -119,6 +119,7 @@ export const DEFAULT_INDEX_HTML = String.raw` googlecpp: '#f34b7d', catch2: '#f34b7d', julia: '#a270ba', + jmh: '#b07219', benchmarkdotnet: '#178600', customBiggerIsBetter: '#38ff38', customSmallerIsBetter: '#ff3838', diff --git a/src/extract.ts b/src/extract.ts index 49e4349e2..1b6b2ff25 100644 --- a/src/extract.ts +++ b/src/extract.ts @@ -171,6 +171,44 @@ type JuliaBenchmarkGroup = [ type JuliaBenchmarkJson = [object, JuliaBenchmarkGroup[]]; +export interface JmhBenchmarkJson { + jmhVersion: string; + benchmark: string; + mode: string; + threads: number; + forks: number; + jvm: string; + jvmArgs: string[]; + jdkVersion: string; + vmName: string; + vmVersion: string; + warmupIterations: number; + warmupTime: string; + warmupBatchSize: number; + measurementIterations: number; + measurementTime: string; + measurementBatchSize: number; + primaryMetric: { + score: number; + scoreError: number; + scoreConfidence: number[]; + scorePercentiles: { + 0.0: number; + 50.0: number; + 90.0: number; + 95.0: number; + 99.0: number; + 99.9: number; + 99.99: number; + 99.999: number; + 99.9999: number; + 100.0: number; + }; + scoreUnit: string; + rawData: number[][]; + }; +} + export interface BenchmarkDotnetBenchmark { FullName: string; Statistics: { @@ -561,6 +599,22 @@ function extractJuliaBenchmarkResult(output: string): BenchmarkResult[] { return res; } +function extractJmhResult(output: string): BenchmarkResult[] { + let json: JmhBenchmarkJson[]; + try { + json = JSON.parse(output); + } catch (err: any) { + throw new Error(`Output file for 'jmh' must be JSON file generated by -rf json option: ${err.message}`); + } + return json.map((b) => { + const name = b.benchmark; + const value = b.primaryMetric.score; + const unit = b.primaryMetric.scoreUnit; + const extra = `iterations: ${b.measurementIterations}\nforks: ${b.forks}\nthreads: ${b.threads}`; + return { name, value, unit, extra }; + }); +} + function extractBenchmarkDotnetResult(output: string): BenchmarkResult[] { let json: BenchmarkDotNetBenchmarkJson; try { @@ -641,6 +695,10 @@ export async function extractResult(config: Config): Promise { case 'julia': benches = extractJuliaBenchmarkResult(output); break; + break; + case 'jmh': + benches = extractJmhResult(output); + break; case 'benchmarkdotnet': benches = extractBenchmarkDotnetResult(output); break; diff --git a/src/write.ts b/src/write.ts index d7a60f779..d55bb1fa2 100644 --- a/src/write.ts +++ b/src/write.ts @@ -75,6 +75,8 @@ function biggerIsBetter(tool: ToolType): boolean { return false; case 'julia': return false; + case 'jmh': + return false; case 'benchmarkdotnet': return false; case 'customBiggerIsBetter': diff --git a/test/data/extract/jmh_output.json b/test/data/extract/jmh_output.json new file mode 100644 index 000000000..d9714b285 --- /dev/null +++ b/test/data/extract/jmh_output.json @@ -0,0 +1,53 @@ +[ + { + "jmhVersion" : "1.29", + "benchmark" : "org.openjdk.jmh.samples.JMHSample_01_HelloWorld.wellHelloThere", + "mode" : "thrpt", + "threads" : 1, + "forks" : 1, + "jvm" : "/home/mmior/.jabba/jdk/1.8.202/jre/bin/java", + "jvmArgs" : [ + ], + "jdkVersion" : "1.8.0_202", + "vmName" : "Java HotSpot(TM) 64-Bit Server VM", + "vmVersion" : "25.202-b08", + "warmupIterations" : 1, + "warmupTime" : "10 s", + "warmupBatchSize" : 1, + "measurementIterations" : 3, + "measurementTime" : "10 s", + "measurementBatchSize" : 1, + "primaryMetric" : { + "score" : 3.3762388731228185E9, + "scoreError" : 1.4287985743935993E7, + "scoreConfidence" : [ + 3.3619508873788824E9, + 3.3905268588667545E9 + ], + "scorePercentiles" : { + "0.0" : 3.375353964729294E9, + "50.0" : 3.37651988895058E9, + "90.0" : 3.376842765688582E9, + "95.0" : 3.376842765688582E9, + "99.0" : 3.376842765688582E9, + "99.9" : 3.376842765688582E9, + "99.99" : 3.376842765688582E9, + "99.999" : 3.376842765688582E9, + "99.9999" : 3.376842765688582E9, + "100.0" : 3.376842765688582E9 + }, + "scoreUnit" : "ops/s", + "rawData" : [ + [ + 3.376842765688582E9, + 3.37651988895058E9, + 3.375353964729294E9 + ] + ] + }, + "secondaryMetrics" : { + } + } +] + + diff --git a/test/extract.spec.ts b/test/extract.spec.ts index 1532d1af0..b61bf681d 100644 --- a/test/extract.spec.ts +++ b/test/extract.spec.ts @@ -361,6 +361,18 @@ describe('extractResult()', function () { }, ], }, + { + tool: 'jmh', + file: 'jmh_output.json', + expected: [ + { + extra: 'iterations: 3\nforks: 1\nthreads: 1', + name: 'org.openjdk.jmh.samples.JMHSample_01_HelloWorld.wellHelloThere', + unit: 'ops/s', + value: 3.3762388731228185e9, + }, + ], + }, { tool: 'benchmarkdotnet', file: 'benchmarkdotnet.json', @@ -466,7 +478,7 @@ describe('extractResult()', function () { file: string; expected: RegExp; }> = [ - ...(['pytest', 'googlecpp', 'customBiggerIsBetter', 'customSmallerIsBetter'] as const).map((tool) => ({ + ...(['pytest', 'googlecpp', 'jmh', 'customBiggerIsBetter', 'customSmallerIsBetter'] as const).map((tool) => ({ it: `raises an error when output file is not in JSON with tool '${tool}'`, tool, file: 'go_output.txt', From 8953b62f45a66f5bef16634752b7203afbb11505 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Chris=20Trze=C5=9Bniewski?= Date: Thu, 22 Sep 2022 15:15:32 +0200 Subject: [PATCH 2/4] Update .github/workflows/ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index accbc8be2..28e021fe6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -328,7 +328,7 @@ jobs: output-file-path: examples/java/jmh-result.json skip-fetch-gh-pages: true fail-on-alert: true - - run: node ./scripts/ci_validate_modification.js before_data.js 'JMH Benchmark' + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'JMH Benchmark' only-alert-with-cache: name: Run alert check with actions/cache From 667cda511a1021bab5907ed0163fd461a773e25e Mon Sep 17 00:00:00 2001 From: Michael Mior Date: Thu, 22 Sep 2022 12:01:08 -0400 Subject: [PATCH 3/4] Update node version for CI --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 28e021fe6..fcfb21a55 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -298,7 +298,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: actions/setup-node@v1 + - uses: actions/setup-node@v3 + with: + node-version: 16 - uses: actions/setup-java@v2 with: distribution: 'adopt' From c0760fb22de521ae5a5db9496c3f9ce64aeef91b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Chris=20Trze=C5=9Bniewski?= Date: Thu, 3 Nov 2022 08:44:12 +0100 Subject: [PATCH 4/4] Use actions/checkout@v3 --- .github/workflows/ci.yml | 2 +- .github/workflows/java.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fcfb21a55..66c58e2a9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -297,7 +297,7 @@ jobs: name: Run JMH Java Benchmark Framework example runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: node-version: 16 diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index fa7763cc9..2094678ea 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -13,7 +13,7 @@ jobs: name: Run JMH benchmark example runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: actions/setup-java@v2 with: distribution: 'adopt'