Merge pull request #688 from dart-lang/merge-benchmark_harness-package

Merge `package:benchmark_harness`

.github/ISSUE_TEMPLATE/benchmark_harness.md

@@ -0,0 +1,5 @@
+---
+name: "package:benchmark_harness"
+about: "Create a bug or file a feature request against package:benchmark_harness."
+labels: "package:benchmark_harness"
+---

.github/labeler.yml

@@ -8,6 +8,10 @@
   - changed-files:
       - any-glob-to-any-file: 'pkgs/bazel_worker/**'
 
+'package:benchmark_harness':
+  - changed-files:
+      - any-glob-to-any-file: 'pkgs/benchmark_harness/**'
+
 'package:boolean_selector':
   - changed-files:
       - any-glob-to-any-file: 'pkgs/boolean_selector/**'

.github/workflows/benchmark_harness.yaml

@@ -0,0 +1,74 @@
+name: package:benchmark_harness
+
+on:
+  # Run on PRs and pushes to the default branch.
+  push:
+    branches: [ main ]
+    paths:
+      - '.github/workflows/benchmark_harness.yml'
+      - 'pkgs/benchmark_harness/**'
+  pull_request:
+    branches: [ main ]
+    paths:
+      - '.github/workflows/benchmark_harness.yml'
+      - 'pkgs/benchmark_harness/**'
+  schedule:
+    - cron: "0 0 * * 0"
+
+env:
+  PUB_ENVIRONMENT: bot.github
+
+defaults:
+  run:
+    working-directory: pkgs/benchmark_harness/
+
+jobs:
+  # Check code formatting and static analysis on a single OS (linux)
+  # against Dart dev.
+  analyze:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        sdk: [dev]
+    steps:
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938
+      - uses: dart-lang/setup-dart@0a8a0fc875eb934c15d08629302413c671d3f672
+        with:
+          sdk: ${{ matrix.sdk }}
+      - id: install
+        name: Install dependencies
+        run: dart pub get
+      - name: Check formatting
+        run: dart format --output=none --set-exit-if-changed .
+        if: always() && steps.install.outcome == 'success'
+      - name: Analyze code
+        run: dart analyze --fatal-infos
+        if: always() && steps.install.outcome == 'success'
+
+  # Run tests on a matrix consisting of two dimensions:
+  # 1. OS: ubuntu-latest, (macos-latest, windows-latest)
+  # 2. release channel: dev
+  test:
+    needs: analyze
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        # Add macos-latest and/or windows-latest if relevant for this package.
+        os: [ubuntu-latest]
+        sdk: [3.2, dev]
+    steps:
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938
+      - uses: dart-lang/setup-dart@0a8a0fc875eb934c15d08629302413c671d3f672
+        with:
+          sdk: ${{ matrix.sdk }}
+      - id: install
+        name: Install dependencies
+        run: dart pub get
+      - name: Run VM tests
+        run: dart test --platform vm
+        if: always() && steps.install.outcome == 'success'
+      - name: Run Chrome tests
+        run: dart test --platform chrome
+        if: always() && steps.install.outcome == 'success'

README.md

@@ -15,6 +15,7 @@ don't naturally belong to other topic monorepos (like
 | Package | Description | Version |
 | --- | --- | --- |
 | [bazel_worker](pkgs/bazel_worker/) | Protocol and utilities to implement or invoke persistent bazel workers. | [![pub package](https://img.shields.io/pub/v/bazel_worker.svg)](https://pub.dev/packages/bazel_worker) |
+| [benchmark_harness](pkgs/benchmark_harness/) | The official Dart project benchmark harness. | [![pub package](https://img.shields.io/pub/v/benchmark_harness.svg)](https://pub.dev/packages/benchmark_harness) |
 | [boolean_selector](pkgs/boolean_selector/) | A flexible syntax for boolean expressions, based on a simplified version of Dart's expression syntax. | [![pub package](https://img.shields.io/pub/v/boolean_selector.svg)](https://pub.dev/packages/boolean_selector) |
 | [browser_launcher](pkgs/browser_launcher/) | Provides a standardized way to launch web browsers for testing and tools. | [![pub package](https://img.shields.io/pub/v/browser_launcher.svg)](https://pub.dev/packages/browser_launcher) |
 | [cli_config](pkgs/cli_config/) | A library to take config values from configuration files, CLI arguments, and environment variables. | [![pub package](https://img.shields.io/pub/v/cli_config.svg)](https://pub.dev/packages/cli_config) |

pkgs/benchmark_harness/.gitignore

@@ -0,0 +1,4 @@
+.dart_tool
+.packages
+.pub
+pubspec.lock

pkgs/benchmark_harness/CHANGELOG.md

@@ -0,0 +1,50 @@
+## 2.3.1
+
+- Move to `dart-lang/tools` monorepo.
+
+## 2.3.0
+
+- Require Dart 3.2.
+- Add ScoreEmitterV2 interface, documented with the intention to change
+ScoreEmitter interface to match it in the next major release,
+ a breaking change.
+- Add `PerfBenchmarkBase` class which runs the 'perf stat' command from
+linux-tools on a benchmark and reports metrics from the hardware
+performance counters and the iteration count, as well as the run time
+measurement reported by `BenchmarkBase`.
+
+## 2.2.2
+
+- Added package topics to the pubspec file.
+- Require Dart 2.19.
+
+## 2.2.1
+
+- Improve convergence speed of `BenchmarkBase` measuring algorithm by allowing
+some degree of measuring jitter.
+
+## 2.2.0
+
+- Change measuring algorithm in `BenchmarkBase` to avoid calling stopwatch
+methods repeatedly in the measuring loop. This makes measurement work better
+for `run` methods which are small themselves.
+
+## 2.1.0
+
+- Add AsyncBenchmarkBase.
+
+## 2.0.0
+
+- Stable null safety release.
+
+## 2.0.0-nullsafety.0
+
+- Opt in to null safety.
+
+## 1.0.6
+
+- Require at least Dart 2.1.
+
+## 1.0.5
+
+- Updates to support Dart 2.

pkgs/benchmark_harness/LICENSE

@@ -0,0 +1,26 @@
+Copyright 2021, the Dart project authors.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above
+      copyright notice, this list of conditions and the following
+      disclaimer in the documentation and/or other materials provided
+      with the distribution.
+    * Neither the name of Google LLC nor the names of its
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

pkgs/benchmark_harness/README.md

@@ -0,0 +1,116 @@
+[![Build Status](https://github.com/dart-lang/tools/actions/workflows/benchmark_harness.yaml/badge.svg)](https://github.com/dart-lang/tools/actions/workflows/benchmark_harness.yaml)
+[![pub package](https://img.shields.io/pub/v/benchmark_harness.svg)](https://pub.dev/packages/benchmark_harness)
+[![package publisher](https://img.shields.io/pub/publisher/benchmark_harness.svg)](https://pub.dev/packages/benchmark_harness/publisher)
+
+The Dart project benchmark harness is the recommended starting point when
+building a benchmark for Dart.
+
+## Interpreting Results
+
+By default, the reported runtime in `BenchmarkBase` is not for a single call to
+`run()`, but for the average time it takes to call `run()` __10 times__ for
+legacy reasons. The benchmark harness executes a 10-call timing loop repeatedly
+until 2 seconds have elapsed; the reported result is the average of the runtimes
+for each loop. This behavior will change in a future major version.
+
+Benchmarks extending `BenchmarkBase` can opt into the reporting the average time
+to call `run()` once by overriding the `exercise` method:
+
+```dart
+  @override
+  void exercise() => run();
+```
+
+`AsyncBenchmarkBase` already reports the average time to call `run()` __once__.
+
+## Comparing Results
+
+If you are running the same benchmark, on the same machine, running the same OS,
+the reported run times can be carefully compared across runs.
+Carefully because there are a variety of factors which
+could cause error in the run time, for example, the load from
+other applications running on your machine could alter the result.
+
+Comparing the run time of different benchmarks is not recommended. 
+In other words, don't compare apples with oranges.
+
+## Features
+
+* `BenchmarkBase` class that all new benchmarks should `extend`.
+* `AsyncBenchmarkBase` for asynchronous benchmarks.
+* Template benchmark that you can copy and paste when building new benchmarks.
+
+## Getting Started
+
+1\. Add the following to your project's **pubspec.yaml**
+
+```yaml
+dependencies:
+    benchmark_harness: any
+```
+
+2\. Install pub packages
+
+```sh
+dart pub install
+```
+
+3\. Add the following import:
+
+```dart
+import 'package:benchmark_harness/benchmark_harness.dart';
+```
+
+4\. Create a benchmark class which inherits from `BenchmarkBase` or
+    `AsyncBenchmarkBase`.
+
+## Example
+
+Create a dart file in the
+[`benchmark/`](https://dart.dev/tools/pub/package-layout#tests-and-benchmarks)
+folder of your package.
+
+```dart
+// Import BenchmarkBase class.
+import 'package:benchmark_harness/benchmark_harness.dart';
+
+// Create a new benchmark by extending BenchmarkBase
+class TemplateBenchmark extends BenchmarkBase {
+  const TemplateBenchmark() : super('Template');
+
+  static void main() {
+    const TemplateBenchmark().report();
+  }
+
+  // The benchmark code.
+  @override
+  void run() {}
+
+  // Not measured setup code executed prior to the benchmark runs.
+  @override
+  void setup() {}
+
+  // Not measured teardown code executed after the benchmark runs.
+  @override
+  void teardown() {}
+
+  // To opt into the reporting the time per run() instead of per 10 run() calls.
+  //@override
+  //void exercise() => run();
+}
+
+void main() {
+  // Run TemplateBenchmark
+  TemplateBenchmark.main();
+}
+```
+
+### Output
+
+```console
+Template(RunTime): 0.1568472448997197 us.
+```
+
+This is the average amount of time it takes to run `run()` 10 times for
+`BenchmarkBase` and once for `AsyncBenchmarkBase`.
+> µs is an abbreviation for microseconds.

pkgs/benchmark_harness/analysis_options.yaml

@@ -0,0 +1,14 @@
+include: package:dart_flutter_team_lints/analysis_options.yaml
+
+analyzer:
+  language:
+    strict-inference: true
+    strict-casts: true
+
+linter:
+  rules:
+    - avoid_unused_constructor_parameters
+    - cancel_subscriptions
+    - literal_only_boolean_expressions
+    - no_adjacent_strings_in_list
+    - unnecessary_await_in_return

pkgs/benchmark_harness/example/template.dart

@@ -0,0 +1,36 @@
+// Copyright (c) 2021, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+// Import BenchmarkBase class.
+import 'package:benchmark_harness/benchmark_harness.dart';
+
+// Create a new benchmark by extending BenchmarkBase
+class TemplateBenchmark extends BenchmarkBase {
+  const TemplateBenchmark() : super('Template');
+
+  static void main() {
+    const TemplateBenchmark().report();
+  }
+
+  // The benchmark code.
+  @override
+  void run() {}
+
+  // Not measured setup code executed prior to the benchmark runs.
+  @override
+  void setup() {}
+
+  // Not measures teardown code executed after the benchmark runs.
+  @override
+  void teardown() {}
+
+  // To opt into the reporting the time per run() instead of per 10 run() calls.
+  //@override
+  //void exercise() => run();
+}
+
+void main() {
+  // Run TemplateBenchmark
+  TemplateBenchmark.main();
+}

pkgs/benchmark_harness/integration_test/perf_benchmark_test.dart

@@ -0,0 +1,26 @@
+// Copyright 2024, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:benchmark_harness/perf_benchmark_harness.dart';
+import 'package:test/test.dart';
+
+class PerfBenchmark extends PerfBenchmarkBase {
+  PerfBenchmark(super.name);
+  int runCount = 0;
+
+  @override
+  void run() {
+    runCount++;
+    for (final i in List.filled(1000, 7)) {
+      runCount += i - i;
+    }
+  }
+}
+
+void main() {
+  test('run is called', () async {
+    final benchmark = PerfBenchmark('ForLoop');
+    await benchmark.reportPerf();
+  });
+}

pkgs/benchmark_harness/lib/benchmark_harness.dart

@@ -0,0 +1,7 @@
+// Copyright (c) 2021, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+export 'src/async_benchmark_base.dart';
+export 'src/benchmark_base.dart' show BenchmarkBase;
+export 'src/score_emitter.dart';

pkgs/benchmark_harness/lib/perf_benchmark_harness.dart

@@ -0,0 +1,7 @@
+// Copyright (c) 2024, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+export 'src/perf_benchmark_base_stub.dart'
+    if (dart.library.io) 'src/perf_benchmark_base.dart';
+export 'src/score_emitter.dart';