-
Notifications
You must be signed in to change notification settings - Fork 75
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Runner Cleanup: SuiteRunner & TestRunner classes (#452)
- Loading branch information
1 parent
e70778f
commit 67bc21f
Showing
4 changed files
with
204 additions
and
160 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,104 @@ | ||
| import { TestRunner } from "./test-runner.mjs"; | ||
| import { WarmupSuite } from "./benchmark-runner.mjs"; | ||
|
|
||
| // FIXME: Create AsyncSuiteRunner subclass. | ||
| // FIXME: Create RemoteSuiteRunner subclass. | ||
| export class SuiteRunner { | ||
| #frame; | ||
| #page; | ||
| #params; | ||
| #suite; | ||
| #client; | ||
| #suiteResults; | ||
|
|
||
| constructor(frame, page, params, suite, client, measuredValues) { | ||
| // FIXME: Create SuiteRunner-local measuredValues. | ||
| this.#suiteResults = measuredValues.tests[suite.name]; | ||
| if (!this.#suiteResults) { | ||
| this.#suiteResults = { tests: {}, total: 0 }; | ||
| measuredValues.tests[suite.name] = this.#suiteResults; | ||
| } | ||
| this.#frame = frame; | ||
| this.#page = page; | ||
| this.#client = client; | ||
| this.#suite = suite; | ||
| this.#params = params; | ||
| } | ||
|
|
||
| async run() { | ||
| await this._prepareSuite(); | ||
| await this._runSuite(); | ||
| } | ||
|
|
||
| async _prepareSuite() { | ||
| const suiteName = this.#suite.name; | ||
| const suitePrepareStartLabel = `suite-${suiteName}-prepare-start`; | ||
| const suitePrepareEndLabel = `suite-${suiteName}-prepare-end`; | ||
|
|
||
| performance.mark(suitePrepareStartLabel); | ||
| await this._loadFrame(); | ||
| await this.#suite.prepare(this.#page); | ||
| performance.mark(suitePrepareEndLabel); | ||
|
|
||
| performance.measure(`suite-${suiteName}-prepare`, suitePrepareStartLabel, suitePrepareEndLabel); | ||
| } | ||
|
|
||
| async _runSuite() { | ||
| const suiteName = this.#suite.name; | ||
| const suiteStartLabel = `suite-${suiteName}-start`; | ||
| const suiteEndLabel = `suite-${suiteName}-end`; | ||
|
|
||
| performance.mark(suiteStartLabel); | ||
| for (const test of this.#suite.tests) { | ||
| if (this.#client?.willRunTest) | ||
| await this.#client.willRunTest(this.#suite, test); | ||
|
|
||
| const testRunner = new TestRunner(this.#frame, this.#page, this.#params, this.#suite, test, this._recordTestResults); | ||
| await testRunner.runTest(); | ||
| } | ||
| performance.mark(suiteEndLabel); | ||
|
|
||
| performance.measure(`suite-${suiteName}`, suiteStartLabel, suiteEndLabel); | ||
| this._validateSuiteTotal(); | ||
| } | ||
|
|
||
| _validateSuiteTotal() { | ||
| // When the test is fast and the precision is low (for example with Firefox' | ||
| // privacy.resistFingerprinting preference), it's possible that the measured | ||
| // total duration for an entire is 0. | ||
| const suiteTotal = this.#suiteResults.total; | ||
| if (suiteTotal === 0) | ||
| throw new Error(`Got invalid 0-time total for suite ${this.#suite.name}: ${suiteTotal}`); | ||
| } | ||
|
|
||
| async _loadFrame() { | ||
| return new Promise((resolve, reject) => { | ||
| const frame = this.#frame; | ||
| frame.onload = () => resolve(); | ||
| frame.onerror = () => reject(); | ||
| frame.src = this.#suite.url; | ||
| }); | ||
| } | ||
|
|
||
| _recordTestResults = async (test, syncTime, asyncTime) => { | ||
| // Skip reporting updates for the warmup suite. | ||
| if (this.#suite === WarmupSuite) | ||
| return; | ||
|
|
||
| const total = syncTime + asyncTime; | ||
| this.#suiteResults.tests[test.name] = { tests: { Sync: syncTime, Async: asyncTime }, total: total }; | ||
| this.#suiteResults.total += total; | ||
|
|
||
| if (this.#client?.didRunTest) | ||
| await this.#client.didRunTest(this.#suite, test); | ||
| }; | ||
| } | ||
|
|
||
| // FIXME: implement remote steps | ||
| class RemoteSuiteRunner extends SuiteRunner {} | ||
|
|
||
| export const SUITE_RUNNER_LOOKUP = { | ||
| __proto__: null, | ||
| default: SuiteRunner, | ||
| remote: RemoteSuiteRunner, | ||
| }; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,79 @@ | ||
| import { TEST_INVOKER_LOOKUP } from "./test-invoker.mjs"; | ||
|
|
||
| export class TestRunner { | ||
| #frame; | ||
| #page; | ||
| #params; | ||
| #suite; | ||
| #test; | ||
| #callback; | ||
|
|
||
| constructor(frame, page, params, suite, test, callback) { | ||
| this.#suite = suite; | ||
| this.#test = test; | ||
| this.#params = params; | ||
| this.#callback = callback; | ||
|
|
||
| this.#page = page; | ||
| this.#frame = frame; | ||
| } | ||
|
|
||
| async runTest() { | ||
| // Prepare all mark labels outside the measuring loop. | ||
| const suiteName = this.#suite.name; | ||
| const testName = this.#test.name; | ||
| const syncStartLabel = `${suiteName}.${testName}-start`; | ||
| const syncEndLabel = `${suiteName}.${testName}-sync-end`; | ||
| const asyncStartLabel = `${suiteName}.${testName}-async-start`; | ||
| const asyncEndLabel = `${suiteName}.${testName}-async-end`; | ||
|
|
||
| let syncTime; | ||
| let asyncStartTime; | ||
| let asyncTime; | ||
|
|
||
| const runSync = () => { | ||
| if (this.#params.warmupBeforeSync) { | ||
| performance.mark("warmup-start"); | ||
| const startTime = performance.now(); | ||
| // Infinite loop for the specified ms. | ||
| while (performance.now() - startTime < this.#params.warmupBeforeSync) | ||
| continue; | ||
| performance.mark("warmup-end"); | ||
| } | ||
| performance.mark(syncStartLabel); | ||
| const syncStartTime = performance.now(); | ||
| this.#test.run(this.#page); | ||
| const syncEndTime = performance.now(); | ||
| performance.mark(syncEndLabel); | ||
|
|
||
| syncTime = syncEndTime - syncStartTime; | ||
|
|
||
| performance.mark(asyncStartLabel); | ||
| asyncStartTime = performance.now(); | ||
| }; | ||
| const measureAsync = () => { | ||
| const bodyReference = this.#frame ? this.#frame.contentDocument.body : document.body; | ||
| const windowReference = this.#frame ? this.#frame.contentWindow : window; | ||
| // Some browsers don't immediately update the layout for paint. | ||
| // Force the layout here to ensure we're measuring the layout time. | ||
| const height = bodyReference.getBoundingClientRect().height; | ||
| windowReference._unusedHeightValue = height; // Prevent dead code elimination. | ||
|
|
||
| const asyncEndTime = performance.now(); | ||
| performance.mark(asyncEndLabel); | ||
|
|
||
| asyncTime = asyncEndTime - asyncStartTime; | ||
|
|
||
| if (this.#params.warmupBeforeSync) | ||
| performance.measure("warmup", "warmup-start", "warmup-end"); | ||
| performance.measure(`${suiteName}.${testName}-sync`, syncStartLabel, syncEndLabel); | ||
| performance.measure(`${suiteName}.${testName}-async`, asyncStartLabel, asyncEndLabel); | ||
| }; | ||
|
|
||
| const report = () => this.#callback(this.#test, syncTime, asyncTime); | ||
| const invokerClass = TEST_INVOKER_LOOKUP[this.#params.measurementMethod]; | ||
| const invoker = new invokerClass(runSync, measureAsync, report, this.#params); | ||
|
|
||
| return invoker.start(); | ||
| } | ||
| } |
Oops, something went wrong.