Add param for shuffling suite order between runs (#292)

* Add param for shuffling suite order between runs * pr feedback * move into runner-prepare and log seed
WebKit · Aug 18, 2023 · d86dad3 · d86dad3
1 parent 5107c73
commit d86dad3
Show file tree

Hide file tree

Showing 2 changed files with 47 additions and 1 deletion.
diff --git a/resources/benchmark-runner.mjs b/resources/benchmark-runner.mjs
@@ -314,6 +314,16 @@ class RAFTestInvoker extends TestInvoker {
     }
 }
 
+// https://stackoverflow.com/a/47593316
+function seededHashRandomNumberGenerator(a) {
+    return function() {
+      var t = a += 0x6D2B79F5;
+      t = Math.imul(t ^ t >>> 15, t | 1);
+      t ^= t + Math.imul(t ^ t >>> 7, t | 61);
+      return ((t ^ t >>> 14) >>> 0);
+    }
+}
+
 export class BenchmarkRunner {
     constructor(suites, client) {
         this._suites = suites;
@@ -323,6 +333,9 @@ export class BenchmarkRunner {
         this._page = null;
         this._metrics = null;
         this._iterationCount = params.iterationCount;
+        if (params.shuffleSeed !== "off") {
+            this._suiteOrderRandomNumberGenerator = seededHashRandomNumberGenerator(params.shuffleSeed);
+        }
     }
 
     async runMultipleIterations(iterationCount) {
@@ -381,10 +394,23 @@ export class BenchmarkRunner {
         this._removeFrame();
         await this._appendFrame();
         this._page = new Page(this._frame);
+
+        let suites = [...this._suites];
+        if (this._suiteOrderRandomNumberGenerator) {
+            // We just do a simple Fisher-Yates shuffle based on the repeated hash of the
+            // seed. This is not a high quality RNG, but it's plenty good enough.
+            for (let i = 0; i < suites.length - 1; i++) {
+                let j = i + (this._suiteOrderRandomNumberGenerator() % (suites.length - i));
+                let tmp = suites[i];
+                suites[i] = suites[j];
+                suites[j] = tmp;
+            }
+        }
+
         performance.mark(prepareEndLabel);
         performance.measure("runner-prepare", prepareStartLabel, prepareEndLabel);
 
-        for (const suite of this._suites) {
+        for (const suite of suites) {
             if (!suite.disabled)
                 await this._runSuite(suite);
         }

diff --git a/resources/params.mjs b/resources/params.mjs
@@ -20,6 +20,11 @@ class Params {
     waitBeforeSync = 0;
     // Warmup time before the sync step in ms.
     warmupBeforeSync = 0;
+    // Seed for shuffling the execution order of suites.
+    // "off": do not shuffle
+    // "generate": generate a random seed
+    // <integer>: use the provided integer as a seed
+    shuffleSeed = "off";
 
     constructor(searchParams = undefined) {
         if (searchParams)
@@ -102,6 +107,21 @@ class Params {
             searchParams.delete("measurementMethod");
         }
 
+        if (searchParams.has("shuffleSeed")) {
+            this.shuffleSeed = searchParams.get("shuffleSeed");
+            if (this.shuffleSeed !== "off") {
+                if (this.shuffleSeed === "generate") {
+                    this.shuffleSeed = Math.floor(Math.random() * 1 << 16);
+                    console.log(`Generated a random suite order seed: ${this.shuffleSeed}`);
+                } else {
+                    this.shuffleSeed = parseInt(this.shuffleSeed);
+                }
+                if (!Number.isInteger(this.shuffleSeed))
+                    throw new Error(`Invalid shuffle seed: '${this.shuffleSeed}', must be either 'off', 'generate' or an integer.`);
+            }
+            searchParams.delete("shuffleSeed");
+        }
+
         const unused = Array.from(searchParams.keys());
         if (unused.length > 0)
             console.error("Got unused search params", unused);