chore: Add performance benchmark to diff-sequences package

CHANGELOG.md

@@ -116,6 +116,7 @@
 - `[*]` [**BREAKING**] Require Node.js 6+ for all packages ([#7258](https://github.com/facebook/jest/pull/7258))
 - `[*]` [**BREAKING**] Use ESM `import` and `export` all over ([#7548](https://github.com/facebook/jest/pull/7548))
 - `[jest-util]` [**BREAKING**] Remove long-deprecated globals for fake timers ([#7285](https://github.com/facebook/jest/pull/7285))
+- `[diff-sequences]` Add performance benchmark to package ([#7603](https://github.com/facebook/jest/pull/7603))
 - `[docs]` Fix message property in custom matcher example to return a function instead of a constant. ([#7426](https://github.com/facebook/jest/pull/7426))
 - `[jest-circus]` Standardize file naming in `jest-circus` ([#7301](https://github.com/facebook/jest/pull/7301))
 - `[docs]` Add synchronous test.each setup ([#7150](https://github.com/facebook/jest/pull/7150))

packages/diff-sequences/package.json

@@ -17,5 +17,12 @@
   "engines": {
     "node": ">= 6"
   },
-  "main": "build/index.js"
+  "main": "build/index.js",
+  "scripts": {
+    "perf": "node --expose-gc perf/index.js"
+  },
+  "devDependencies": {
+    "benchmark": "^2.1.4",
+    "diff": "^4.0.1"
+  }
 }

packages/diff-sequences/perf/example.md

@@ -0,0 +1,48 @@
+## Benchmark time for `diff-sequences` versus `diff`
+
+A ratio less than 1.0 means `diff-sequences` is faster.
+
+### n = 20
+
+| name   |    % | ratio  | improved  |   rme | baseline  |   rme |
+| :----- | ---: | :----- | :-------- | ----: | :-------- | ----: |
+| delete |  20% | 0.1824 | 3.0639e-6 | 1.11% | 1.6795e-5 | 0.78% |
+| insert |  20% | 0.1367 | 2.5786e-6 | 0.75% | 1.8866e-5 | 0.85% |
+| delete |  40% | 0.1015 | 3.0534e-6 | 1.00% | 3.0090e-5 | 0.70% |
+| insert |  40% | 0.0791 | 2.6722e-6 | 0.61% | 3.3791e-5 | 0.56% |
+| delete |  80% | 0.0410 | 2.8870e-6 | 0.93% | 7.0411e-5 | 0.72% |
+| insert |  80% | 0.0371 | 2.5786e-6 | 0.83% | 6.9431e-5 | 0.62% |
+| change |  10% | 0.1504 | 2.8703e-6 | 0.71% | 1.9087e-5 | 0.83% |
+| change |  20% | 0.1706 | 3.2637e-6 | 0.78% | 1.9127e-5 | 0.62% |
+| change |  50% | 0.0944 | 1.2012e-5 | 0.55% | 1.2724e-4 | 0.76% |
+| change | 100% | 0.0522 | 1.5422e-5 | 0.61% | 2.9566e-4 | 0.66% |
+
+### n = 200
+
+| name   |    % | ratio  | improved  |   rme | baseline  |   rme |
+| :----- | ---: | :----- | :-------- | ----: | :-------- | ----: |
+| delete |  20% | 0.1524 | 7.2866e-5 | 0.75% | 4.7797e-4 | 0.80% |
+| insert |  20% | 0.1226 | 6.1561e-5 | 0.58% | 5.0198e-4 | 0.66% |
+| delete |  40% | 0.1118 | 1.5674e-4 | 0.67% | 1.4020e-3 | 0.58% |
+| insert |  40% | 0.0894 | 1.2906e-4 | 0.64% | 1.4435e-3 | 0.53% |
+| delete |  80% | 0.0796 | 3.0119e-4 | 0.58% | 3.7852e-3 | 0.52% |
+| insert |  80% | 0.0734 | 2.4713e-4 | 0.67% | 3.3653e-3 | 0.54% |
+| change |  10% | 0.1572 | 7.2965e-5 | 0.48% | 4.6426e-4 | 0.73% |
+| change |  20% | 0.1446 | 7.0056e-5 | 0.69% | 4.8456e-4 | 0.53% |
+| change |  50% | 0.0764 | 6.5638e-4 | 0.67% | 8.5946e-3 | 0.70% |
+| change | 100% | 0.0525 | 1.1160e-3 | 0.51% | 2.1249e-2 | 0.63% |
+
+### n = 2000
+
+| name   |    % | ratio  | improved  |   rme | baseline  |   rme |
+| :----- | ---: | :----- | :-------- | ----: | :-------- | ----: |
+| delete |  20% | 0.0669 | 3.4073e-3 | 0.54% | 5.0922e-2 | 0.54% |
+| insert |  20% | 0.0588 | 2.8273e-3 | 0.51% | 4.8111e-2 | 0.46% |
+| delete |  40% | 0.0517 | 1.1048e-2 | 0.52% | 2.1367e-1 | 0.47% |
+| insert |  40% | 0.0460 | 9.1469e-3 | 0.37% | 1.9878e-1 | 0.26% |
+| delete |  80% | 0.0563 | 2.7426e-2 | 0.56% | 4.8674e-1 | 0.36% |
+| insert |  80% | 0.0506 | 2.2208e-2 | 0.35% | 4.3888e-1 | 0.47% |
+| change |  10% | 0.0716 | 3.1267e-3 | 1.21% | 4.3652e-2 | 0.56% |
+| change |  20% | 0.0621 | 3.0197e-3 | 0.72% | 4.8652e-2 | 0.45% |
+| change |  50% | 0.0083 | 5.4250e-2 | 0.62% | 6.5595e+0 | 3.60% |
+| change | 100% | 0.0493 | 1.0534e-1 | 0.71% | 2.1362e+0 | 0.21% |

packages/diff-sequences/perf/index.js

@@ -0,0 +1,161 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Make sure to run node with --expose-gc option!
+
+// The times are reliable if about 1% relative mean error if you run it:
+
+// * immediately after restart
+// * with 100% battery charge
+// * not connected to network
+
+/* eslint import/no-extraneous-dependencies: "off" */
+
+const Benchmark = require('benchmark');
+
+const diffBaseline = require('diff').diffLines;
+const diffImproved = require('../build/index.js').default;
+
+const testBaseline = (a, b) => {
+  const benchmark = new Benchmark({
+    fn() {
+      diffBaseline(a, b);
+    },
+    name: 'baseline',
+    onCycle() {
+      global.gc(); // after run cycle
+    },
+    onStart() {
+      global.gc(); // when benchmark starts
+    },
+  });
+
+  benchmark.run({async: false});
+
+  return benchmark.stats;
+};
+
+const testImproved = function(a, b) {
+  const benchmark = new Benchmark({
+    fn() {
+      // Split string arguments to make fair comparison with baseline.
+      const aItems = a.split('\n');
+      const bItems = b.split('\n');
+
+      const isCommon = (aIndex, bIndex) => aItems[aIndex] === bItems[bIndex];
+
+      // This callback obviously does less than baseline `diff` package,
+      // but avoiding double work and memory churn is the goal.
+      // For example, `jest-diff` has had to split strings that `diff` joins.
+      const foundSubsequence = () => {};
+
+      diffImproved(aItems.length, bItems.length, isCommon, foundSubsequence);
+    },
+    name: 'improved',
+    onCycle() {
+      global.gc(); // after run cycle
+    },
+    onStart() {
+      global.gc(); // when benchmark starts
+    },
+  });
+
+  benchmark.run({async: false});
+
+  return benchmark.stats;
+};
+
+const writeHeading2 = () => {
+  process.stdout.write(
+    '## Benchmark time for `diff-sequences` versus `diff`\n\n',
+  );
+  process.stdout.write(
+    'A ratio less than 1.0 means `diff-sequences` is faster.\n',
+  );
+};
+
+const writeHeading3 = n => {
+  process.stdout.write(`\n### n = ${n}\n\n`);
+  process.stdout.write(
+    '| name | % | ratio | improved | rme | baseline | rme |\n',
+  );
+  process.stdout.write('| :--- | ---: | :--- | :--- | ---: | :--- | ---: |\n');
+};
+
+const writeRow = (name, percent, statsImproved, statsBaseline) => {
+  const {mean: meanImproved, rme: rmeImproved} = statsImproved;
+  const {mean: meanBaseline, rme: rmeBaseline} = statsBaseline;
+  const ratio = meanImproved / meanBaseline;
+
+  process.stdout.write(
+    `| ${name} | ${percent}% | ${ratio.toFixed(
+      4,
+    )} | ${meanImproved.toExponential(4)} | ${rmeImproved.toFixed(
+      2,
+    )}% | ${meanBaseline.toExponential(4)} | ${rmeBaseline.toFixed(2)}% |\n`,
+  );
+};
+
+const testDeleteInsert = (tenths, more, less) => {
+  // For improved `diff-sequences` package, delete is often slower than insert.
+  const statsDeleteImproved = testImproved(more, less);
+  const statsDeleteBaseline = testBaseline(more, less);
+  writeRow('delete', tenths * 10, statsDeleteImproved, statsDeleteBaseline);
+
+  // For baseline `diff` package, many insertions is serious perf problem.
+  // However, the benchmark package cannot accurately measure for large n.
+  const statsInsertBaseline = testBaseline(less, more);
+  const statsInsertImproved = testImproved(less, more);
+  writeRow('insert', tenths * 10, statsInsertImproved, statsInsertBaseline);
+};
+
+const testChange = (tenths, expected, received) => {
+  const statsImproved = testImproved(expected, received);
+  const statsBaseline = testBaseline(expected, received);
+  writeRow('change', tenths * 10, statsImproved, statsBaseline);
+};
+
+const getItems = (n, callback) => {
+  const items = [];
+
+  for (let i = 0; i !== n; i += 1) {
+    const item = callback(i);
+    if (typeof item === 'string') {
+      items.push(item);
+    }
+  }
+
+  return items.join('\n');
+};
+
+// Simulate change of property name which is usually not same line.
+// Expected: 0 1 2 3 4 5 6 7 8 9 and so on
+// Received: 1 2 3 4 x0 5 6 7 8 9 and so on
+const change2 = i => {
+  const j = i % 10;
+  return j === 4 ? `x${i - 4}` : j < 4 ? `${i + 1}` : `${i}`;
+};
+
+const testLength = n => {
+  const all = getItems(n, i => `${i}`);
+
+  writeHeading3(n);
+
+  [2, 4, 8].forEach(tenth => {
+    testDeleteInsert(tenth, all, getItems(n, i => i % 10 >= tenth && `${i}`));
+  });
+  testChange(1, all, getItems(n, i => (i % 10 === 0 ? `x${i}` : `${i}`)));
+  testChange(2, all, getItems(n, change2));
+  testChange(5, all, getItems(n, i => (i % 2 === 0 ? `x${i}` : `${i}`)));
+  testChange(10, all, getItems(n, i => `x${i}`)); // simulate TDD
+};
+
+writeHeading2();
+
+testLength(20);
+testLength(200);
+testLength(2000);

yarn.lock

@@ -2582,6 +2582,14 @@ beeper@^1.0.0:
   resolved "https://registry.yarnpkg.com/beeper/-/beeper-1.1.1.tgz#e6d5ea8c5dad001304a70b22638447f69cb2f809"
   integrity sha1-5tXqjF2tABMEpwsiY4RH9pyy+Ak=
 
+benchmark@^2.1.4:
+  version "2.1.4"
+  resolved "https://registry.yarnpkg.com/benchmark/-/benchmark-2.1.4.tgz#09f3de31c916425d498cc2ee565a0ebf3c2a5629"
+  integrity sha1-CfPeMckWQl1JjMLuVloOvzwqVik=
+  dependencies:
+    lodash "^4.17.4"
+    platform "^1.3.3"
+
 better-assert@~1.0.0:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/better-assert/-/better-assert-1.0.2.tgz#40866b9e1b9e0b55b481894311e68faffaebc522"
@@ -4508,6 +4516,11 @@ [email protected], diff@^3.2.0:
   resolved "https://registry.yarnpkg.com/diff/-/diff-3.5.0.tgz#800c0dd1e0a8bfbc95835c202ad220fe317e5a12"
   integrity sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==
 
+diff@^4.0.1:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/diff/-/diff-4.0.1.tgz#0c667cb467ebbb5cea7f14f135cc2dba7780a8ff"
+  integrity sha512-s2+XdvhPCOF01LRQBC8hf4vhbVmI2CGS5aZnxLJlT5FtdhPCDFq80q++zK2KlrVorVDdL5BOGZ/VfLrVtYNF+Q==
+
 diffie-hellman@^5.0.0:
   version "5.0.3"
   resolved "https://registry.yarnpkg.com/diffie-hellman/-/diffie-hellman-5.0.3.tgz#40e8ee98f55a2149607146921c63e1ae5f3d2875"
@@ -10234,6 +10247,11 @@ pkg-dir@^3.0.0:
   dependencies:
     find-up "^3.0.0"
 
+platform@^1.3.3:
+  version "1.3.5"
+  resolved "https://registry.yarnpkg.com/platform/-/platform-1.3.5.tgz#fb6958c696e07e2918d2eeda0f0bc9448d733444"
+  integrity sha512-TuvHS8AOIZNAlE77WUDiR4rySV/VMptyMfcfeoMgs4P8apaZM3JrnbzBiixKUv+XR6i+BXrQh8WAnjaSPFO65Q==
+
 [email protected]:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/plist/-/plist-2.0.1.tgz#0a32ca9481b1c364e92e18dc55c876de9d01da8b"