test(protocol-designer): add benchmark test setup + example (#6174)

.gitignore

@@ -92,6 +92,7 @@ labware-library/cypress/screenshots/
 labware-library/cypress/videos/
 protocol-designer/cypress/screenshots/
 protocol-designer/cypress/videos/
+protocol-designer/benchmarks/output
 
 # Local Calibration Data
 calibrations/

package.json

@@ -31,6 +31,7 @@
     "@babel/preset-env": "^7.8.4",
     "@babel/preset-flow": "^7.8.3",
     "@babel/preset-react": "^7.8.3",
+    "@babel/register": "^7.10.5",
     "@cypress/webpack-preprocessor": "^5.1.2",
     "ajv": "^6.10.2",
     "aws-sdk": "^2.493.0",
@@ -89,6 +90,8 @@
     "mime": "^2.4.4",
     "mini-css-extract-plugin": "^0.9.0",
     "multer": "^1.4.2",
+    "nanobench": "^2.1.1",
+    "ntee": "^2.0.0",
     "onchange": "^6.1.0",
     "optimize-css-assets-webpack-plugin": "^5.0.3",
     "portfinder": "^1.0.13",

protocol-designer/Makefile

@@ -6,6 +6,8 @@ SHELL := bash
 # add node_modules/.bin to PATH
 PATH := $(shell cd .. && yarn bin):$(PATH)
 
+benchmark_output := $(shell node -e 'console.log(new Date());')
+
 # standard targets
 #####################################################################
 
@@ -32,6 +34,13 @@ build:
 # development
 #####################################################################
 
+.PHONY: benchmarks
+benchmarks: export NODE_ENV := test
+benchmarks:
+	shx mkdir -p benchmarks/output
+	node ../scripts/runBenchmarks "./benchmarks/*.js" | \
+	ntee "./benchmarks/output/$(benchmark_output)"
+
 .PHONY: dev
 dev: export NODE_ENV := development
 dev:

protocol-designer/benchmarks/README.md

@@ -0,0 +1,12 @@
+# Protocol Designer Benchmarks
+
+Note: this tooling around benchmark testing is very minimal and subject to change!
+
+To run all PD benchmarks, `make -C protocol-designer benchmarks`. This will output to a file `protocol-designer/benchmarks/output/$(date_time)`. To set the name of the file, specify `benchmark_output`
+
+These benchmarks use [Nanobench](https://github.com/mafintosh/nanobench), so you can pipe the output to a file and compare benchmark runs with `yarn nanobench-compare fileA fileB`
+
+## Local benchmarking guidelines
+
+- Do not compare benchmarks across different machines.
+- Make sure the same resources are available between runs (eg if you kill your dev servers and editor etc, it will likely affect the benchmarks from the run that competed with those processes)

protocol-designer/benchmarks/timelineGeneration.js

@@ -0,0 +1,65 @@
+// @flow
+import assert from 'assert'
+import bench from 'nanobench'
+import {
+  commandCreatorsTimeline,
+  curryCommandCreator,
+  mix,
+} from '../src/step-generation'
+import { getStateAndContextTempTCModules } from '../src/step-generation/__fixtures__'
+
+const times = 200
+
+bench(`commandCreatorsTimeline: mix ${times} times`, b => {
+  const {
+    robotState: initialRobotState,
+    invariantContext,
+  } = getStateAndContextTempTCModules({
+    temperatureModuleId: 'someTemperatureModuleId',
+    thermocyclerId: 'someTCId',
+  })
+
+  const curriedCommandCreators = [
+    curryCommandCreator(mix, {
+      commandCreatorFnName: 'mix',
+      name: null,
+      description: null,
+      labware: 'sourcePlateId',
+      pipette: 'p300SingleId',
+      wells: ['A1'],
+      volume: 5,
+      times,
+      touchTip: false,
+      touchTipMmFromBottom: 10,
+      changeTip: 'once',
+      blowoutLocation: null,
+      blowoutFlowRateUlSec: 42,
+      blowoutOffsetFromTopMm: 1,
+      aspirateOffsetFromBottomMm: 1,
+      dispenseOffsetFromBottomMm: 2,
+      aspirateFlowRateUlSec: 5,
+      dispenseFlowRateUlSec: 6,
+    }),
+  ]
+
+  b.start()
+
+  const timeline = commandCreatorsTimeline(
+    curriedCommandCreators,
+    invariantContext,
+    initialRobotState
+  )
+
+  b.end()
+
+  if (timeline.errors !== null) {
+    assert(
+      false,
+      `Expected no timeline errors but got ${JSON.stringify(
+        timeline.errors,
+        null,
+        4
+      ) ?? 'undefined'}`
+    )
+  }
+})

scripts/runBenchmarks.js

@@ -0,0 +1,25 @@
+// script to run benchmark tests given a glob path
+'use strict'
+
+const assert = require('assert')
+const path = require('path')
+const globby = require('globby')
+// eslint-disable-next-line no-unused-vars
+const bench = require('nanobench')
+require('@babel/register')({
+  cwd: path.join(__dirname, '..'),
+  plugins: ['@babel/plugin-transform-modules-commonjs'],
+})
+
+const USAGE =
+  "\nUsage:\n  node ./scripts/runBenchmarks 'path/to/benchmarks/*.js'"
+assert(process.argv.length === 3, USAGE)
+
+const benchmarkFiles = globby.sync(process.argv[2])
+
+// NOTE: adapted from nanobench/run.js
+global.__NANOBENCH__ = require.resolve('nanobench')
+
+benchmarkFiles.forEach(f => {
+  require(path.join(process.cwd(), f))
+})

yarn.lock

@@ -987,6 +987,17 @@
     "@babel/plugin-transform-react-jsx-self" "^7.8.3"
     "@babel/plugin-transform-react-jsx-source" "^7.8.3"
 
+"@babel/register@^7.10.5":
+  version "7.10.5"
+  resolved "https://registry.yarnpkg.com/@babel/register/-/register-7.10.5.tgz#354f3574895f1307f79efe37a51525e52fd38d89"
+  integrity sha512-eYHdLv43nyvmPn9bfNfrcC4+iYNwdQ8Pxk1MFJuU/U5LpSYl/PH4dFMazCYZDFVi8ueG3shvO+AQfLrxpYulQw==
+  dependencies:
+    find-cache-dir "^2.0.0"
+    lodash "^4.17.19"
+    make-dir "^2.1.0"
+    pirates "^4.0.0"
+    source-map-support "^0.5.16"
+
 "@babel/runtime-corejs3@^7.8.3":
   version "7.10.5"
   resolved "https://registry.yarnpkg.com/@babel/runtime-corejs3/-/runtime-corejs3-7.10.5.tgz#a57fe6c13045ca33768a2aa527ead795146febe1"
@@ -11823,6 +11834,11 @@ lodash@^4.15.0, lodash@^4.17.10, lodash@^4.17.11:
   version "4.17.11"
   resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.11.tgz#b39ea6229ef607ecd89e2c8df12536891cac9b8d"
 
+lodash@^4.17.19:
+  version "4.17.19"
+  resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.19.tgz#e48ddedbe30b3321783c5b4301fbd353bc1e4a4b"
+  integrity sha512-JNvd8XER9GQX0v2qJgsaN/mzFCNA5BRe/j8JN9d+tWyGLSodKQHKFicdwNYzWwI3wjRnaKPsGj1XkBjx/F96DQ==
+
 [email protected], log-symbols@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/log-symbols/-/log-symbols-3.0.0.tgz#f3a08516a5dea893336a7dee14d18a1cfdab77c4"
@@ -12713,6 +12729,11 @@ [email protected]:
   resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-0.0.8.tgz#1630c42b2251ff81e2a283de96a5497ea92e5e0d"
   integrity sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==
 
+mutexify@^1.1.0:
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/mutexify/-/mutexify-1.3.0.tgz#b32bee9319050963cf6b5d5bce7fbd6ea08ab609"
+  integrity sha512-WNPlgZ3AHETGSa4jeRP4aW6BPQ/a++MwoMFFIgrDg80+m70mbxuNOrevANfBDmur82zxTFAY3OwvMAvqrkV2sA==
+
 mz@^2.5.0:
   version "2.7.0"
   resolved "https://registry.yarnpkg.com/mz/-/mz-2.7.0.tgz#95008057a56cafadc2bc63dde7f9ff6955948e32"
@@ -12727,6 +12748,16 @@ nan@^2.12.1, nan@^2.13.2:
   resolved "https://registry.yarnpkg.com/nan/-/nan-2.14.0.tgz#7818f722027b2459a86f0295d434d1fc2336c52c"
   integrity sha512-INOFj37C7k3AfaNTtX8RhsTw7qRy7eLET14cROi9+5HAVbbHuIWUHEauBv5qT4Av2tWasiTY1Jw6puUNqRJXQg==
 
+nanobench@^2.1.1:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/nanobench/-/nanobench-2.1.1.tgz#c2f23fcce116d50b4998b1954ba114674c137269"
+  integrity sha512-z+Vv7zElcjN+OpzAxAquUayFLGK3JI/ubCl0Oh64YQqsTGG09CGqieJVQw4ui8huDnnAgrvTv93qi5UaOoNj8A==
+  dependencies:
+    browser-process-hrtime "^0.1.2"
+    chalk "^1.1.3"
+    mutexify "^1.1.0"
+    pretty-hrtime "^1.0.2"
+
 nanomatch@^1.2.5:
   version "1.2.7"
   resolved "https://registry.yarnpkg.com/nanomatch/-/nanomatch-1.2.7.tgz#53cd4aa109ff68b7f869591fdc9d10daeeea3e79"
@@ -13143,6 +13174,11 @@ npm-run-path@^4.0.0:
     gauge "~2.7.3"
     set-blocking "~2.0.0"
 
+ntee@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/ntee/-/ntee-2.0.0.tgz#8c1e7410d9ae9b3a026f57ef1b8dabf8a0d8dd21"
+  integrity sha512-VVRjywWlfLEWcfyZpHrc2wCgE4DCs0QuTyuUn16PJO77XJFquUEBtnBpxpD0BGZDCvg7+8JWNHKuUVvhqRH+dg==
+
 nth-check@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/nth-check/-/nth-check-1.0.2.tgz#b2bd295c37e3dd58a3bf0700376663ba4d9cf05c"
@@ -14034,7 +14070,7 @@ pinkie@^2.0.0:
   version "2.0.4"
   resolved "https://registry.yarnpkg.com/pinkie/-/pinkie-2.0.4.tgz#72556b80cfa0d48a974e80e77248e80ed4f7f870"
 
-pirates@^4.0.1:
+pirates@^4.0.0, pirates@^4.0.1:
   version "4.0.1"
   resolved "https://registry.yarnpkg.com/pirates/-/pirates-4.0.1.tgz#643a92caf894566f91b2b986d2c66950a8e2fb87"
   integrity sha512-WuNqLTbMI3tmfef2TKxlQmAiLHKtFhlsCZnPIpuv2Ow0RDVO8lfy1Opf4NUzlMXLjPl+Men7AuVdX6TA+s+uGA==
@@ -15021,6 +15057,11 @@ pretty-format@^3.5.1:
   version "3.8.0"
   resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-3.8.0.tgz#bfbed56d5e9a776645f4b1ff7aa1a3ac4fa3c385"
 
+pretty-hrtime@^1.0.2:
+  version "1.0.3"
+  resolved "https://registry.yarnpkg.com/pretty-hrtime/-/pretty-hrtime-1.0.3.tgz#b7e3ea42435a4c9b2759d99e0f201eb195802ee1"
+  integrity sha1-t+PqQkNaTJsnWdmeDyAesZWALuE=
+
 pretty-ms@^5.0.0:
   version "5.1.0"
   resolved "https://registry.yarnpkg.com/pretty-ms/-/pretty-ms-5.1.0.tgz#b906bdd1ec9e9799995c372e2b1c34f073f95384"
@@ -17469,6 +17510,14 @@ source-map-support@^0.5.13, source-map-support@~0.5.12:
     buffer-from "^1.0.0"
     source-map "^0.6.0"
 
+source-map-support@^0.5.16:
+  version "0.5.19"
+  resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.19.tgz#a98b62f86dcaf4f67399648c085291ab9e8fed61"
+  integrity sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw==
+  dependencies:
+    buffer-from "^1.0.0"
+    source-map "^0.6.0"
+
 source-map-support@^0.5.6:
   version "0.5.9"
   resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.9.tgz#41bc953b2534267ea2d605bccfa7bfa3111ced5f"