Benchmark task

This adds a benchmark task inspired by #1167 and #1163. Invoke with `yarn benchmark`.

Allows filtering down which benchmark to run, ala jest, and supplying which revisions to run against.

.npmignore

@@ -16,4 +16,5 @@ coverage
 resources
 src
 dist
+__tests__
 npm

package.json

@@ -25,6 +25,7 @@
     "testonly:cover": "babel-node ./node_modules/.bin/isparta cover --root src --report html _mocha -- $npm_package_options_mocha",
     "testonly:coveralls": "babel-node ./node_modules/.bin/isparta cover --root src --report lcovonly _mocha -- $npm_package_options_mocha && cat ./coverage/lcov.info | coveralls",
     "lint": "eslint --rulesdir ./resources/lint src || (printf '\\033[33mTry: \\033[7m npm run lint -- --fix \\033[0m\\n' && exit 1)",
+    "benchmark": "node ./resources/benchmark.js",
     "prettier": "prettier --write 'src/**/*.js'",
     "check": "flow check",
     "check-cover": "for file in {src/*.js,src/**/*.js}; do echo $file; flow coverage $file; done",
@@ -52,6 +53,8 @@
     "babel-plugin-transform-flow-strip-types": "6.22.0",
     "babel-plugin-transform-object-rest-spread": "6.26.0",
     "babel-preset-env": "^1.5.2",
+    "beautify-benchmark": "0.2.4",
+    "benchmark": "2.1.4",
     "chai": "4.1.2",
     "chai-json-equal": "0.0.1",
     "chai-spies-next": "0.9.3",

resources/benchmark.js

@@ -0,0 +1,142 @@
+/**
+ * Copyright (c) 2015-present, Facebook, Inc.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+const {Suite} = require('benchmark');
+const beautifyBenchmark = require('beautify-benchmark');
+const {execSync} = require('child_process');
+const os = require('os');
+const path = require('path');
+
+// Like build:cjs, but includes __tests__ and copies other files.
+const BUILD_CMD = 'babel src --optional runtime --copy-files --out-dir dist/';
+const LOCAL = 'local';
+const LOCAL_DIR = path.join(__dirname, '../');
+const TEMP_DIR = os.tmpdir();
+
+// Returns the complete git hash for a given git revision reference.
+function hashForRevision(revision) {
+  if (revision === LOCAL) {
+    return revision;
+  }
+  const out = execSync(`git rev-parse "${revision}"`, {encoding: 'utf8'});
+  const match = /[0-9a-f]{8,40}/.exec(out);
+  if (!match) {
+    throw new Error(`Bad results for revision ${revision}: ${out}`);
+  }
+  return match[0];
+}
+
+// Returns the temporary directory which hosts the files for this git hash.
+function dirForHash(hash) {
+  if (hash === LOCAL) {
+    return path.join(__dirname, '../');
+  }
+  return path.join(TEMP_DIR, 'graphql-js-benchmark', hash);
+}
+
+// Build a benchmarkable environment for the given revision.
+function prepareRevision(revision) {
+  const hash = hashForRevision(revision);
+  const dir = dirForHash(hash);
+  if (hash === LOCAL) {
+    execSync(`(cd "${dir}" && yarn run ${BUILD_CMD})`);
+  } else {
+    execSync(`
+      if [ ! -d "${dir}" ]; then
+        mkdir -p "${dir}" &&
+        git archive "${hash}" | tar -xC "${dir}" &&
+        (cd "${dir}" && yarn install);
+      fi &&
+      # Copy in local tests so the same logic applies to each revision.
+      for file in $(cd "${LOCAL_DIR}src"; find . -path '*/__tests__/*.js');
+        do cp "${LOCAL_DIR}src/$file" "${dir}/src/$file";
+      done &&
+      (cd "${dir}" && yarn run ${BUILD_CMD})
+    `);
+  }
+}
+
+// Find all benchmark tests to be run.
+function findBenchmarks() {
+  const out = execSync(
+    `(cd ${LOCAL_DIR}src; find . -path '*/__tests__/*-benchmark.js')`,
+    {encoding: 'utf8'}
+  );
+  return out.split('\n').filter(Boolean);
+}
+
+// Run a given benchmark test with the provided revisions.
+function runBenchmark(benchmark, revisions) {
+  const modules = revisions.map(revision =>
+    require(path.join(dirForHash(hashForRevision(revision)), 'dist', benchmark))
+  );
+  const suite = new Suite(modules[0].name, {
+    onStart(event) {
+      console.log('⏱️  ' + event.currentTarget.name);
+    },
+    onCycle(event) {
+      beautifyBenchmark.add(event.target);
+    },
+    onComplete() {
+      beautifyBenchmark.log();
+    },
+  });
+  for (let i = 0; i < revisions.length; i++) {
+    suite.add(revisions[i], modules[i].measure);
+  }
+  suite.run();
+}
+
+// Prepare all revisions and run benchmarks matching a pattern against them.
+function prepareAndRunBenchmarks(benchmarkPatterns, revisions) {
+  const benchmarks = findBenchmarks().filter(benchmark =>
+    benchmarkPatterns.length === 0 ||
+    benchmarkPatterns.some(pattern => benchmark.indexOf(pattern) !== -1)
+  );
+  if (benchmarks.length === 0) {
+    console.warn(`No benchmarks matching: \u001b[1m${benchmarkPatterns.join('\u001b[0m or \u001b[1m')}\u001b[0m`);
+    return;
+  }
+  revisions.forEach(revision => {
+    console.log(`🍳  Preparing ${revision}...`);
+    prepareRevision(revision);
+  });
+  benchmarks.forEach(benchmark =>
+    runBenchmark(benchmark, revisions)
+  );
+}
+
+function getArguments(argv) {
+  const revsIdx = argv.indexOf('--revs');
+  const revsArgs = revsIdx === -1 ? [] : argv.slice(revsIdx + 1);
+  const benchmarkPatterns = revsIdx === -1 ? argv : argv.slice(0, revsIdx);
+  let assumeArgs;
+  let revisions;
+  switch (revsArgs.length) {
+    case 0:
+      assumeArgs = [...benchmarkPatterns, '--revs', 'local', 'HEAD'];
+      revisions = [LOCAL, 'HEAD']
+      break;
+    case 1:
+      assumeArgs = [...benchmarkPatterns, '--revs', 'local', revsArgs[0]];
+      revisions = [LOCAL, revsArgs[0]]
+      break;
+    default:
+      revisions = revsArgs;
+      break;
+  }
+  if (assumeArgs) {
+    console.warn(`Assuming you meant: \u001b[1mbenchmark ${assumeArgs.join(' ')}\u001b[0m`);
+  }
+  return {benchmarkPatterns, revisions};
+}
+
+// Get the revisions and make things happen!
+if (require.main === module) {
+  const { benchmarkPatterns, revisions } = getArguments(process.argv.slice(2));
+  prepareAndRunBenchmarks(benchmarkPatterns, revisions);
+}

src/language/__tests__/parser-benchmark.js

@@ -0,0 +1,19 @@
+/**
+ * Copyright (c) 2015-present, Facebook, Inc.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import { join } from 'path';
+import { readFileSync } from 'fs';
+import { parse } from '../parser';
+
+const kitchenSink = readFileSync(join(__dirname, '/kitchen-sink.graphql'), {
+  encoding: 'utf8',
+});
+
+export const name = 'Parse kitchen sink';
+export function measure() {
+  parse(kitchenSink);
+}

yarn.lock

@@ -786,6 +786,17 @@ bcrypt-pbkdf@^1.0.0:
   dependencies:
     tweetnacl "^0.14.3"
 
+[email protected]:
+  version "0.2.4"
+  resolved "https://registry.yarnpkg.com/beautify-benchmark/-/beautify-benchmark-0.2.4.tgz#3151def14c1a2e0d07ff2e476861c7ed0e1ae39b"
+
+[email protected]:
+  version "2.1.4"
+  resolved "https://registry.yarnpkg.com/benchmark/-/benchmark-2.1.4.tgz#09f3de31c916425d498cc2ee565a0ebf3c2a5629"
+  dependencies:
+    lodash "^4.17.4"
+    platform "^1.3.3"
+
 binary-extensions@^1.0.0:
   version "1.11.0"
   resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-1.11.0.tgz#46aa1751fb6a2f93ee5e689bb1087d4b14c6c205"
@@ -2240,6 +2251,10 @@ pinkie@^2.0.0:
   version "2.0.4"
   resolved "https://registry.yarnpkg.com/pinkie/-/pinkie-2.0.4.tgz#72556b80cfa0d48a974e80e77248e80ed4f7f870"
 
+platform@^1.3.3:
+  version "1.3.5"
+  resolved "https://registry.yarnpkg.com/platform/-/platform-1.3.5.tgz#fb6958c696e07e2918d2eeda0f0bc9448d733444"
+
 pluralize@^7.0.0:
   version "7.0.0"
   resolved "https://registry.yarnpkg.com/pluralize/-/pluralize-7.0.0.tgz#298b89df8b93b0221dbf421ad2b1b1ea23fc6777"