Use standalone .js bundle in dist tarball rather than individual JS f…

…iles (#3030)

Instead of including all the raw JS files in the dist tarball, just use the single Yarn JS file that's built as part of the build, along with a few other files that are required. This significantly reduces the number of files in the tarball:

```
C:\src\yarn\dist (bundle-as-dist) ([email protected])
λ find .
.
./bin
./bin/node-gyp-bin
./bin/node-gyp-bin/node-gyp
./bin/node-gyp-bin/node-gyp.cmd
./bin/yarn
./bin/yarn.cmd
./bin/yarn.js
./bin/yarnpkg
./bin/yarnpkg.cmd
./lib
./lib/v8-compile-cache.js
./lib/yarn-cli.js
./LICENSE
./package.json
```

There are three .js files in the archive:
 - `lib/v8-compile-cache.js`:  Speeds up instantiation time by using the V8 code cache (https://www.npmjs.com/package/v8-compile-cache). This needs to be separate as it has to load **before** the bulk of the application code is loaded, so it can **not** be bundled
 - `lib/yarn-cli.js`: Contains all the bundled Yarn code
 - `bin/yarn.js`: Entry point to the app, just like today. Loads `v8-compile-cache` then loads `yarn-cli`

This change means that **only** the JavaScript files that are actually used are included, resulting in a nice file size reduction for the installation packages:
![](http://ss.dan.cx/2017/04/Yarn_bundle_dist_metrics_-_Google_Sheets_-_Google__01-13.51.49.png)

Differences are due to differing compression algorithms: Debian packages use xz or LZMA, RedHat uses gzip, Windows installer uses Cabinet

They're also slightly faster to extract:
![image 3](https://cloud.githubusercontent.com/assets/91933/24582332/483b41f4-16e2-11e7-9509-8024b1e78a39.png)

Testing was performed on my desktop computer (Intel Core i5 6500, Samsung 850 Evo 1TB SSD, Windows 10), with testing for Linux stuff (like installing the Debian package) tested in a Docker container.

Raw data: https://docs.google.com/spreadsheets/d/1d8jdf3DU_GUFdotlPl08PkYa8SkzStK2tgnQ54ivsm0/edit?usp=sharing

Performance is very slightly faster when using `v8-compile-cache` along with the bundled file, but it's not extremely significant (`yarn --version` went from 0.19s to 0.14s on my BuyVM server). The difference might be bigger on servers with slower disks (HDD) or with more overloaded servers.

I also deleted the `build-dist.ps1` file because we _should_ be able to assume that Bash is available on Windows, particularly if Git is installed (as it comes with Git Bash). I need to verify that this works on AppVeyor.

.gitignore

@@ -5,6 +5,7 @@
 /.nyc_output
 /coverage
 /dist
+/dist-debug
 /artifacts
 /updates
 /.roadrunner.json

.npmignore

@@ -24,6 +24,7 @@
 /scripts
 /coverage
 /dist
+/dist-debug
 /__tests__
 /.roadrunner.json
 .vs

appveyor.yml

@@ -14,9 +14,8 @@ install:
 
 build_script:
   - ps: if ($env:APPVEYOR_REPO_BRANCH -eq 'master') { node ./scripts/set-dev-version.js }
-  - npm run build
-  - ps: ./scripts/build-dist.ps1
-  - npm run build-win-installer
+  - yarn run build-dist
+  - yarn run build-win-installer
 
 test_script:
   - node --version

bin/yarn-bundle-entry.js

@@ -0,0 +1,7 @@
+#!/usr/bin/env node
+
+/* eslint-disable flowtype/require-valid-file-annotation */
+'use strict';
+
+require('../lib/v8-compile-cache');
+module.exports = require('../lib/yarn-cli');

circle.yml

@@ -31,10 +31,12 @@ test:
     - yarn lint
     - yarn test-ci -- -- --maxWorkers 3
     - yarn check-lockfile
-    - yarn build-dist
-    - node ./scripts/build-webpack.js
-    - ls -t1 artifacts | head -n2 | while read entry; do node "./artifacts/$entry" --version; done
-    - ./scripts/build-deb.sh
+    - yarn run build-dist
+    - yarn run build-deb
+    # Test that the standalone .js build works as expected
+    - ./artifacts/yarn-`./dist/bin/yarn --version`.js --version
+    - ./artifacts/yarn-legacy-`./dist/bin/yarn --version`.js --version
+
     # Test that installing as root works and that it also works
     # behind a user namespace which Circle CI tests are run under
     - sudo env "PATH=$PATH" bin/yarn install --force

package.json

@@ -92,7 +92,9 @@
     "test-only": "jest --coverage --verbose",
     "lint": "eslint . && flow check",
     "release-branch": "./scripts/release-branch.sh",
-    "build-dist": "./scripts/build-dist.sh",
+    "build-bundle": "node ./scripts/build-webpack.js",
+    "build-deb": "./scripts/build-deb.sh",
+    "build-dist": "bash ./scripts/build-dist.sh",
     "build-chocolatey": "powershell ./scripts/build-chocolatey.ps1",
     "build-win-installer": "scripts\\build-windows-installer.bat"
   },

scripts/build-deb.sh

@@ -27,30 +27,18 @@ mkdir -p $OUTPUT_DIR
 # Remove old packages
 rm -f $OUTPUT_DIR/*.deb $OUTPUT_DIR/*.rpm
 
-# Extract to a temporary directory
+# Create temporary directory to start building up the package
 rm -rf $PACKAGE_TMPDIR
 mkdir -p $PACKAGE_TMPDIR/
 umask 0022 # Ensure permissions are correct (0755 for dirs, 0644 for files)
-tar zxf $TARBALL_NAME -C $PACKAGE_TMPDIR/
 PACKAGE_TMPDIR_ABSOLUTE=$(readlink -f $PACKAGE_TMPDIR)
 
 # Create Linux package structure
 mkdir -p $PACKAGE_TMPDIR/usr/share/yarn/
 mkdir -p $PACKAGE_TMPDIR/usr/share/doc/yarn/
-mv $PACKAGE_TMPDIR/dist/bin $PACKAGE_TMPDIR/usr/share/yarn/
-mv $PACKAGE_TMPDIR/dist/lib $PACKAGE_TMPDIR/usr/share/yarn/
-mv $PACKAGE_TMPDIR/dist/lib-legacy $PACKAGE_TMPDIR/usr/share/yarn/
-mv $PACKAGE_TMPDIR/dist/node_modules $PACKAGE_TMPDIR/usr/share/yarn/
-mv $PACKAGE_TMPDIR/dist/package.json $PACKAGE_TMPDIR/usr/share/yarn/
+tar zxf $TARBALL_NAME -C $PACKAGE_TMPDIR/usr/share/yarn/ --strip 1
 cp resources/debian/copyright $PACKAGE_TMPDIR/usr/share/doc/yarn/copyright
 
-# These are unneeded and throw lintian lint errors
-rm -f $PACKAGE_TMPDIR/usr/share/yarn/node_modules/node-uuid/benchmark/bench.gnu
-find $PACKAGE_TMPDIR/usr/share/yarn \( -name '*.md' -o  -name '*.md~' -o -name '*.gitmodules' \) -delete
-
-# Assume everything else is junk we don't need
-rm -rf $PACKAGE_TMPDIR/dist
-
 # The Yarn executable expects to be in the same directory as the libraries, so
 # we can't just copy it directly to /usr/bin. Symlink them instead.
 mkdir -p $PACKAGE_TMPDIR/usr/bin/

scripts/build-dist-debug.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+set -ex
+
+# This is similar to build-dist.sh, except it includes the original .js files
+# rather than bundling them into a single .js file. This distribution can
+# potentially be useful for debugging purposes, but it's more bloated than the
+# regular distribution.
+
+npm run build
+npm pack
+rm -rf dist-debug
+mkdir dist-debug
+mkdir -p artifacts
+mv yarn-*.tgz dist-debug/pack.tgz
+
+cd dist-debug
+umask 0022 # Ensure permissions are correct (0755 for dirs, 0644 for files)
+tar -xzf pack.tgz --strip 1
+rm -rf pack.tgz
+# Change this to "yarn install --production" once #1115 is fixed
+npm install --production
+../scripts/set-installation-method.js $(readlink -f package.json) tar
+cd ..
+
+tar -cvzf artifacts/yarn-v`dist-debug/bin/yarn --version`.tar.gz dist-debug/*
+shasum -a 256 artifacts/yarn-*.tar.gz

scripts/build-dist.sh

@@ -1,23 +1,40 @@
-#!/bin/sh
-
+#!/bin/bash
 set -ex
+# Builds the release tarball for Yarn.
+
+umask 0022 # Ensure permissions are correct (0755 for dirs, 0644 for files)
 
-npm run build
-npm pack
+# Workaround for https://github.com/yarnpkg/yarn/issues/2591
+case "$(uname -s)" in
+  *CYGWIN*|MSYS*|MINGW*)
+    dist_yarn=dist/bin/yarn.cmd
+    system_yarn=yarn.cmd
+    ;;
+  *)
+    dist_yarn=dist/bin/yarn
+    system_yarn=yarn
+    ;;
+esac
+
+rm -rf artifacts dist
 rm -rf dist
-mkdir dist
-mkdir -p artifacts
-mv yarn-*.tgz dist/pack.tgz
+mkdir artifacts
+mkdir dist{,/bin,/lib}
 
-cd dist
-umask 0022 # Ensure permissions are correct (0755 for dirs, 0644 for files)
-tar -xzf pack.tgz --strip 1
-rm -rf pack.tgz
-# Change this to "yarn install --production" once #1115 is fixed
-npm install --production
-../scripts/clean-node-modules.sh
-../scripts/set-installation-method.js $(readlink -f package.json) tar
-cd ..
+# Workaround for https://github.com/yarnpkg/yarn/issues/2591
+eval $system_yarn run build
+eval $system_yarn run build-bundle
+chmod +x artifacts/*.js
+
+cp package.json dist/
+cp LICENSE dist/
+cp artifacts/yarn-legacy-*.js dist/lib/yarn-cli.js
+cp bin/yarn-bundle-entry.js dist/bin/yarn.js
+cp bin/{yarn,yarnpkg,*.cmd} dist/bin/
+cp -r bin/node-gyp-bin dist/bin/
+# We cannot bundle v8-compile-cache as it must be loaded separately to be effective.
+cp node_modules/v8-compile-cache/v8-compile-cache.js dist/lib/v8-compile-cache.js
 
-tar -cvzf artifacts/yarn-v`dist/bin/yarn --version`.tar.gz dist/*
-shasum -a 256 artifacts/yarn-*.tar.gz
+version=`exec $dist_yarn --version`
+./scripts/set-installation-method.js $(readlink -f dist/package.json) tar
+tar -cvzf artifacts/yarn-v$version.tar.gz dist/*

scripts/build-webpack.js

@@ -38,7 +38,7 @@ const compiler = webpack({
 });
 
 compiler.run((err, stats) => {
-  const {fileDependencies} = stats.compilation;
+  const fileDependencies = stats.compilation.fileDependencies;
   const filenames = fileDependencies.map(x => x.replace(basedir, ''));
   console.log(util.inspect(filenames, {maxArrayLength: null}));
 });