perf: get rid of xz

package.json

@@ -16,7 +16,7 @@
     "docker-images": "docker/build.sh",
     "update-deps": "node ./scripts/update-deps.js",
     "set-versions": "node test/out/helpers/setVersions.js",
-    "npm-publish": "yarn set-versions && yarn compile && ./scripts/npm-publish.sh && conventional-changelog -p angular -i CHANGELOG.md -s",
+    "release": "yarn set-versions && yarn compile && ./scripts/npm-publish.sh && conventional-changelog -p angular -i CHANGELOG.md -s",
     "schema": "typescript-json-schema packages/electron-builder/tsconfig.json Config --out packages/electron-builder/scheme.json --noExtraProps --useTypeOfKeyword --strictNullChecks --titles --required",
     "jsdoc": "ts2jsdoc packages/electron-builder-http packages/electron-updater packages/electron-builder-util packages/electron-builder packages/electron-publish",
     "jsdoc2md": "node scripts/jsdoc2md.js",
@@ -26,7 +26,7 @@
   "repository": "electron-userland/electron-builder",
   "///": "all dependencies for all packages (hoisted)",
   "dependencies": {
-    "7zip-bin": "^2.1.0",
+    "7zip-bin": "^2.2.2",
     "ajv": "^5.2.2",
     "ajv-keywords": "^2.1.0",
     "archiver": "^2.0.0",
@@ -63,7 +63,6 @@
     "source-map-support": "^0.4.16",
     "stat-mode": "^0.2.2",
     "temp-file": "^1.0.2",
-    "ts-jsdoc": "^2.0.5",
     "tunnel-agent": "^0.6.0",
     "update-notifier": "^2.2.0",
     "uuid-1345": "^0.99.6",
@@ -86,14 +85,13 @@
     "env-paths": "^1.0.0",
     "globby": "^6.1.0",
     "jest-cli": "^20.0.4",
-    "jest-environment-node-debug": "^2.0.0",
     "jest-junit": "^2.1.0",
     "jsdoc-to-markdown": "^3.0.0",
     "path-sort": "^0.1.0",
-    "shebang-loader": "^0.0.1",
-    "ts-babel": "^3.2.1",
+    "ts-babel": "^4.0.1",
+    "ts-jsdoc": "^2.0.5",
     "tslint": "^5.6.0",
-    "typescript": "^2.4.2",
+    "typescript": "^2.5.0",
     "v8-compile-cache": "^1.1.0",
     "whitespace": "^2.1.0",
     "worker-farm": "^1.5.0",

packages/electron-builder/src/targets/archive.ts

@@ -1,4 +1,4 @@
-import { path7za } from "7zip-bin"
+import { path7x, path7za } from "7zip-bin"
 import { debug7z, debug7zArgs, isMacOsSierra, spawn } from "electron-builder-util"
 import { computeEnv, getLinuxToolsPath } from "electron-builder-util/out/bundledTool"
 import { exists } from "electron-builder-util/out/fs"
@@ -7,12 +7,12 @@ import * as path from "path"
 import { CompressionLevel } from "../core"
 
 class CompressionDescriptor {
-  constructor(public flag: string, public env: string, public minLevel: string, public maxLevel: string = "-9") {
+  constructor(readonly flag: string, readonly env: string, readonly minLevel: string, readonly maxLevel: string = "-9") {
   }
 }
 
 const extToCompressionDescriptor: { [key: string]: CompressionDescriptor; } = {
-  "tar.xz": new CompressionDescriptor("--xz", "XZ_OPT", "-0", "-9e"),
+  "tar.xz": new CompressionDescriptor(`-I'${path7x}'`, "XZ_OPT", "-0", "-9e"),
   "tar.lz": new CompressionDescriptor("--lzip", "LZOP", "-0"),
   "tar.gz": new CompressionDescriptor("--gz", "GZIP", "-1"),
   "tar.bz2": new CompressionDescriptor("--bzip2", "BZIP2", "-1"),
@@ -45,13 +45,14 @@ export async function tar(compression: CompressionLevel | null | undefined, form
       ...tarEnv,
       PATH: computeEnv(process.env.PATH, [path.join(linuxToolsPath, "bin")]),
       LANG: "en_US.UTF-8",
-      LC_CTYPE: "UTF-8"
+      LC_CTYPE: "UTF-8",
+      SZA_PATH: path7za,
     }
   }
 
   await spawn(process.platform === "darwin" || process.platform === "freebsd" ? "gtar" : "tar", args, {
     cwd: isMacApp ? path.dirname(dirToArchive) : dirToArchive,
-    env: tarEnv
+    env: tarEnv,
   })
   return outFile
 }
@@ -93,6 +94,7 @@ export function addZipArgs(args: Array<string>) {
   args.push("-mcu")
   // disable "Stores NTFS timestamps for files: Modification time, Creation time, Last access time." to produce the same archive for the same data
   args.push("-mtc=off")
+  // noinspection SpellCheckingInspection
   args.push("-tzip")
 }
 

packages/electron-builder/src/targets/fpm.ts

@@ -1,30 +1,37 @@
+import { path7x, path7za } from "7zip-bin"
 import BluebirdPromise from "bluebird-lst"
 import { Arch, debug, exec, isMacOsSierra, log, smarten, TmpDir, toLinuxArchString, use, warn } from "electron-builder-util"
-import { getBin } from "electron-builder-util/out/binDownload"
+import { getBin, getBinFromGithub } from "electron-builder-util/out/binDownload"
 import { computeEnv, getLinuxToolsPath } from "electron-builder-util/out/bundledTool"
 import { unlinkIfExists } from "electron-builder-util/out/fs"
 import { ensureDir, outputFile, readFile } from "fs-extra-p"
+import { Lazy } from "lazy-val"
 import * as path from "path"
 import { Target } from "../core"
 import * as errorMessages from "../errorMessages"
 import { LinuxPackager } from "../linuxPackager"
 import { DebOptions, LinuxTargetSpecificOptions } from "../options/linuxOptions"
 import { installPrefix, LinuxTargetHelper } from "./LinuxTargetHelper"
 
-const fpmPath = (process.platform === "win32" || process.env.USE_SYSTEM_FPM === "true") ?
-  BluebirdPromise.resolve("fpm") : downloadFpm()
+const fpmPath = new Lazy(() => {
+  if (process.platform === "win32" || process.env.USE_SYSTEM_FPM === "true") {
+    return BluebirdPromise.resolve("fpm")
+  }
 
-// can be called in parallel, all calls for the same version will get the same promise - will be downloaded only once
-function downloadFpm(): Promise<string> {
-  const version = process.platform === "darwin" ? "fpm-1.8.1-20150715-2.2.2" : "fpm-1.8.1-2.3.1"
   const osAndArch = process.platform === "darwin" ? "mac" : `linux-x86${process.arch === "ia32" ? "" : "_64"}`
-  //noinspection SpellCheckingInspection
-  const sha2 = process.platform === "darwin" ? "97352e184a1f54e5ed0d12f38ac383edebbe421db5a3fb59898e8c9a1c407ed7" :
-    (process.arch === "ia32" ? "8380331f7d9762a36d7c7181501c3fc9342745b8499b962f6ea37c7dc3778f99" : "6538fcd2486c2831949562abfd0017b67eff502addad5b444baec4899b0babc6")
 
+  if (process.platform === "darwin") {
+    //noinspection SpellCheckingInspection
+    return getBinFromGithub("fpm", "1.9.2-20150715-2.2.2-mac", "6sZZoRKkxdmv3a6E5dnZgVl23apGnImhDtGHKhgCE1WOtXBUJnx+w0WvB2HD2/sitz4f93Mf7+QqDCIbfP7LOw==")
+      .then(it => path.join(it, "fpm"))
+  }
+
+  const version = "fpm-1.8.1-2.3.1"
+  //noinspection SpellCheckingInspection
+  const sha2 = (process.arch === "ia32" ? "8380331f7d9762a36d7c7181501c3fc9342745b8499b962f6ea37c7dc3778f99" : "6538fcd2486c2831949562abfd0017b67eff502addad5b444baec4899b0babc6")
   return getBin("fpm", version, `https://dl.bintray.com/electron-userland/bin/${version}-${osAndArch}.7z`, sha2)
     .then(it => path.join(it, "fpm"))
-}
+})
 
 export default class FpmTarget extends Target {
   readonly options: LinuxTargetSpecificOptions = {...this.packager.platformSpecificBuildOptions, ...(this.packager.config as any)[this.name]}
@@ -131,7 +138,7 @@ export default class FpmTarget extends Target {
     }
 
     if (target === "deb") {
-      args.push("--deb-compression", (options as DebOptions).compression || (packager.config.compression === "store" ? "gz" : "xz"))
+      args.push("--deb-compression", (options as DebOptions).compression || "xz")
       use((options as DebOptions).priority, it => args.push("--deb-priority", it!))
     }
     else if (target === "rpm") {
@@ -192,7 +199,10 @@ export default class FpmTarget extends Target {
     const env = {
       ...process.env,
       LANG: "en_US.UTF-8",
-      LC_CTYPE: "UTF-8"
+      LC_CTYPE: "UTF-8",
+      FPM_COMPRESS_PROGRAM: path7x,
+      SZA_PATH: path7za,
+      SZA_COMPRESSION_LEVEL: packager.config.compression === "store" ? "0" : "9",
     }
 
     // rpmbuild wants directory rpm with some default config files. Even if we can use dylibbundler, path to such config files are not changed (we need to replace in the binary)
@@ -204,7 +214,7 @@ export default class FpmTarget extends Target {
         DYLD_LIBRARY_PATH: computeEnv(process.env.DYLD_LIBRARY_PATH, [path.join(linuxToolsPath, "lib")]),
       })
     }
-    await exec(await fpmPath, args, {env})
+    await exec(await fpmPath.value, args, {env})
 
     this.packager.dispatchArtifactCreated(destination, this, arch)
   }
@@ -221,7 +231,7 @@ async function writeConfigFile(tmpDir: TmpDir, templatePath: string, options: an
     }
   }
   const config = (await readFile(templatePath, "utf8"))
-    .replace(/\$\{([a-zA-Z]+)\}/g, replacer)
+    .replace(/\${([a-zA-Z]+)}/g, replacer)
     .replace(/<%=([a-zA-Z]+)%>/g, (match, p1) => {
       warn("<%= varName %> is deprecated, please use ${varName} instead")
       return replacer(match, p1.trim())

packages/electron-builder/src/util/config.ts

@@ -1,4 +1,4 @@
-import Ajv from "ajv"
+import Ajv, { AdditionalPropertiesParams, ErrorObject, TypeParams } from "ajv"
 import { asArray, debug, log, warn } from "electron-builder-util"
 import { statOrNull } from "electron-builder-util/out/fs"
 import { readJson } from "fs-extra-p"
@@ -8,9 +8,6 @@ import { getConfig as _getConfig, loadParentConfig, orNullIfFileNotExist, ReadCo
 import { deepAssign } from "read-config-file/out/deepAssign"
 import { Config } from "../metadata"
 import { reactCra } from "../presets/rectCra"
-import AdditionalPropertiesParams = ajv.AdditionalPropertiesParams
-import ErrorObject = ajv.ErrorObject
-import TypeParams = ajv.TypeParams
 
 /** @internal */
 export async function getConfig(projectDir: string, configPath: string | null, configFromOptions: Config | null | undefined, packageMetadata: Lazy<{ [key: string]: any } | null> = new Lazy(() => orNullIfFileNotExist(readJson(path.join(projectDir, "package.json"))))): Promise<Config> {
@@ -58,15 +55,13 @@ export async function getConfig(projectDir: string, configPath: string | null, c
   return deepAssign(parentConfig, config)
 }
 
-let validatorPromise: Promise<any> | null = null
-
-async function createConfigValidator() {
+const validatorPromise = new Lazy(async () => {
   const ajv = new Ajv({allErrors: true, coerceTypes: true})
   ajv.addMetaSchema(require("ajv/lib/refs/json-schema-draft-04.json"))
   require("ajv-keywords")(ajv, ["typeof"])
   const schema = await readJson(path.join(__dirname, "..", "..", "scheme.json"))
   return ajv.compile(schema)
-}
+})
 
 /** @internal */
 export async function validateConfig(config: Config) {
@@ -85,14 +80,10 @@ export async function validateConfig(config: Config) {
     config.buildDependenciesFromSource = false
   }
 
-  if (validatorPromise == null) {
-    validatorPromise = createConfigValidator()
-  }
-
-  const validator = await validatorPromise
+  const validator = await validatorPromise.value
   if (!validator(config)) {
     debug(JSON.stringify(validator.errors, null, 2))
-    throw new Error(`Config is invalid:
+    throw new Error(`Configuration is invalid:
 ${JSON.stringify(normaliseErrorMessages(validator.errors!), null, 2)}
 
 How to fix:

packages/tsconfig-base.json

@@ -1,7 +1,6 @@
 {
   "extends": "../node_modules/ts-babel/tsconfig-base.json",
   "compilerOptions": {
-    "moduleResolution": "Node",
     "baseUrl": ".",
     "stripInternal": true
   }

test/jestSetup.js

@@ -66,4 +66,7 @@ if (process.env.TEST_APP_TMP_DIR == null) {
 
 if (!process.env.USE_HARD_LINKS) {
   process.env.USE_HARD_LINKS = "true"
+}
+if (!process.env.SZA_COMPRESSION_LEVEL) {
+  process.env.SZA_COMPRESSION_LEVEL = "0"
 }

test/out/__snapshots__/ExtraBuildTest.js.snap

@@ -117,7 +117,7 @@ How to fix:
 `;
 
 exports[`scheme validation 2 1`] = `
-"Config is invalid:
+"Configuration is invalid:
 {
   \\"appId\\": \\"Should be null,string\\"
 }

test/out/linux/__snapshots__/debTest.js.snap

@@ -84,7 +84,7 @@ Object {
   "Package": "testapp",
   "Priority": "extra",
   "Section": "devel",
-  "Size": "95919",
+  "Size": "95920",
   "Vendor": "Foo Bar <[email protected]>",
 }
 `;

test/out/mac/__snapshots__/macArchiveTest.js.snap

@@ -13,7 +13,7 @@ Object {
 `;
 
 exports[`invalid target 1`] = `
-"Config is invalid:
+"Configuration is invalid:
 {
   \\"mac\\": [
     {

test/src/helpers/packTester.ts

@@ -1,6 +1,7 @@
+import { path7x, path7za } from "7zip-bin"
 import BluebirdPromise from "bluebird-lst"
 import DecompressZip from "decompress-zip"
-import { Arch, ArtifactCreated, Config, DIR_TARGET, getArchSuffix, MacOsTargetName, Packager, PackagerOptions, Platform, Target } from "electron-builder"
+import { Arch, ArtifactCreated, DIR_TARGET, getArchSuffix, MacOsTargetName, Packager, PackagerOptions, Platform, Target } from "electron-builder"
 import { CancellationToken } from "electron-builder-http"
 import { convertVersion } from "electron-builder-squirrel-windows/out/squirrelPack"
 import { addValue, exec, log, spawn, warn } from "electron-builder-util"
@@ -214,7 +215,7 @@ async function packAndCheck(packagerOptions: PackagerOptions, checkOptions: Asse
         await checkMacResult(packager, packagerOptions, checkOptions, packedAppDir)
       }
       else if (platform === Platform.LINUX) {
-        await checkLinuxResult(outDir, packager, arch, nameToTarget, packagerOptions.config != null && (packagerOptions.config as Config).compression === "normal")
+        await checkLinuxResult(outDir, packager, arch, nameToTarget)
       }
       else if (platform === Platform.WINDOWS) {
         await checkWindowsResult(packager, checkOptions, artifacts.get(platform)!!, nameToTarget)
@@ -225,16 +226,16 @@ async function packAndCheck(packagerOptions: PackagerOptions, checkOptions: Asse
   return {packager, outDir}
 }
 
-async function checkLinuxResult(outDir: string, packager: Packager, arch: Arch, nameToTarget: Map<string, Target>, isNormalCompression: boolean) {
+async function checkLinuxResult(outDir: string, packager: Packager, arch: Arch, nameToTarget: Map<string, Target>) {
   if (!nameToTarget.has("deb")) {
     return
   }
 
   const appInfo = packager.appInfo
   const packageFile = `${outDir}/TestApp_${appInfo.version}_${arch === Arch.ia32 ? "i386" : (arch === Arch.x64 ? "amd64" : "armv7l")}.deb`
-  expect(await getContents(packageFile, isNormalCompression)).toMatchSnapshot()
+  expect(await getContents(packageFile)).toMatchSnapshot()
   if (arch === Arch.ia32) {
-    expect(await getContents(`${outDir}/TestApp_${appInfo.version}_i386.deb`, isNormalCompression)).toMatchSnapshot()
+    expect(await getContents(`${outDir}/TestApp_${appInfo.version}_i386.deb`)).toMatchSnapshot()
   }
 
   const control = parseDebControl(await execShell(`ar p '${packageFile}' control.tar.gz | ${await getTarExecutable()} zx --to-stdout ./control`, {
@@ -350,14 +351,15 @@ async function getTarExecutable() {
   return process.platform === "darwin" ? path.join(await getLinuxToolsPath(), "bin", "gtar") : "tar"
 }
 
-async function getContents(packageFile: string, isNormalCompression: boolean) {
+async function getContents(packageFile: string) {
   // without LC_CTYPE dpkg can returns encoded unicode symbols
-  const result = await execShell(`ar p '${packageFile}' data.tar.${isNormalCompression ? "xz" : "gz"} | ${await getTarExecutable()} ${isNormalCompression ? "J" : "z"}t`, {
+  const result = await execShell(`ar p '${packageFile}' data.tar.xz | ${await getTarExecutable()} -t -I'${path7x}'`, {
     maxBuffer: 10 * 1024 * 1024,
     env: {
       ...process.env,
       LANG: "en_US.UTF-8",
-      LC_CTYPE: "UTF-8"
+      LC_CTYPE: "UTF-8",
+      SZA_PATH: path7za,
     }
   })
   return pathSorter(parseFileList(result, true)

test/src/helpers/runTests.ts

@@ -173,10 +173,16 @@ async function runTests() {
     runInBand,
     testPathPattern: args.length > 0 ? args.join("|") : null,
   }, [rootDir], (result: any) => {
-    process.exitCode = !result || result.success ? 0 : 1
+    const exitCode = !result || result.success ? 0 : 1
+    process.exitCode = exitCode
     remove(TEST_DIR)
       .catch(e => {
         console.error(e.stack)
       })
+
+    // strange, without this code process exit code always 0
+    if (exitCode > 0) {
+      process.on("exit", () => process.exit(exitCode))
+    }
   })
 }

test/src/linux/debTest.ts

@@ -4,10 +4,6 @@ import { app } from "../helpers/packTester"
 
 test.ifNotWindows("deb", app({
   targets: Platform.LINUX.createTarget("deb"),
-  config: {
-    // to test xz (on macos we bundle xz and gnu-tar and it should be tested)
-    compression: "normal"
-  }
 }))
 
 test.ifNotWindows("arm", app({targets: Platform.LINUX.createTarget("deb", Arch.armv7l)}))
@@ -22,7 +18,7 @@ test.ifNotWindows("custom depends", app({
       depends: ["foo"],
     },
   },
-  effectiveOptionComputed: async (it) => {
+  effectiveOptionComputed: async it => {
     const content = await readFile(it[1], "utf8")
     expect(content).toMatchSnapshot()
     return false