fix: Building Linux AppImage in >19.33 sometimes fails with tar error…

… "file changed as we read it

Close #2187

README.md

@@ -7,7 +7,7 @@ A complete solution to package and build a ready for distribution Electron app f
 * [Code Signing](https://www.electron.build/code-signing) on a CI server or development machine.
 * [Auto Update](https://www.electron.build/auto-update) ready application packaging.
 * Numerous target formats:
-  * All platforms: `7z`, `zip`, `tar.xz`, `tar.lz`, `tar.gz`, `tar.bz2`, `dir` (unpacked directory).
+  * All platforms: `7z`, `zip`, `tar.xz`, `tar.7z`, `tar.lz`, `tar.gz`, `tar.bz2`, `dir` (unpacked directory).
   * [macOS](https://www.electron.build/configuration/mac): `dmg`, `pkg`, `mas`.
   * [Linux](https://www.electron.build/configuration/linux): [AppImage](http://appimage.org), [snap](http://snapcraft.io), debian package (`deb`), `rpm`, `freebsd`, `pacman`, `p5p`, `apk`.
   * [Windows](https://www.electron.build/configuration/win): `nsis` (Installer), `nsis-web` (Web installer), `portable` (portable app without installation), AppX (Windows Store), Squirrel.Windows.
@@ -88,15 +88,14 @@ See [Command Line Interface](https://www.electron.build/cli).
 ## Programmatic Usage
 See `node_modules/electron-builder/out/index.d.ts`. Typings for TypeScript is provided.
 
+To build for current platform and current arch:
 ```js
 "use strict"
 
 const builder = require("electron-builder")
-const Platform = builder.Platform
 
 // Promise is returned
 builder.build({
-  targets: Platform.MAC.createTarget(),
   config: {
    "//": "build options, see https://goo.gl/ZhRfla"
   }
@@ -109,6 +108,10 @@ builder.build({
   })
 ```
 
+Add `win: []` to build for Windows default target. Add `win: ["nsis-web]` to build specified target (web installer) for Windows. The same for `mac: []` and `linux: []`
+
+Add `ia32: true` to build `ia32` (or `x64: true`, or `armv7l: true`). Several can be specified and built at once.
+
 ## Pack Only in a Distributable Format
 
 You can use electron-builder only to pack your electron app in a AppImage, Snaps, Debian package, NSIS, macOS installer component package (`pkg`) 

package.json

@@ -10,7 +10,7 @@
     "///": "Please see https://github.com/electron-userland/electron-builder/blob/master/CONTRIBUTING.md#run-test-using-cli how to run particular test instead full (and very slow) run",
     "test": "node ./test/out/helpers/runTests.js skipArtifactPublisher ALL_TESTS=isCi",
     "test-all": "yarn pretest && node ./test/out/helpers/runTests.js",
-    "test-linux": "docker run --rm -ti --env TEST_FILES='linuxPackagerTest' -v ${PWD}:/project -v ${PWD##*/}-node-modules:/project/node_modules -v ~/.electron:/root/.electron electronuserland/builder:wine /bin/bash -c \"yarn && yarn test\"",
+    "test-linux": "docker run --rm -ti -v ~/Library/Caches/electron:/root/.cache/electron -v ${PWD}:/project -v ${PWD##*/}-node-modules:/project/node_modules -v ~/.electron:/root/.electron electronuserland/builder:wine /bin/bash -c \"yarn && TEST_FILES=linux node ./test/out/helpers/runTests.js\"",
     "whitespace": "whitespace 'src/**/*.ts'",
     "docker-images": "docker/build.sh",
     "update-deps": "npm-check-updates -a -x gitbook-plugin-github && node ./scripts/update-deps.js",
@@ -28,10 +28,10 @@
   "///": "All dependencies for all packages (hoisted)",
   "////": "All typings are added into root `package.json` to avoid duplication errors in the IDE compiler (several `node.d.ts` files).",
   "dependencies": {
-    "7zip-bin": "^2.2.4",
+    "7zip-bin": "^2.2.7",
     "archiver": "^2.0.3",
     "async-exit-hook": "^2.0.1",
-    "aws-sdk": "^2.131.0",
+    "aws-sdk": "^2.132.0",
     "bluebird-lst": "^1.0.4",
     "chalk": "^2.1.0",
     "chromium-pickle-js": "^0.2.0",

packages/builder-util/package.json

@@ -21,7 +21,7 @@
     "node-emoji": "^1.8.1",
     "builder-util-runtime": "^0.0.0-semantic-release",
     "source-map-support": "^0.5.0",
-    "7zip-bin": "^2.2.4",
+    "7zip-bin": "^2.2.7",
     "ini": "^1.3.4",
     "tunnel-agent": "^0.6.0",
     "semver": "^5.4.1",

packages/builder-util/src/bundledTool.ts

@@ -23,5 +23,5 @@ export function computeToolEnv(libPath: Array<string>): any {
 
 export function getLinuxToolsPath() {
   //noinspection SpellCheckingInspection
-  return getBinFromGithub("linux-tools", "mac-10.12.2", "i+D1SGCPSKNuR4wZd/lpiW5l1emfX1MgkNwESqOXKgh5SpN3TNV9oi0W6zD9fEDwCBJaMYsZJtSjulh16TzKEA==")
+  return getBinFromGithub("linux-tools", "mac-10.12.3", "SQ8fqIRVXuQVWnVgaMTDWyf2TLAJjJYw3tRSqQJECmgF6qdM7Kogfa6KD49RbGzzMYIFca9Uw3MdsxzOPRWcYw==")
 }

packages/electron-builder/package.json

@@ -51,7 +51,7 @@
   "bugs": "https://github.com/electron-userland/electron-builder/issues",
   "homepage": "https://github.com/electron-userland/electron-builder",
   "dependencies": {
-    "7zip-bin": "^2.2.4",
+    "7zip-bin": "^2.2.7",
     "async-exit-hook": "^2.0.1",
     "bluebird-lst": "^1.0.4",
     "chalk": "^2.1.0",

packages/electron-builder/src/targets/ArchiveTarget.ts

@@ -21,7 +21,7 @@ export class ArchiveTarget extends Target {
     // tslint:disable:no-invalid-template-strings
     const outFile = path.join(this.outDir, packager.expandArtifactNamePattern(this.options, format, arch, packager.platform === Platform.LINUX ? "${name}-${version}-${arch}.${ext}" : "${productName}-${version}-${arch}-${os}.${ext}"))
     if (format.startsWith("tar.")) {
-      await tar(packager.compression, format, outFile, appOutDir, isMac)
+      await tar(packager.compression, format, outFile, appOutDir, isMac, packager.info.tempDirManager)
     }
     else {
       await archive(packager.compression, format, outFile, appOutDir, {withoutDir: !isMac})

packages/electron-builder/src/targets/appImage.ts

@@ -17,6 +17,7 @@ const appRunTemplate = new Lazy<(data: any) => string>(async () => {
   return ejs.compile(await readFile(path.join(getTemplatePath("linux"), "AppRun.sh"), "utf-8"))
 })
 
+// https://unix.stackexchange.com/questions/375191/append-to-sub-directory-inside-squashfs-file
 export default class AppImageTarget extends Target {
   readonly options: AppImageOptions = {...this.packager.platformSpecificBuildOptions, ...(this.packager.config as any)[this.name]}
   private readonly desktopEntry: Lazy<string>

packages/electron-builder/src/targets/archive.ts

@@ -1,58 +1,79 @@
-import { path7x, path7za } from "7zip-bin"
-import { debug7z, debug7zArgs, isMacOsSierra, spawn } from "builder-util"
-import { computeEnv, getLinuxToolsPath } from "builder-util/out/bundledTool"
-import { exists } from "builder-util/out/fs"
-import { unlink } from "fs-extra-p"
+import { path7za } from "7zip-bin"
+import { debug7z, debug7zArgs, exec } from "builder-util"
+import { exists, unlinkIfExists } from "builder-util/out/fs"
 import * as path from "path"
 import { CompressionLevel } from "../core"
-
-class CompressionDescriptor {
-  constructor(readonly flag: string, readonly env: string, readonly minLevel: string, readonly maxLevel: string = "-9") {
-  }
-}
-
-const extToCompressionDescriptor: { [key: string]: CompressionDescriptor; } = {
-  "tar.xz": new CompressionDescriptor(`-I'${path7x}'`, "XZ_OPT", "-0", "-9e"),
-  "tar.lz": new CompressionDescriptor("--lzip", "LZOP", "-0"),
-  "tar.gz": new CompressionDescriptor("--gz", "GZIP", "-1"),
-  "tar.bz2": new CompressionDescriptor("--bzip2", "BZIP2", "-1"),
-}
+import { TmpDir } from "temp-file"
+import BluebirdPromise from "bluebird-lst"
+import { getLinuxToolsPath } from "builder-util/out/bundledTool"
+import { move } from "fs-extra-p"
 
 /** @internal */
-export async function tar(compression: CompressionLevel | null | undefined, format: string, outFile: string, dirToArchive: string, isMacApp: boolean = false) {
-  // we don't use 7z here - develar: I spent a lot of time making pipe working - but it works on MacOS and often hangs on Linux (even if use pipe-io lib)
-  // and in any case it is better to use system tools (in the light of docker - it is not problem for user because we provide complete docker image).
-  const info = extToCompressionDescriptor[format]
-  let tarEnv = process.env
-  if (process.env.ELECTRON_BUILDER_COMPRESSION_LEVEL != null) {
-    tarEnv = {...tarEnv}
-    tarEnv[info.env] = "-" + process.env.ELECTRON_BUILDER_COMPRESSION_LEVEL
-  }
-  else if (compression != null && compression !== "normal") {
-    tarEnv = {...tarEnv}
-    tarEnv[info.env] = compression === "store" ? info.minLevel : info.maxLevel
-  }
+export async function tar(compression: CompressionLevel | any | any, format: string, outFile: string, dirToArchive: string, isMacApp: boolean, tempDirManager: TmpDir): Promise<void> {
+  const tarFile = await tempDirManager.getTempFile({suffix: ".tar"})
+  const tarArgs = debug7zArgs("a")
+  tarArgs.push(tarFile)
+  tarArgs.push(path.basename(dirToArchive))
+
+  await BluebirdPromise.all([
+    exec(path7za, tarArgs, {cwd: path.dirname(dirToArchive)}),
+    // remove file before - 7z doesn't overwrite file, but update
+    unlinkIfExists(outFile),
+  ])
 
-  const args = [info.flag, "-cf", outFile]
   if (!isMacApp) {
-    args.push("--transform", `s,^\\.,${path.basename(outFile, "." + format)},`)
-  }
-  args.push(isMacApp ? path.basename(dirToArchive) : ".")
-
-  if (await isMacOsSierra()) {
-    const linuxToolsPath = await getLinuxToolsPath()
-    tarEnv = {
-      ...tarEnv,
-      PATH: computeEnv(process.env.PATH, [path.join(linuxToolsPath, "bin")]),
-      SZA_PATH: path7za,
-    }
+    await exec(path7za, ["rn", tarFile, path.basename(dirToArchive), path.basename(outFile, `.${format}`)])
   }
 
-  await spawn(process.platform === "darwin" || process.platform === "freebsd" ? "gtar" : "tar", args, {
-    cwd: isMacApp ? path.dirname(dirToArchive) : dirToArchive,
-    env: tarEnv,
-  })
-  return outFile
+  if (format === "tar.lz") {
+    // noinspection SpellCheckingInspection
+    let lzipPath = "lzip"
+    if (process.platform === "darwin") {
+      lzipPath = path.join(await getLinuxToolsPath(), "bin", lzipPath)
+    }
+    await exec(lzipPath, [compression === "store" ? "-1" : "-9", "--keep" /* keep (don't delete) input files */, tarFile])
+    // bloody lzip creates file in the same dir where input file with postfix `.lz`, option --output doesn't work
+    await move(`${tarFile}.lz`, outFile)
+    return
+  }
+
+  const args = compute7zCompressArgs(compression, format === "tar.xz" ? "xz" : (format === "tar.bz2" ? "bzip2" : "gzip"), {isRegularFile: true})
+  args.push(outFile, tarFile)
+  await exec(path7za, args, {
+    cwd: path.dirname(dirToArchive),
+  }, debug7z.enabled)
+
+  // const info: CompressionDescriptor | null = extToCompressionDescriptor[format]
+  // const tarEnv: any = {
+  //   ...process.env,
+  //   SZA_PATH: path7za,
+  //   SZA_ARCHIVE_TYPE: format === "tar.xz" ? "xz" : (format === "tar.bz2" ? "bzip2" : "gzip"),
+  //   SZA_COMPRESSION_LEVEL: process.env.ELECTRON_BUILDER_COMPRESSION_LEVEL || (compression === "store" ? "0" : "9")
+  // }
+  //
+  // if (info != null) {
+  //   if (process.env.ELECTRON_BUILDER_COMPRESSION_LEVEL != null) {
+  //     tarEnv[info.env] = "-" + process.env.ELECTRON_BUILDER_COMPRESSION_LEVEL
+  //   }
+  //   else if (compression != null && compression !== "normal") {
+  //     tarEnv[info.env] = compression === "store" ? info.minLevel : info.maxLevel
+  //   }
+  // }
+  //
+  // const args = [info == null ? `-I'${path7x}'` : info.flag, "-cf", outFile]
+  // if (!isMacApp) {
+  //   args.push("--transform", `s,^\\.,${path.basename(outFile, `.${format}`)},`)
+  // }
+  // args.push(isMacApp ? path.basename(dirToArchive) : ".")
+  //
+  // if (await isMacOsSierra()) {
+  //   tarEnv.PATH = computeEnv(process.env.PATH, [path.join(await getLinuxToolsPath(), "bin")])
+  // }
+  //
+  // await exec(process.platform === "darwin" || process.platform === "freebsd" ? "gtar" : "tar", args, {
+  //   cwd: isMacApp ? path.dirname(dirToArchive) : dirToArchive,
+  //   env: tarEnv,
+  // })
 }
 
 export interface ArchiveOptions {
@@ -77,6 +98,8 @@ export interface ArchiveOptions {
   excluded?: Array<string>
 
   method?: "Copy" | "LZMA" | "Deflate"
+
+  isRegularFile?: boolean
 }
 
 export function compute7zCompressArgs(compression: CompressionLevel | any | any, format: string, options: ArchiveOptions = {}) {
@@ -107,7 +130,9 @@ export function compute7zCompressArgs(compression: CompressionLevel | any | any,
   // https://stackoverflow.com/questions/27136783/7zip-produces-different-output-from-identical-input
   // tc and ta are off by default, but to be sure, we explicitly set it to off
   // disable "Stores NTFS timestamps for files: Modification time, Creation time, Last access time." to produce the same archive for the same data
-  args.push("-mtc=off")
+  if (!options.isRegularFile) {
+    args.push("-mtc=off")
+  }
 
   if (format === "7z" || format.endsWith(".7z")) {
     if (options.solid === false) {
@@ -126,7 +151,7 @@ export function compute7zCompressArgs(compression: CompressionLevel | any | any,
   if (options.method != null) {
     args.push(`-mm=${options.method}`)
   }
-  else if (format === "zip" || storeOnly) {
+  else if (!options.isRegularFile && (format === "zip" || storeOnly)) {
     args.push(`-mm=${storeOnly ? "Copy" : "Deflate"}`)
   }
 
@@ -144,22 +169,17 @@ export function compute7zCompressArgs(compression: CompressionLevel | any | any,
 export async function archive(compression: CompressionLevel | null | undefined, format: string, outFile: string, dirToArchive: string, options: ArchiveOptions = {}): Promise<string> {
   const args = compute7zCompressArgs(compression, format, options)
   // remove file before - 7z doesn't overwrite file, but update
-  try {
-    await unlink(outFile)
-  }
-  catch (e) {
-    // ignore
-  }
+  await unlinkIfExists(outFile)
 
   args.push(outFile, options.listFile == null ? (options.withoutDir ? "." : path.basename(dirToArchive)) : `@${options.listFile}`)
   if (options.excluded != null) {
     args.push(...options.excluded)
   }
 
   try {
-    await spawn(path7za, args, {
+    await exec(path7za, args, {
       cwd: options.withoutDir ? dirToArchive : path.dirname(dirToArchive),
-    }, {isDebugEnabled: debug7z.enabled})
+    }, debug7z.enabled)
   }
   catch (e) {
     if (e.code === "ENOENT" && !(await exists(dirToArchive))) {

packages/electron-builder/src/targets/fpm.ts

@@ -200,6 +200,7 @@ export default class FpmTarget extends Target {
       FPM_COMPRESS_PROGRAM: path7x,
       SZA_PATH: path7za,
       SZA_COMPRESSION_LEVEL: packager.compression === "store" ? "0" : "9",
+      SZA_ARCHIVE_TYPE: "xz",
     }
 
     // rpmbuild wants directory rpm with some default config files. Even if we can use dylibbundler, path to such config files are not changed (we need to replace in the binary)

packages/electron-publisher-s3/package.json

@@ -12,7 +12,7 @@
   ],
   "dependencies": {
     "fs-extra-p": "^4.4.3",
-    "aws-sdk": "^2.131.0",
+    "aws-sdk": "^2.132.0",
     "mime": "^2.0.3",
     "electron-publish": "~0.0.0-semantic-release",
     "builder-util": "^0.0.0-semantic-release",

test/src/helpers/runTests.ts

@@ -38,9 +38,6 @@ async function runTests() {
   const args = []
   if (!isEmptyOrSpaces(testFiles)) {
     args.push(...testFiles!!.split(",").map(it => `${it.trim()}.js`))
-    if (process.platform === "linux") {
-      args.push("httpRequestTest.js", "RepoSlugTest.js")
-    }
   }
   else if (!isEmptyOrSpaces(process.env.CIRCLE_NODE_INDEX)) {
     const circleNodeIndex = parseInt(process.env.CIRCLE_NODE_INDEX!!, 10)

yarn.lock

@@ -14,9 +14,9 @@
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/7zip-bin-win/-/7zip-bin-win-2.1.1.tgz#8acfc28bb34e53a9476b46ae85a97418e6035c20"
 
-"7zip-bin@^2.2.4":
-  version "2.2.4"
-  resolved "https://registry.yarnpkg.com/7zip-bin/-/7zip-bin-2.2.4.tgz#5d0a7da759258b7fa59121fddcec7cb65938a85c"
+"7zip-bin@^2.2.7":
+  version "2.2.7"
+  resolved "https://registry.yarnpkg.com/7zip-bin/-/7zip-bin-2.2.7.tgz#724802b8d6bda0bf2cfe61a4b86a820efc8ece93"
   optionalDependencies:
     "7zip-bin-linux" "^1.1.0"
     "7zip-bin-mac" "^1.0.1"
@@ -453,9 +453,9 @@ asynckit@^0.4.0:
   version "0.4.0"
   resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79"
 
-aws-sdk@^2.131.0:
-  version "2.131.0"
-  resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.131.0.tgz#5b213fa493ce36a88130b50bef0e9a0cb6cd9f72"
+aws-sdk@^2.132.0:
+  version "2.132.0"
+  resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.132.0.tgz#6ece84bf355a6626fc8ba24023809c08ca1ee372"
   dependencies:
     buffer "4.9.1"
     crypto-browserify "1.0.9"