fix: Building Linux AppImage in >19.33 sometimes fails with tar error…

… "file changed as we read it

Close electron-userland#2187

README.md

@@ -88,15 +88,14 @@ See [Command Line Interface](https://www.electron.build/cli).
 ## Programmatic Usage
 See `node_modules/electron-builder/out/index.d.ts`. Typings for TypeScript is provided.
 
+To build for current platform and current arch:
 ```js
 "use strict"
 
 const builder = require("electron-builder")
-const Platform = builder.Platform
 
 // Promise is returned
 builder.build({
-  targets: Platform.MAC.createTarget(),
   config: {
    "//": "build options, see https://goo.gl/ZhRfla"
   }
@@ -109,6 +108,10 @@ builder.build({
   })
 ```
 
+Add `win: []` to build for Windows default target. Add `win: ["nsis-web]` to build specified target (web installer) for Windows. The same for `mac: []` and `linux: []`
+
+Add `ia32: true` to build `ia32` (or `x64: true`, or `armv7l: true`). Several can be specified and built at once.
+
 ## Pack Only in a Distributable Format
 
 You can use electron-builder only to pack your electron app in a AppImage, Snaps, Debian package, NSIS, macOS installer component package (`pkg`) 

package.json

@@ -28,10 +28,10 @@
   "///": "All dependencies for all packages (hoisted)",
   "////": "All typings are added into root `package.json` to avoid duplication errors in the IDE compiler (several `node.d.ts` files).",
   "dependencies": {
-    "7zip-bin": "^2.2.4",
+    "7zip-bin": "^2.2.5",
     "archiver": "^2.0.3",
     "async-exit-hook": "^2.0.1",
-    "aws-sdk": "^2.131.0",
+    "aws-sdk": "^2.132.0",
     "bluebird-lst": "^1.0.4",
     "chalk": "^2.1.0",
     "chromium-pickle-js": "^0.2.0",

packages/builder-util/package.json

@@ -21,7 +21,7 @@
     "node-emoji": "^1.8.1",
     "builder-util-runtime": "^0.0.0-semantic-release",
     "source-map-support": "^0.5.0",
-    "7zip-bin": "^2.2.4",
+    "7zip-bin": "^2.2.5",
     "ini": "^1.3.4",
     "tunnel-agent": "^0.6.0",
     "semver": "^5.4.1",

packages/electron-builder/package.json

@@ -51,7 +51,7 @@
   "bugs": "https://github.com/electron-userland/electron-builder/issues",
   "homepage": "https://github.com/electron-userland/electron-builder",
   "dependencies": {
-    "7zip-bin": "^2.2.4",
+    "7zip-bin": "^2.2.5",
     "async-exit-hook": "^2.0.1",
     "bluebird-lst": "^1.0.4",
     "chalk": "^2.1.0",

packages/electron-builder/src/targets/ArchiveTarget.ts

@@ -21,7 +21,7 @@ export class ArchiveTarget extends Target {
     // tslint:disable:no-invalid-template-strings
     const outFile = path.join(this.outDir, packager.expandArtifactNamePattern(this.options, format, arch, packager.platform === Platform.LINUX ? "${name}-${version}-${arch}.${ext}" : "${productName}-${version}-${arch}-${os}.${ext}"))
     if (format.startsWith("tar.")) {
-      await tar(packager.compression, format, outFile, appOutDir, isMac)
+      await tar(packager.compression, format, outFile, appOutDir, isMac, packager.info.tempDirManager)
     }
     else {
       await archive(packager.compression, format, outFile, appOutDir, {withoutDir: !isMac})

packages/electron-builder/src/targets/appImage.ts

@@ -17,6 +17,7 @@ const appRunTemplate = new Lazy<(data: any) => string>(async () => {
   return ejs.compile(await readFile(path.join(getTemplatePath("linux"), "AppRun.sh"), "utf-8"))
 })
 
+// https://unix.stackexchange.com/questions/375191/append-to-sub-directory-inside-squashfs-file
 export default class AppImageTarget extends Target {
   readonly options: AppImageOptions = {...this.packager.platformSpecificBuildOptions, ...(this.packager.config as any)[this.name]}
   private readonly desktopEntry: Lazy<string>

packages/electron-builder/src/targets/archive.ts

@@ -1,57 +1,70 @@
-import { path7x, path7za } from "7zip-bin"
-import { debug7z, debug7zArgs, isMacOsSierra, spawn } from "builder-util"
-import { computeEnv, getLinuxToolsPath } from "builder-util/out/bundledTool"
+import { path7za } from "7zip-bin"
+import { debug7z, debug7zArgs, exec } from "builder-util"
 import { exists } from "builder-util/out/fs"
 import { unlink } from "fs-extra-p"
 import * as path from "path"
 import { CompressionLevel } from "../core"
-
-class CompressionDescriptor {
-  constructor(readonly flag: string, readonly env: string, readonly minLevel: string, readonly maxLevel: string = "-9") {
-  }
-}
-
-const extToCompressionDescriptor: { [key: string]: CompressionDescriptor; } = {
-  "tar.xz": new CompressionDescriptor(`-I'${path7x}'`, "XZ_OPT", "-0", "-9e"),
-  "tar.lz": new CompressionDescriptor("--lzip", "LZOP", "-0"),
-  "tar.gz": new CompressionDescriptor("--gz", "GZIP", "-1"),
-  "tar.bz2": new CompressionDescriptor("--bzip2", "BZIP2", "-1"),
-}
+import { TmpDir } from "temp-file"
 
 /** @internal */
-export async function tar(compression: CompressionLevel | null | undefined, format: string, outFile: string, dirToArchive: string, isMacApp: boolean = false) {
-  // we don't use 7z here - develar: I spent a lot of time making pipe working - but it works on MacOS and often hangs on Linux (even if use pipe-io lib)
-  // and in any case it is better to use system tools (in the light of docker - it is not problem for user because we provide complete docker image).
-  const info = extToCompressionDescriptor[format]
-  let tarEnv = process.env
-  if (process.env.ELECTRON_BUILDER_COMPRESSION_LEVEL != null) {
-    tarEnv = {...tarEnv}
-    tarEnv[info.env] = "-" + process.env.ELECTRON_BUILDER_COMPRESSION_LEVEL
-  }
-  else if (compression != null && compression !== "normal") {
-    tarEnv = {...tarEnv}
-    tarEnv[info.env] = compression === "store" ? info.minLevel : info.maxLevel
-  }
+export async function tar(compression: CompressionLevel | any | any, format: string, outFile: string, dirToArchive: string, isMacApp: boolean, tempDirManager: TmpDir) {
+  const tarFile = await tempDirManager.getTempFile({suffix: ".tar"})
+  const tarArgs = debug7zArgs("a")
+  tarArgs.push(tarFile)
+  tarArgs.push(path.basename(dirToArchive))
+  await exec(path7za, tarArgs, {
+    cwd: path.dirname(dirToArchive),
+  })
 
-  const args = [info.flag, "-cf", outFile]
   if (!isMacApp) {
-    args.push("--transform", `s,^\\.,${path.basename(outFile, "." + format)},`)
+    await exec(path7za, ["rn", tarFile, path.basename(dirToArchive), path.basename(outFile, `.${format}`)])
   }
-  args.push(isMacApp ? path.basename(dirToArchive) : ".")
 
-  if (await isMacOsSierra()) {
-    const linuxToolsPath = await getLinuxToolsPath()
-    tarEnv = {
-      ...tarEnv,
-      PATH: computeEnv(process.env.PATH, [path.join(linuxToolsPath, "bin")]),
-      SZA_PATH: path7za,
-    }
+  const args = compute7zCompressArgs(compression, format === "tar.xz" ? "xz" : (format === "tar.bz2" ? "bzip2" : "gzip"), {isRegularFile: true})
+  // remove file before - 7z doesn't overwrite file, but update
+  try {
+    await unlink(outFile)
+  }
+  catch (e) {
+    // ignore
   }
 
-  await spawn(process.platform === "darwin" || process.platform === "freebsd" ? "gtar" : "tar", args, {
-    cwd: isMacApp ? path.dirname(dirToArchive) : dirToArchive,
-    env: tarEnv,
-  })
+  args.push(outFile, tarFile)
+  await exec(path7za, args, {
+    cwd: path.dirname(dirToArchive),
+  }, debug7z.enabled)
+
+  // const info: CompressionDescriptor | null = extToCompressionDescriptor[format]
+  // const tarEnv: any = {
+  //   ...process.env,
+  //   SZA_PATH: path7za,
+  //   SZA_ARCHIVE_TYPE: format === "tar.xz" ? "xz" : (format === "tar.bz2" ? "bzip2" : "gzip"),
+  //   SZA_COMPRESSION_LEVEL: process.env.ELECTRON_BUILDER_COMPRESSION_LEVEL || (compression === "store" ? "0" : "9")
+  // }
+  //
+  // if (info != null) {
+  //   if (process.env.ELECTRON_BUILDER_COMPRESSION_LEVEL != null) {
+  //     tarEnv[info.env] = "-" + process.env.ELECTRON_BUILDER_COMPRESSION_LEVEL
+  //   }
+  //   else if (compression != null && compression !== "normal") {
+  //     tarEnv[info.env] = compression === "store" ? info.minLevel : info.maxLevel
+  //   }
+  // }
+  //
+  // const args = [info == null ? `-I'${path7x}'` : info.flag, "-cf", outFile]
+  // if (!isMacApp) {
+  //   args.push("--transform", `s,^\\.,${path.basename(outFile, `.${format}`)},`)
+  // }
+  // args.push(isMacApp ? path.basename(dirToArchive) : ".")
+  //
+  // if (await isMacOsSierra()) {
+  //   tarEnv.PATH = computeEnv(process.env.PATH, [path.join(await getLinuxToolsPath(), "bin")])
+  // }
+  //
+  // await exec(process.platform === "darwin" || process.platform === "freebsd" ? "gtar" : "tar", args, {
+  //   cwd: isMacApp ? path.dirname(dirToArchive) : dirToArchive,
+  //   env: tarEnv,
+  // })
   return outFile
 }
 
@@ -77,6 +90,8 @@ export interface ArchiveOptions {
   excluded?: Array<string>
 
   method?: "Copy" | "LZMA" | "Deflate"
+
+  isRegularFile?: boolean
 }
 
 export function compute7zCompressArgs(compression: CompressionLevel | any | any, format: string, options: ArchiveOptions = {}) {
@@ -107,7 +122,9 @@ export function compute7zCompressArgs(compression: CompressionLevel | any | any,
   // https://stackoverflow.com/questions/27136783/7zip-produces-different-output-from-identical-input
   // tc and ta are off by default, but to be sure, we explicitly set it to off
   // disable "Stores NTFS timestamps for files: Modification time, Creation time, Last access time." to produce the same archive for the same data
-  args.push("-mtc=off")
+  if (!options.isRegularFile) {
+    args.push("-mtc=off")
+  }
 
   if (format === "7z" || format.endsWith(".7z")) {
     if (options.solid === false) {
@@ -157,9 +174,9 @@ export async function archive(compression: CompressionLevel | null | undefined,
   }
 
   try {
-    await spawn(path7za, args, {
+    await exec(path7za, args, {
       cwd: options.withoutDir ? dirToArchive : path.dirname(dirToArchive),
-    }, {isDebugEnabled: debug7z.enabled})
+    }, debug7z.enabled)
   }
   catch (e) {
     if (e.code === "ENOENT" && !(await exists(dirToArchive))) {

packages/electron-publisher-s3/package.json

@@ -12,7 +12,7 @@
   ],
   "dependencies": {
     "fs-extra-p": "^4.4.3",
-    "aws-sdk": "^2.131.0",
+    "aws-sdk": "^2.132.0",
     "mime": "^2.0.3",
     "electron-publish": "~0.0.0-semantic-release",
     "builder-util": "^0.0.0-semantic-release",

test/src/linux/fpmTest.ts

@@ -6,4 +6,4 @@ test.ifAll.ifDevOrLinuxCi("targets", app({targets: Platform.LINUX.createTarget([
 
 // https://github.com/electron-userland/electron-builder/issues/460
 // for some reasons in parallel to fmp we cannot use tar
-test.ifAll.ifDevOrLinuxCi("rpm and tar.gz", app({targets: Platform.LINUX.createTarget(["rpm", "tar.gz"])}))
+test.ifAll.ifDevOrLinuxCi("rpm and tar.gz", app({targets: Platform.LINUX.createTarget(["appimage ", "tar.gz"])}))

yarn.lock

@@ -14,9 +14,9 @@
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/7zip-bin-win/-/7zip-bin-win-2.1.1.tgz#8acfc28bb34e53a9476b46ae85a97418e6035c20"
 
-"7zip-bin@^2.2.4":
-  version "2.2.4"
-  resolved "https://registry.yarnpkg.com/7zip-bin/-/7zip-bin-2.2.4.tgz#5d0a7da759258b7fa59121fddcec7cb65938a85c"
+"7zip-bin@^2.2.5":
+  version "2.2.5"
+  resolved "https://registry.yarnpkg.com/7zip-bin/-/7zip-bin-2.2.5.tgz#d37f90f3108708e03d4b66ee5e3fbc5624cece82"
   optionalDependencies:
     "7zip-bin-linux" "^1.1.0"
     "7zip-bin-mac" "^1.0.1"
@@ -453,9 +453,9 @@ asynckit@^0.4.0:
   version "0.4.0"
   resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79"
 
-aws-sdk@^2.131.0:
-  version "2.131.0"
-  resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.131.0.tgz#5b213fa493ce36a88130b50bef0e9a0cb6cd9f72"
+aws-sdk@^2.132.0:
+  version "2.132.0"
+  resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.132.0.tgz#6ece84bf355a6626fc8ba24023809c08ca1ee372"
   dependencies:
     buffer "4.9.1"
     crypto-browserify "1.0.9"