feat(AppImage): support type 2 image format

Close electron-userland#2136, Close electron-userland#1998, Close electron-userland#832, Close electron-userland#993

packages/builder-util/src/fs.ts

@@ -137,7 +137,7 @@ export function copyFile(src: string, dest: string, isEnsureDir = true) {
  *
  * ensureDir is not called, dest parent dir must exists
  */
-export function copyOrLinkFile(src: string, dest: string, stats?: Stats | null, isUseHardLink = _isUseHardLink): Promise<any> {
+export function copyOrLinkFile(src: string, dest: string, stats?: Stats | null, isUseHardLink = _isUseHardLink, exDevErrorHandler?: (() => boolean) | null): Promise<any> {
   if (stats != null) {
     const originalModeNumber = stats.mode
     const mode = new Mode(stats)
@@ -167,8 +167,23 @@ export function copyOrLinkFile(src: string, dest: string, stats?: Stats | null,
 
   if (isUseHardLink) {
     return link(src, dest)
+      .catch(e => {
+        if (e.code === "EXDEV") {
+          const isLog = exDevErrorHandler == null ? true : exDevErrorHandler()
+          if (isLog && debug.enabled) {
+            debug(`Cannot copy using hard link: ${e.message}`)
+          }
+          return doCopyFile(src, dest, stats)
+        }
+        else {
+          throw e
+        }
+      })
   }
+  return doCopyFile(src, dest, stats)
+}
 
+function doCopyFile(src: string, dest: string, stats: Stats | null | undefined): Promise<any> {
   if (_nodeCopyFile == null) {
     return new BluebirdPromise((resolve, reject) => {
       const reader = createReadStream(src)
@@ -181,16 +196,15 @@ export function copyOrLinkFile(src: string, dest: string, stats?: Stats | null,
       writer.once("close", resolve)
     })
   }
-  else {
-    // node 8.5.0
-    return _nodeCopyFile(src, dest)
-      .then((): any => {
-        if (stats != null) {
-          return chmod(dest, stats.mode)
-        }
-        return null
-      })
+
+  // node 8.5.0+
+  const promise = _nodeCopyFile(src, dest)
+  if (stats == null) {
+    return promise
   }
+
+  return promise
+    .then(() => chmod(dest, stats.mode))
 }
 
 export class FileCopier {
@@ -201,37 +215,29 @@ export class FileCopier {
   }
 
   async copy(src: string, dest: string, stat: Stats | undefined) {
-    try {
-      if (this.transformer != null && stat != null && stat.isFile()) {
-        let data = this.transformer(src)
-        if (data != null) {
-          if (typeof (data as any).then === "function") {
-            data = await data
-          }
+    if (this.transformer != null && stat != null && stat.isFile()) {
+      let data = this.transformer(src)
+      if (data != null) {
+        if (typeof (data as any).then === "function") {
+          data = await data
+        }
 
-          if (data != null) {
-            await writeFile(dest, data)
-            return
-          }
+        if (data != null) {
+          await writeFile(dest, data)
+          return
         }
       }
-      await copyOrLinkFile(src, dest, stat, (!this.isUseHardLink || this.isUseHardLinkFunction == null) ? this.isUseHardLink : this.isUseHardLinkFunction(dest))
     }
-    catch (e) {
+    await copyOrLinkFile(src, dest, stat, (!this.isUseHardLink || this.isUseHardLinkFunction == null) ? this.isUseHardLink : this.isUseHardLinkFunction(dest), this.isUseHardLink ? () => {
       // files are copied concurrently, so, we must not check here currentIsUseHardLink — our code can be executed after that other handler will set currentIsUseHardLink to false
-      if (e.code === "EXDEV") {
-        // ...but here we want to avoid excess debug log message
-        if (this.isUseHardLink) {
-          debug(`Cannot copy using hard link: ${e}`)
-          this.isUseHardLink = false
-        }
-
-        await copyOrLinkFile(src, dest, stat, false)
+      if (this.isUseHardLink) {
+        this.isUseHardLink = false
+        return true
       }
       else {
-        throw e
+        return false
       }
-    }
+    } : null)
   }
 }
 

packages/electron-builder/src/targets/LinuxTargetHelper.ts

@@ -100,7 +100,14 @@ export class LinuxTargetHelper {
     return options.description || this.packager.appInfo.description
   }
 
-  async computeDesktopEntry(targetSpecificOptions: LinuxTargetSpecificOptions, exec?: string, destination?: string | null, extra?: { [key: string]: string; }): Promise<string> {
+  async writeDesktopEntry(targetSpecificOptions: LinuxTargetSpecificOptions, exec?: string, destination?: string | null, extra?: { [key: string]: string; }): Promise<string> {
+    const data = await this.computeDesktopEntry(targetSpecificOptions, exec, extra)
+    const tempFile = destination || await this.packager.getTempFile(`${this.packager.appInfo.productFilename}.desktop`)
+    await outputFile(tempFile, data)
+    return tempFile
+  }
+
+  async computeDesktopEntry(targetSpecificOptions: LinuxTargetSpecificOptions, exec?: string, extra?: { [key: string]: string; }): Promise<string> {
     if (exec != null && exec.length === 0) {
       throw new Error("Specified exec is emptyd")
     }
@@ -115,7 +122,9 @@ export class LinuxTargetHelper {
       Exec: exec == null ? `"${installPrefix}/${productFilename}/${this.packager.executableName}" %U` : exec,
       Terminal: "false",
       Type: "Application",
-      Icon: this.packager.executableName, ...extra, ...targetSpecificOptions.desktop
+      Icon: this.packager.executableName,
+      ...extra,
+      ...targetSpecificOptions.desktop,
     }
 
     let category = targetSpecificOptions.category
@@ -143,10 +152,7 @@ export class LinuxTargetHelper {
       data += `\n${name}=${value}`
     }
     data += "\n"
-
-    const tempFile = destination || await this.packager.getTempFile(`${productFilename}.desktop`)
-    await outputFile(tempFile, data)
-    return tempFile
+    return data
   }
 
   private async createFromIcns(tempDir: string): Promise<Array<Array<string>>> {

packages/electron-builder/src/targets/appImage.ts

@@ -1,10 +1,10 @@
 import BluebirdPromise from "bluebird-lst"
-import { Arch, exec, log } from "builder-util"
+import { Arch, exec, log, debug } from "builder-util"
 import { UUID } from "builder-util-runtime"
-import { getBin, getBinFromGithub } from "builder-util/out/binDownload"
-import { unlinkIfExists } from "builder-util/out/fs"
+import { getBinFromGithub } from "builder-util/out/binDownload"
+import { unlinkIfExists, copyFile } from "builder-util/out/fs"
 import * as ejs from "ejs"
-import { chmod, close, createReadStream, createWriteStream, open, outputFile, readFile, write } from "fs-extra-p"
+import { emptyDir, ensureDir, readFile, remove, writeFile } from "fs-extra-p"
 import { Lazy } from "lazy-val"
 import * as path from "path"
 import { Target } from "../core"
@@ -13,28 +13,22 @@ import { AppImageOptions } from "../options/linuxOptions"
 import { getTemplatePath } from "../util/pathManager"
 import { LinuxTargetHelper } from "./LinuxTargetHelper"
 
-const appImageVersion = process.platform === "darwin" ? "AppImage-17-06-17-mac" : "AppImage-09-07-16-linux"
-//noinspection SpellCheckingInspection
-const appImagePathPromise = process.platform === "darwin" ?
-  getBinFromGithub("AppImage", "17-06-17-mac", "vIaikS8Z2dEnZXKSgtcTn4gimPHCclp+v62KV2Eh9EhxvOvpDFgR3FCgdOsON4EqP8PvnfifNtxgBixCfuQU0A==") :
-  getBin("AppImage", appImageVersion, `https://dl.bintray.com/electron-userland/bin/${appImageVersion}.7z`, "ac324e90b502f4e995f6a169451dbfc911bb55c0077e897d746838e720ae0221")
-
 const appRunTemplate = new Lazy<(data: any) => string>(async () => {
   return ejs.compile(await readFile(path.join(getTemplatePath("linux"), "AppRun.sh"), "utf-8"))
 })
 
 export default class AppImageTarget extends Target {
   readonly options: AppImageOptions = {...this.packager.platformSpecificBuildOptions, ...(this.packager.config as any)[this.name]}
-  private readonly desktopEntry: Promise<string>
+  private readonly desktopEntry: Lazy<string>
 
   constructor(ignored: string, private readonly packager: LinuxPackager, private readonly helper: LinuxTargetHelper, readonly outDir: string) {
     super("appImage")
 
     // we add X-AppImage-BuildId to ensure that new desktop file will be installed
-    this.desktopEntry = helper.computeDesktopEntry(this.options, "AppRun", null, {
+    this.desktopEntry = new Lazy<string>(() => helper.computeDesktopEntry(this.options, "AppRun", {
       "X-AppImage-Version": `${packager.appInfo.buildVersion}`,
       "X-AppImage-BuildId": UUID.v1(),
-    })
+    }))
   }
 
   async build(appOutDir: string, arch: Arch): Promise<any> {
@@ -46,78 +40,82 @@ export default class AppImageTarget extends Target {
     // https://github.com/electron-userland/electron-builder/issues/1726
     const artifactName = this.options.artifactName == null ? packager.computeSafeArtifactName(null, "AppImage", arch, false)!! : packager.expandArtifactNamePattern(this.options, "AppImage", arch)
     const resultFile = path.join(this.outDir, artifactName)
-    await unlinkIfExists(resultFile)
 
-    const finalDesktopFilename = `${this.packager.executableName}.desktop`
+    // pax doesn't like dir with leading dot (e.g. `.__appimage`)
+    const stageDir = path.join(this.outDir, `__appimage-${Arch[arch]}`)
+    const appInStageDir = path.join(stageDir, "app")
+    await emptyDir(stageDir)
+    // pax requires created dir
+    await ensureDir(appInStageDir)
+    // https://unix.stackexchange.com/questions/202430/how-to-copy-a-directory-recursively-using-hardlinks-for-each-file
+    await coyDirUsingHardLinks(appOutDir, appInStageDir)
 
-    const appRunData = (await appRunTemplate.value)({
-      systemIntegration: this.options.systemIntegration || "ask",
-      desktopFileName: finalDesktopFilename,
-      executableName: this.packager.executableName,
-      resourceName: `appimagekit-${this.packager.executableName}`,
-    })
-    const appRunFile = await packager.getTempFile(".sh")
-    await outputFile(appRunFile, appRunData, {
-      mode: "0755",
-    })
-
-    const desktopFile = await this.desktopEntry
-    const appImagePath = await appImagePathPromise
-    const args = [
-      "-joliet", "on",
-      "-volid", "AppImage",
-      "-dev", resultFile,
-      "-padding", "0",
-      "-map", appOutDir, "/usr/bin",
-      "-map", appRunFile, "/AppRun",
-      // we get executable name in the AppRun by desktop file name, so, must be named as executable
-      "-map", desktopFile, `/${finalDesktopFilename}`,
-    ]
-    for (const [from, to] of (await this.helper.icons)) {
-      args.push("-map", from, `/usr/share/icons/default/${to}`)
-    }
+    const finalDesktopFilename = `${this.packager.executableName}.desktop`
+    const iconDir = path.join(stageDir, "usr/share/icons/default")
+    await BluebirdPromise.all([
+      unlinkIfExists(resultFile),
+      writeFile(path.join(stageDir, "/AppRun"), (await appRunTemplate.value)({
+        systemIntegration: this.options.systemIntegration || "ask",
+        desktopFileName: finalDesktopFilename,
+        executableName: this.packager.executableName,
+        resourceName: `appimagekit-${this.packager.executableName}`,
+      }), {
+        mode: "0755",
+      }),
+      writeFile(path.join(stageDir, finalDesktopFilename), await this.desktopEntry.value),
+      ensureDir(iconDir)
+        .then(() => BluebirdPromise.map(this.helper.icons, it => copyFile(it[0], path.join(iconDir, it[1]))))
+    ])
 
     // must be after this.helper.icons call
     if (this.helper.maxIconPath == null) {
       throw new Error("Icon is not provided")
     }
-    args.push("-map", this.helper.maxIconPath, "/.DirIcon")
+    await copyFile(this.helper.maxIconPath, path.join(stageDir, `${this.packager.executableName}${path.extname(this.helper.maxIconPath)}`))
+
+    //noinspection SpellCheckingInspection
+    const vendorDir = await getBinFromGithub("appimage", "9", "5ilKGGAc9eQOJTxnUsYHVppFBxQJCaVVDFYrTGsabtgqxcPsnHc+t2Ilc6PeDiszietMtkdUPjNYKWL2nAS67Q==")
 
     if (arch === Arch.x64 || arch === Arch.ia32) {
-      // noinspection SpellCheckingInspection
-      args.push("-map", path.join(await getBinFromGithub("appimage-packages", "29-09-17", "sMMu1L1tL4QbzvGDxh1pNiIFC+ARnIOVvVdM0d6FBRtSDl0rHXgZMVLiuIAEz6+bJ+daHvYfLlPo1Y8zS6FXaQ=="), arch === Arch.x64 ? "x86_64-linux-gnu" : "i386-linux-gnu"), "/usr/lib")
+      const libDir = path.join(stageDir, "usr/lib")
+      await ensureDir(libDir)
+      await coyDirUsingHardLinks(path.join(vendorDir, "lib", arch === Arch.x64 ? "x86_64-linux-gnu" : "i386-linux-gnu"), libDir)
     }
 
-    args.push("-chown_r", "0", "/", "--")
-    args.push("-zisofs", `level=${process.env.ELECTRON_BUILDER_COMPRESSION_LEVEL || (packager.config.compression === "store" ? "0" : "9")}:block_size=128k:by_magic=off`)
-    args.push("set_filter_r", "--zisofs", "/")
-
-    if (this.packager.packagerOptions.effectiveOptionComputed != null && await this.packager.packagerOptions.effectiveOptionComputed([args, desktopFile])) {
+    if (this.packager.packagerOptions.effectiveOptionComputed != null && await this.packager.packagerOptions.effectiveOptionComputed({desktop: await this.desktopEntry.value})) {
       return
     }
 
-    await exec(process.arch !== "x64" || (process.env.USE_SYSTEM_XORRISO === "true" || process.env.USE_SYSTEM_XORRISO === "") ? "xorriso" : path.join(appImagePath, "xorriso"), args)
-
-    await new BluebirdPromise((resolve, reject) => {
-      const rd = createReadStream(path.join(appImagePath, arch === Arch.ia32 ? "32" : "64", "runtime"))
-      rd.on("error", reject)
-      const wr = createWriteStream(resultFile, {flags: "r+"})
-      wr.on("error", reject)
-      wr.on("close", resolve)
-      rd.pipe(wr)
-    })
-
-    const fd = await open(resultFile, "r+")
-    try {
-      const magicData = Buffer.from([0x41, 0x49, 0x01])
-      await write(fd, magicData, 0, magicData.length, 8)
+    const env = {
+      ...process.env,
     }
-    finally {
-      await close(fd)
+    let toolPath: string
+    if (process.platform === "darwin") {
+      const vendorToolDir = path.join(vendorDir, "darwin")
+      env.PATH = `${vendorToolDir}:${env.PATH}`
+      toolPath = path.join(vendorToolDir, "appimagetool")
+    }
+    else {
+      toolPath = path.join(vendorDir, `appimagetool-i686${arch === Arch.ia32 ? "i686" : "x86_64.AppImage"}`)
     }
 
-    await chmod(resultFile, "0755")
-
+    // default gzip compression - 51.9, xz - 50.4 difference is negligible, start time - well, it seems, a little bit longer (but on Parallels VM on external SSD disk)
+    // so, to be decided later, is it worth to use xz by default
+    const args = ["--runtime-file", path.join(vendorDir, `runtime-${arch === Arch.ia32 ? "i686" : (arch === Arch.x64 ? "x86_64" : "armv7l")}`)]
+    if (debug.enabled) {
+      args.push("--verbose")
+    }
+    args.push(stageDir, resultFile)
+    await exec(toolPath, args, {env})
+    if (!debug.enabled) {
+      await remove(stageDir)
+    }
     packager.dispatchArtifactCreated(resultFile, this, arch, packager.computeSafeArtifactName(artifactName, "AppImage", arch, false))
   }
 }
+
+function coyDirUsingHardLinks(source: string, destination: string) {
+  return exec("pax", ["-rwl", "-p", "amp" /* Do not preserve file access times, Do not preserve file modification times, Preserve the file mode	bits */, ".", destination], {
+    cwd: source,
+  })
+}

packages/electron-builder/src/targets/fpm.ts

@@ -188,7 +188,7 @@ export default class FpmTarget extends Target {
       args.push(mapping.join("=/usr/share/icons/hicolor/"))
     }
 
-    const desktopFilePath = await this.helper.computeDesktopEntry(this.options)
+    const desktopFilePath = await this.helper.writeDesktopEntry(this.options)
     args.push(`${desktopFilePath}=/usr/share/applications/${this.packager.executableName}.desktop`)
 
     if (this.packager.packagerOptions.effectiveOptionComputed != null && await this.packager.packagerOptions.effectiveOptionComputed([args, desktopFilePath])) {

packages/electron-builder/src/targets/snap.ts

@@ -48,7 +48,7 @@ export default class SnapTarget extends Target {
       await copyFile(this.helper.maxIconPath, path.join(snapDir, "gui", "icon.png"))
     }
 
-    const desktopFile = await this.helper.computeDesktopEntry(this.options, packager.executableName, path.join(snapDir, "gui", `${snap.name}.desktop`), {
+    const desktopFile = await this.helper.writeDesktopEntry(this.options, packager.executableName, path.join(snapDir, "gui", `${snap.name}.desktop`), {
       // tslint:disable:no-invalid-template-strings
       Icon: "${SNAP}/meta/gui/icon.png"
     })

packages/electron-builder/templates/linux/AppRun.sh

@@ -33,14 +33,14 @@ if [ -z $APPDIR ] ; then
   APPDIR=$(find-up "AppRun")
 fi
 
-export PATH="${APPDIR}/usr/bin:${APPDIR}/usr/sbin:${PATH}"
+export PATH="${APPDIR}/app:${APPDIR}/usr/sbin:${PATH}"
 export XDG_DATA_DIRS="./share/:/usr/share/gnome:/usr/local/share/:/usr/share/:${XDG_DATA_DIRS}"
 export LD_LIBRARY_PATH="${APPDIR}/usr/lib:${LD_LIBRARY_PATH}"
 export XDG_DATA_DIRS="${APPDIR}"/usr/share/:"${XDG_DATA_DIRS}":/usr/share/gnome/:/usr/local/share/:/usr/share/
 export GSETTINGS_SCHEMA_DIR="${APPDIR}/usr/share/glib-2.0/schemas:${GSETTINGS_SCHEMA_DIR}"
 
 DESKTOP_FILE="$APPDIR/<%= desktopFileName %>"
-BIN="$APPDIR/usr/bin/<%= executableName %>"
+BIN="$APPDIR/app/<%= executableName %>"
 
 trap atexit EXIT