WIP: Delta updates for NSIS electron-userland#1523

develar · Jun 23, 2017 · fcae4c2 · fcae4c2
1 parent 95e4453
commit fcae4c2
Show file tree

Hide file tree

Showing 12 changed files with 97 additions and 54 deletions.
diff --git a/.idea/electron-builder.iml b/.idea/electron-builder.iml
diff --git a/package.json b/package.json
@@ -30,7 +30,7 @@
     "ajv": "^5.2.0",
     "ajv-keywords": "^2.1.0",
     "archiver": "^1.3.0",
-    "aws-sdk": "^2.75.0",
+    "aws-sdk": "^2.76.0",
     "bluebird-lst": "^1.0.2",
     "chalk": "^1.1.3",
     "chromium-pickle-js": "^0.2.0",

diff --git a/packages/electron-builder-util/src/util.ts b/packages/electron-builder-util/src/util.ts
@@ -79,10 +79,12 @@ export function doSpawn(command: string, args: Array<string>, options?: SpawnOpt
 
   const isDebugEnabled = extraOptions == null || extraOptions.isDebugEnabled == null ? debug.enabled : extraOptions.isDebugEnabled
   if (options.stdio == null) {
-    options.stdio = [extraOptions != null && extraOptions.isPipeInput ? "pipe" : "ignore", isDebugEnabled ? "inherit" : "ignore", isDebugEnabled ? "inherit" : "ignore"]
+    // do not ignore stdout/stderr if not debug, because in this case we will read into buffer and print on error
+    options.stdio = [extraOptions != null && extraOptions.isPipeInput ? "pipe" : "ignore", isDebugEnabled ? "inherit" : "pipe", isDebugEnabled ? "inherit" : "pipe"]
   }
 
-  if (isDebugEnabled) {
+  // use general debug.enabled to log spawn, because it doesn't produce a lot of output (the only line), but important in any case
+  if (debug.enabled) {
     const argsString = args.join(" ")
     debug(`Spawning ${command} ${command === "docker" ? argsString : removePassword(argsString)}`)
   }

diff --git a/packages/electron-builder-util/tsconfig.json b/packages/electron-builder-util/tsconfig.json
@@ -13,7 +13,6 @@
     "../../typings/chalk.d.ts",
     "../../typings/debug.d.ts",
     "../../typings/node-emoji.d.ts",
-    "../../typings/pretty-ms.d.ts",
     "../../typings/ansi-escapes.d.ts",
     "../../typings/fcopy-pre-bundled.d.ts"
   ]

diff --git a/packages/electron-builder/src/packager.ts b/packages/electron-builder/src/packager.ts
@@ -5,6 +5,7 @@ import { deepAssign } from "electron-builder-util/out/deepAssign"
 import { all, executeFinally, orNullIfFileNotExist } from "electron-builder-util/out/promise"
 import { EventEmitter } from "events"
 import { ensureDir } from "fs-extra-p"
+import { safeDump } from "js-yaml"
 import * as path from "path"
 import { AppInfo } from "./appInfo"
 import { readAsarJson } from "./asar"
@@ -122,7 +123,7 @@ export class Packager implements BuildInfo {
     const devMetadata = this.devMetadata
     const config = await getConfig(projectDir, configPath, devMetadata, configFromOptions)
     if (debug.enabled) {
-      debug(`Effective config: ${safeStringifyJson(config)}`)
+      debug(`Effective config:\n${safeDump(JSON.parse(safeStringifyJson(config)))}`)
     }
     await validateConfig(config)
     this._config = config

diff --git a/packages/electron-builder/src/targets/archive.ts b/packages/electron-builder/src/targets/archive.ts
@@ -71,9 +71,12 @@ export interface ArchiveOptions {
 
   dictSize?: number
   excluded?: Array<string>
+
+  method?: "Copy" | "LZMA"
 }
 
 // 7z is very fast, so, use ultra compression
+/** @internal */
 export function addUltraArgs(args: Array<string>, options: ArchiveOptions) {
   // https://stackoverflow.com/questions/27136783/7zip-produces-different-output-from-identical-input
   // https://sevenzip.osdn.jp/chm/cmdline/switches/method.htm#7Z
@@ -82,6 +85,17 @@ export function addUltraArgs(args: Array<string>, options: ArchiveOptions) {
   args.push("-mx=9", `-md=${options.dictSize || 64}m`, `-ms=${options.solid === false ? "off" : "on"}`, "-mtm=off", "-mtc=off", "-mta=off")
 }
 
+/** @internal */
+export function addZipArgs(args: Array<string>) {
+  // -mcu switch:  7-Zip uses UTF-8, if there are non-ASCII symbols.
+  // because default mode: 7-Zip uses UTF-8, if the local code page doesn't contain required symbols.
+  // but archive should be the same regardless where produced
+  args.push("-mcu")
+  // disable "Stores NTFS timestamps for files: Modification time, Creation time, Last access time." to produce the same archive for the same data
+  args.push("-mtc=off")
+  args.push("-tzip")
+}
+
 /** @internal */
 export async function archive(compression: CompressionLevel | null | undefined, format: string, outFile: string, dirToArchive: string, options: ArchiveOptions = {}): Promise<string> {
   let storeOnly = compression === "store"
@@ -115,10 +129,17 @@ export async function archive(compression: CompressionLevel | null | undefined,
     // ignore
   }
 
-  if (format === "zip" || storeOnly) {
+  if (options.method != null) {
+    args.push(`-mm=${options.method}`)
+  }
+  else if (format === "zip" || storeOnly) {
     args.push("-mm=" + (storeOnly ? "Copy" : "Deflate"))
   }
 
+  if (format === "zip") {
+    addZipArgs(args)
+  }
+
   args.push(outFile, options.listFile == null ? (options.withoutDir ? "." : path.basename(dirToArchive)) : `@${options.listFile}`)
   if (options.excluded != null) {
     args.push(...options.excluded)

diff --git a/packages/electron-builder/src/targets/blockMap.ts b/packages/electron-builder/src/targets/blockMap.ts
@@ -1,13 +1,14 @@
 import { createHash } from "crypto"
-import { fstat, open, read } from "fs-extra-p"
+import { walk } from "electron-builder-util/out/fs"
+import { open, read, Stats } from "fs-extra-p"
+import { safeDump } from "js-yaml"
+import * as path from "path"
 
-// main exe not changed if unsigned, but if signed - each sign noticeably changes the file
-export async function computeBlocks(inputFile: string): Promise<Array<string>> {
+async function computeBlocks(inputFile: string, stat: Stats): Promise<Array<string>> {
   const fd = await open(inputFile, "r")
 
   const chunkSize = 64 * 1024
   const buffer = Buffer.allocUnsafe(chunkSize)
-  const stat = await fstat(fd)
   const size = stat.size
   const blocks = []
 
@@ -21,4 +22,20 @@ export async function computeBlocks(inputFile: string): Promise<Array<string>> {
   }
 
   return blocks
+}
+
+export async function computeBlockMap(appOutDir: string): Promise<string> {
+  const files = new Map<string, Stats>()
+  await walk(appOutDir, (it: string) => !it.endsWith(`${path.sep}.DS_Store`), (file, fileStat) => {
+    if (fileStat.isFile()) {
+      files.set(file, fileStat)
+    }
+  })
+
+  const info: Array<any> = []
+  for (const [file, stat] of files.entries()) {
+    const blocks = await computeBlocks(file, stat)
+    info.push({name: file.substring(appOutDir.length + 1).replace(/\\/g, "/"), blocks: blocks})
+  }
+  return safeDump(info)
 }
diff --git a/packages/electron-builder/src/targets/nsis.ts b/packages/electron-builder/src/targets/nsis.ts
@@ -13,10 +13,11 @@ import { v5 as uuid5 } from "uuid-1345"
 import { Arch, Target } from "../core"
 import { NsisOptions, PortableOptions } from "../options/winOptions"
 import { normalizeExt } from "../platformPackager"
+import { time } from "../util/timer"
 import { execWine } from "../util/wine"
 import { WinPackager } from "../winPackager"
-import { addUltraArgs, archive, ArchiveOptions } from "./archive"
-import { computeBlocks } from "./blockMap"
+import { addZipArgs, archive, ArchiveOptions } from "./archive"
+import { computeBlockMap } from "./blockMap"
 import { bundledLanguages, getLicenseFiles, lcid, toLangWithRegion } from "./license"
 
 const debug = _debug("electron-builder:nsis")
@@ -119,35 +120,35 @@ export class NsisTarget extends Target {
       await copyFile(path.join(await nsisPathPromise, "elevate.exe"), path.join(appOutDir, "resources", "elevate.exe"), false)
     }
 
-    const format = options.useZip ? "zip" : "7z"
+    const isDifferentialPackage = options.differentialPackage
+    const format = isDifferentialPackage || options.useZip ? "zip" : "7z"
     const archiveFile = path.join(this.outDir, `${packager.appInfo.name}-${packager.appInfo.version}-${Arch[arch]}.nsis.${format}`)
-    const archiveOptions: ArchiveOptions = {withoutDir: true, solid: !options.differentialPackage}
+    const archiveOptions: ArchiveOptions = {withoutDir: true, solid: !isDifferentialPackage}
     let compression = packager.config.compression
-    if (options.differentialPackage) {
-      // 7zip doesn't allow to specify files order even if listFile is used, so, we exclude asar files and main exe and add it later
+
+    const timer = time(`nsis package, ${Arch[arch]}`)
+    if (isDifferentialPackage) {
       // reduce dict size to avoid large block invalidation on change
       archiveOptions.dictSize = 16
-      archiveOptions.excluded = ["-x!*.exe", `-x!resources${path.sep}app.asar`]
+      archiveOptions.method = "LZMA"
       // do not allow to change compression level to avoid different packages
       compression = null
     }
     await archive(compression, format, archiveFile, appOutDir, archiveOptions)
+    timer.end()
 
     if (options.differentialPackage) {
       const args = debug7zArgs("a")
-      addUltraArgs(args, archiveOptions)
+      addZipArgs(args)
+      args.push(`-mm=${archiveOptions.method}`, "-mx=9")
       args.push(archiveFile)
-      await spawn(path7za, args.concat("*.exe"), {
-        cwd: appOutDir,
-      })
 
-      // todo no lack - 7za adds file into the resources dir in the middle of archive
-      await spawn(path7za, args.concat(`resources${path.sep}*.asar`), {
-        cwd: appOutDir,
+      const blockMap = await computeBlockMap(appOutDir)
+      const blockMapFile = path.join(this.outDir, `${packager.appInfo.name}-${packager.appInfo.version}-${Arch[arch]}.blockMap.yml`)
+      await writeFile(blockMapFile, blockMap)
+      await spawn(path7za, args.concat(blockMapFile), {
+        cwd: this.outDir,
       })
-
-      const blockMap = await computeBlocks(archiveFile)
-      await writeFile(path.join(this.outDir, `${packager.appInfo.name}-${packager.appInfo.version}-${Arch[arch]}.nsis.txt`), blockMap.join("\n"))
     }
 
     return archiveFile

diff --git a/packages/electron-builder/src/util/timer.ts b/packages/electron-builder/src/util/timer.ts
@@ -5,12 +5,14 @@ export interface Timer {
 }
 
 class DevTimer implements Timer {
+  private start = process.hrtime()
+
   constructor(private readonly label: string) {
-    console.time(label)
   }
 
   end(): void {
-    console.timeEnd(this.label)
+    const end = process.hrtime(this.start)
+    console.info(`${this.label}: %ds %dms`, end[0], end[1] / 1000000)
   }
 }
 

diff --git a/packages/electron-builder/templates/nsis/readme.md b/packages/electron-builder/templates/nsis/readme.md
@@ -31,4 +31,16 @@ And compression time is also greatly reduced.
 Since NSIS is awesome, no disadvantages in our approach — [compression is disabled](http://nsis.sourceforge.net/Reference/SetCompress) before `File /oname=app.7z "${APP_ARCHIVE}"` and enabled after (it is the reasons why `SOLID` compression is not used).
 So, opposite to Squirrel.Windows, archive is not twice compressed.
 
-So, in your custom NSIS scripts you should not use any compression instructions. Only `SetCompress` if you need to disable compression for already archived file.
+So, in your custom NSIS scripts you should not use any compression instructions. Only `SetCompress` if you need to disable compression for already archived file.
+
+
+## Package File
+
+https://sourceforge.net/p/sevenzip/discussion/45798/thread/222c71f9/
+
+(size as in windows explorer)
+7z - 34.134 Compression time 26s
+zip(lzma) - 37.612 Compression time 26s (~ the same time (as expected, because filters, as documented, are very fast ())
+zip(xz) - 37.619  Not clear why. xz supports filters, but it seems 7z doesn't apply it correctly.
+
+
diff --git a/typings/pretty-ms.d.ts b/typings/pretty-ms.d.ts
diff --git a/yarn.lock b/yarn.lock
@@ -27,16 +27,16 @@
   resolved "https://registry.yarnpkg.com/@types/ini/-/ini-1.3.29.tgz#1325e981e047d40d13ce0359b821475b97741d2f"
 
 "@types/jest@^20.0.1":
-  version "20.0.1"
-  resolved "https://registry.yarnpkg.com/@types/jest/-/jest-20.0.1.tgz#8643a195d925a00f7bdee5257d12c3aac743f3b4"
+  version "20.0.2"
+  resolved "https://registry.yarnpkg.com/@types/jest/-/jest-20.0.2.tgz#86c751121fb53dbd39bb1a08c45083da13f2dc67"
 
 "@types/js-yaml@^3.5.31":
   version "3.5.31"
   resolved "https://registry.yarnpkg.com/@types/js-yaml/-/js-yaml-3.5.31.tgz#54aeb8bcaaf94a7b1a64311bc318dbfe601a593a"
 
 "@types/node@*":
-  version "8.0.1"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-8.0.1.tgz#89c271e0c3b9ebb6a3756dd601336970b6228b77"
+  version "8.0.2"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-8.0.2.tgz#8ab9456efb87d57f11d04f313d3da1041948fb4d"
 
 "@types/source-map-support@^0.4.0":
   version "0.4.0"
@@ -263,9 +263,9 @@ asynckit@^0.4.0:
   version "0.4.0"
   resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79"
 
-aws-sdk@^2.75.0:
-  version "2.75.0"
-  resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.75.0.tgz#f78802e46f95b7044a094da259057dd8d5d8da17"
+aws-sdk@^2.76.0:
+  version "2.76.0"
+  resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.76.0.tgz#2c37bf04e37ab4a26b5ff7c7583d596034c76309"
   dependencies:
     buffer "5.0.6"
     crypto-browserify "1.0.9"
@@ -800,8 +800,8 @@ command-line-usage@^4.0.0:
     typical "^2.6.0"
 
 commander@^2.9.0:
-  version "2.9.0"
-  resolved "https://registry.yarnpkg.com/commander/-/commander-2.9.0.tgz#9c99094176e12240cb22d6c5146098400fe0f7d4"
+  version "2.10.0"
+  resolved "https://registry.yarnpkg.com/commander/-/commander-2.10.0.tgz#e1f5d3245de246d1a5ca04702fa1ad1bd7e405fe"
   dependencies:
     graceful-readlink ">= 1.0.0"
 
@@ -2925,10 +2925,6 @@ safe-buffer@^5.0.1, safe-buffer@~5.1.0:
   version "5.1.1"
   resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853"
 
-safe-buffer@~5.0.1:
-  version "5.0.1"
-  resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.0.1.tgz#d263ca54696cd8a306b5ca6551e92de57918fbe7"
-
 sane@~1.6.0:
   version "1.6.0"
   resolved "https://registry.yarnpkg.com/sane/-/sane-1.6.0.tgz#9610c452307a135d29c1fdfe2547034180c46775"
@@ -3115,10 +3111,10 @@ string_decoder@~0.10.x:
   resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94"
 
 string_decoder@~1.0.0:
-  version "1.0.2"
-  resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.0.2.tgz#b29e1f4e1125fa97a10382b8a533737b7491e179"
+  version "1.0.3"
+  resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.0.3.tgz#0fc67d7c141825de94282dd536bec6b9bce860ab"
   dependencies:
-    safe-buffer "~5.0.1"
+    safe-buffer "~5.1.0"
 
 stringstream@~0.0.4:
   version "0.0.5"