From fcae4c2d035f8329e51f22435d5dd79fe0f4e5de Mon Sep 17 00:00:00 2001 From: develar Date: Fri, 23 Jun 2017 18:34:05 +0200 Subject: [PATCH] WIP: Delta updates for NSIS #1523 --- .idea/electron-builder.iml | 2 -- package.json | 2 +- packages/electron-builder-util/src/util.ts | 6 ++-- packages/electron-builder-util/tsconfig.json | 1 - packages/electron-builder/src/packager.ts | 3 +- .../electron-builder/src/targets/archive.ts | 23 +++++++++++- .../electron-builder/src/targets/blockMap.ts | 25 ++++++++++--- packages/electron-builder/src/targets/nsis.ts | 35 ++++++++++--------- packages/electron-builder/src/util/timer.ts | 6 ++-- .../electron-builder/templates/nsis/readme.md | 14 +++++++- typings/pretty-ms.d.ts | 6 ---- yarn.lock | 28 +++++++-------- 12 files changed, 97 insertions(+), 54 deletions(-) delete mode 100644 typings/pretty-ms.d.ts diff --git a/.idea/electron-builder.iml b/.idea/electron-builder.iml index 7b6d306495d..a44e18e3a8a 100644 --- a/.idea/electron-builder.iml +++ b/.idea/electron-builder.iml @@ -3,7 +3,6 @@ - @@ -28,7 +27,6 @@ - diff --git a/package.json b/package.json index fac0de36e2d..af6e6235c3d 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "ajv": "^5.2.0", "ajv-keywords": "^2.1.0", "archiver": "^1.3.0", - "aws-sdk": "^2.75.0", + "aws-sdk": "^2.76.0", "bluebird-lst": "^1.0.2", "chalk": "^1.1.3", "chromium-pickle-js": "^0.2.0", diff --git a/packages/electron-builder-util/src/util.ts b/packages/electron-builder-util/src/util.ts index f7f1df77248..9c7b67f7369 100644 --- a/packages/electron-builder-util/src/util.ts +++ b/packages/electron-builder-util/src/util.ts @@ -79,10 +79,12 @@ export function doSpawn(command: string, args: Array, options?: SpawnOpt const isDebugEnabled = extraOptions == null || extraOptions.isDebugEnabled == null ? debug.enabled : extraOptions.isDebugEnabled if (options.stdio == null) { - options.stdio = [extraOptions != null && extraOptions.isPipeInput ? "pipe" : "ignore", isDebugEnabled ? "inherit" : "ignore", isDebugEnabled ? "inherit" : "ignore"] + // do not ignore stdout/stderr if not debug, because in this case we will read into buffer and print on error + options.stdio = [extraOptions != null && extraOptions.isPipeInput ? "pipe" : "ignore", isDebugEnabled ? "inherit" : "pipe", isDebugEnabled ? "inherit" : "pipe"] } - if (isDebugEnabled) { + // use general debug.enabled to log spawn, because it doesn't produce a lot of output (the only line), but important in any case + if (debug.enabled) { const argsString = args.join(" ") debug(`Spawning ${command} ${command === "docker" ? argsString : removePassword(argsString)}`) } diff --git a/packages/electron-builder-util/tsconfig.json b/packages/electron-builder-util/tsconfig.json index bd81bc98d22..827323c60a3 100644 --- a/packages/electron-builder-util/tsconfig.json +++ b/packages/electron-builder-util/tsconfig.json @@ -13,7 +13,6 @@ "../../typings/chalk.d.ts", "../../typings/debug.d.ts", "../../typings/node-emoji.d.ts", - "../../typings/pretty-ms.d.ts", "../../typings/ansi-escapes.d.ts", "../../typings/fcopy-pre-bundled.d.ts" ] diff --git a/packages/electron-builder/src/packager.ts b/packages/electron-builder/src/packager.ts index fa949fb0b94..b1173fe35ea 100644 --- a/packages/electron-builder/src/packager.ts +++ b/packages/electron-builder/src/packager.ts @@ -5,6 +5,7 @@ import { deepAssign } from "electron-builder-util/out/deepAssign" import { all, executeFinally, orNullIfFileNotExist } from "electron-builder-util/out/promise" import { EventEmitter } from "events" import { ensureDir } from "fs-extra-p" +import { safeDump } from "js-yaml" import * as path from "path" import { AppInfo } from "./appInfo" import { readAsarJson } from "./asar" @@ -122,7 +123,7 @@ export class Packager implements BuildInfo { const devMetadata = this.devMetadata const config = await getConfig(projectDir, configPath, devMetadata, configFromOptions) if (debug.enabled) { - debug(`Effective config: ${safeStringifyJson(config)}`) + debug(`Effective config:\n${safeDump(JSON.parse(safeStringifyJson(config)))}`) } await validateConfig(config) this._config = config diff --git a/packages/electron-builder/src/targets/archive.ts b/packages/electron-builder/src/targets/archive.ts index 169775d3c18..456169536b6 100644 --- a/packages/electron-builder/src/targets/archive.ts +++ b/packages/electron-builder/src/targets/archive.ts @@ -71,9 +71,12 @@ export interface ArchiveOptions { dictSize?: number excluded?: Array + + method?: "Copy" | "LZMA" } // 7z is very fast, so, use ultra compression +/** @internal */ export function addUltraArgs(args: Array, options: ArchiveOptions) { // https://stackoverflow.com/questions/27136783/7zip-produces-different-output-from-identical-input // https://sevenzip.osdn.jp/chm/cmdline/switches/method.htm#7Z @@ -82,6 +85,17 @@ export function addUltraArgs(args: Array, options: ArchiveOptions) { args.push("-mx=9", `-md=${options.dictSize || 64}m`, `-ms=${options.solid === false ? "off" : "on"}`, "-mtm=off", "-mtc=off", "-mta=off") } +/** @internal */ +export function addZipArgs(args: Array) { + // -mcu switch: 7-Zip uses UTF-8, if there are non-ASCII symbols. + // because default mode: 7-Zip uses UTF-8, if the local code page doesn't contain required symbols. + // but archive should be the same regardless where produced + args.push("-mcu") + // disable "Stores NTFS timestamps for files: Modification time, Creation time, Last access time." to produce the same archive for the same data + args.push("-mtc=off") + args.push("-tzip") +} + /** @internal */ export async function archive(compression: CompressionLevel | null | undefined, format: string, outFile: string, dirToArchive: string, options: ArchiveOptions = {}): Promise { let storeOnly = compression === "store" @@ -115,10 +129,17 @@ export async function archive(compression: CompressionLevel | null | undefined, // ignore } - if (format === "zip" || storeOnly) { + if (options.method != null) { + args.push(`-mm=${options.method}`) + } + else if (format === "zip" || storeOnly) { args.push("-mm=" + (storeOnly ? "Copy" : "Deflate")) } + if (format === "zip") { + addZipArgs(args) + } + args.push(outFile, options.listFile == null ? (options.withoutDir ? "." : path.basename(dirToArchive)) : `@${options.listFile}`) if (options.excluded != null) { args.push(...options.excluded) diff --git a/packages/electron-builder/src/targets/blockMap.ts b/packages/electron-builder/src/targets/blockMap.ts index 06d87c8a291..28442163250 100644 --- a/packages/electron-builder/src/targets/blockMap.ts +++ b/packages/electron-builder/src/targets/blockMap.ts @@ -1,13 +1,14 @@ import { createHash } from "crypto" -import { fstat, open, read } from "fs-extra-p" +import { walk } from "electron-builder-util/out/fs" +import { open, read, Stats } from "fs-extra-p" +import { safeDump } from "js-yaml" +import * as path from "path" -// main exe not changed if unsigned, but if signed - each sign noticeably changes the file -export async function computeBlocks(inputFile: string): Promise> { +async function computeBlocks(inputFile: string, stat: Stats): Promise> { const fd = await open(inputFile, "r") const chunkSize = 64 * 1024 const buffer = Buffer.allocUnsafe(chunkSize) - const stat = await fstat(fd) const size = stat.size const blocks = [] @@ -21,4 +22,20 @@ export async function computeBlocks(inputFile: string): Promise> { } return blocks +} + +export async function computeBlockMap(appOutDir: string): Promise { + const files = new Map() + await walk(appOutDir, (it: string) => !it.endsWith(`${path.sep}.DS_Store`), (file, fileStat) => { + if (fileStat.isFile()) { + files.set(file, fileStat) + } + }) + + const info: Array = [] + for (const [file, stat] of files.entries()) { + const blocks = await computeBlocks(file, stat) + info.push({name: file.substring(appOutDir.length + 1).replace(/\\/g, "/"), blocks: blocks}) + } + return safeDump(info) } \ No newline at end of file diff --git a/packages/electron-builder/src/targets/nsis.ts b/packages/electron-builder/src/targets/nsis.ts index 7332e173166..20b5598864c 100644 --- a/packages/electron-builder/src/targets/nsis.ts +++ b/packages/electron-builder/src/targets/nsis.ts @@ -13,10 +13,11 @@ import { v5 as uuid5 } from "uuid-1345" import { Arch, Target } from "../core" import { NsisOptions, PortableOptions } from "../options/winOptions" import { normalizeExt } from "../platformPackager" +import { time } from "../util/timer" import { execWine } from "../util/wine" import { WinPackager } from "../winPackager" -import { addUltraArgs, archive, ArchiveOptions } from "./archive" -import { computeBlocks } from "./blockMap" +import { addZipArgs, archive, ArchiveOptions } from "./archive" +import { computeBlockMap } from "./blockMap" import { bundledLanguages, getLicenseFiles, lcid, toLangWithRegion } from "./license" const debug = _debug("electron-builder:nsis") @@ -119,35 +120,35 @@ export class NsisTarget extends Target { await copyFile(path.join(await nsisPathPromise, "elevate.exe"), path.join(appOutDir, "resources", "elevate.exe"), false) } - const format = options.useZip ? "zip" : "7z" + const isDifferentialPackage = options.differentialPackage + const format = isDifferentialPackage || options.useZip ? "zip" : "7z" const archiveFile = path.join(this.outDir, `${packager.appInfo.name}-${packager.appInfo.version}-${Arch[arch]}.nsis.${format}`) - const archiveOptions: ArchiveOptions = {withoutDir: true, solid: !options.differentialPackage} + const archiveOptions: ArchiveOptions = {withoutDir: true, solid: !isDifferentialPackage} let compression = packager.config.compression - if (options.differentialPackage) { - // 7zip doesn't allow to specify files order even if listFile is used, so, we exclude asar files and main exe and add it later + + const timer = time(`nsis package, ${Arch[arch]}`) + if (isDifferentialPackage) { // reduce dict size to avoid large block invalidation on change archiveOptions.dictSize = 16 - archiveOptions.excluded = ["-x!*.exe", `-x!resources${path.sep}app.asar`] + archiveOptions.method = "LZMA" // do not allow to change compression level to avoid different packages compression = null } await archive(compression, format, archiveFile, appOutDir, archiveOptions) + timer.end() if (options.differentialPackage) { const args = debug7zArgs("a") - addUltraArgs(args, archiveOptions) + addZipArgs(args) + args.push(`-mm=${archiveOptions.method}`, "-mx=9") args.push(archiveFile) - await spawn(path7za, args.concat("*.exe"), { - cwd: appOutDir, - }) - // todo no lack - 7za adds file into the resources dir in the middle of archive - await spawn(path7za, args.concat(`resources${path.sep}*.asar`), { - cwd: appOutDir, + const blockMap = await computeBlockMap(appOutDir) + const blockMapFile = path.join(this.outDir, `${packager.appInfo.name}-${packager.appInfo.version}-${Arch[arch]}.blockMap.yml`) + await writeFile(blockMapFile, blockMap) + await spawn(path7za, args.concat(blockMapFile), { + cwd: this.outDir, }) - - const blockMap = await computeBlocks(archiveFile) - await writeFile(path.join(this.outDir, `${packager.appInfo.name}-${packager.appInfo.version}-${Arch[arch]}.nsis.txt`), blockMap.join("\n")) } return archiveFile diff --git a/packages/electron-builder/src/util/timer.ts b/packages/electron-builder/src/util/timer.ts index abe9ec06625..4ac3964ab1d 100644 --- a/packages/electron-builder/src/util/timer.ts +++ b/packages/electron-builder/src/util/timer.ts @@ -5,12 +5,14 @@ export interface Timer { } class DevTimer implements Timer { + private start = process.hrtime() + constructor(private readonly label: string) { - console.time(label) } end(): void { - console.timeEnd(this.label) + const end = process.hrtime(this.start) + console.info(`${this.label}: %ds %dms`, end[0], end[1] / 1000000) } } diff --git a/packages/electron-builder/templates/nsis/readme.md b/packages/electron-builder/templates/nsis/readme.md index 709ae914e62..b8ed1f1d44d 100644 --- a/packages/electron-builder/templates/nsis/readme.md +++ b/packages/electron-builder/templates/nsis/readme.md @@ -31,4 +31,16 @@ And compression time is also greatly reduced. Since NSIS is awesome, no disadvantages in our approach — [compression is disabled](http://nsis.sourceforge.net/Reference/SetCompress) before `File /oname=app.7z "${APP_ARCHIVE}"` and enabled after (it is the reasons why `SOLID` compression is not used). So, opposite to Squirrel.Windows, archive is not twice compressed. -So, in your custom NSIS scripts you should not use any compression instructions. Only `SetCompress` if you need to disable compression for already archived file. \ No newline at end of file +So, in your custom NSIS scripts you should not use any compression instructions. Only `SetCompress` if you need to disable compression for already archived file. + + +## Package File + +https://sourceforge.net/p/sevenzip/discussion/45798/thread/222c71f9/ + +(size as in windows explorer) +7z - 34.134 Compression time 26s +zip(lzma) - 37.612 Compression time 26s (~ the same time (as expected, because filters, as documented, are very fast ()) +zip(xz) - 37.619 Not clear why. xz supports filters, but it seems 7z doesn't apply it correctly. + + diff --git a/typings/pretty-ms.d.ts b/typings/pretty-ms.d.ts deleted file mode 100644 index c72b5934e94..00000000000 --- a/typings/pretty-ms.d.ts +++ /dev/null @@ -1,6 +0,0 @@ -// ms reports too rounded values compared to pretty-ms -declare module "pretty-ms" { - function prettyMs(ms: number): string - - export default prettyMs -} \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index b6c343763ad..df852eb254d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -27,16 +27,16 @@ resolved "https://registry.yarnpkg.com/@types/ini/-/ini-1.3.29.tgz#1325e981e047d40d13ce0359b821475b97741d2f" "@types/jest@^20.0.1": - version "20.0.1" - resolved "https://registry.yarnpkg.com/@types/jest/-/jest-20.0.1.tgz#8643a195d925a00f7bdee5257d12c3aac743f3b4" + version "20.0.2" + resolved "https://registry.yarnpkg.com/@types/jest/-/jest-20.0.2.tgz#86c751121fb53dbd39bb1a08c45083da13f2dc67" "@types/js-yaml@^3.5.31": version "3.5.31" resolved "https://registry.yarnpkg.com/@types/js-yaml/-/js-yaml-3.5.31.tgz#54aeb8bcaaf94a7b1a64311bc318dbfe601a593a" "@types/node@*": - version "8.0.1" - resolved "https://registry.yarnpkg.com/@types/node/-/node-8.0.1.tgz#89c271e0c3b9ebb6a3756dd601336970b6228b77" + version "8.0.2" + resolved "https://registry.yarnpkg.com/@types/node/-/node-8.0.2.tgz#8ab9456efb87d57f11d04f313d3da1041948fb4d" "@types/source-map-support@^0.4.0": version "0.4.0" @@ -263,9 +263,9 @@ asynckit@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" -aws-sdk@^2.75.0: - version "2.75.0" - resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.75.0.tgz#f78802e46f95b7044a094da259057dd8d5d8da17" +aws-sdk@^2.76.0: + version "2.76.0" + resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.76.0.tgz#2c37bf04e37ab4a26b5ff7c7583d596034c76309" dependencies: buffer "5.0.6" crypto-browserify "1.0.9" @@ -800,8 +800,8 @@ command-line-usage@^4.0.0: typical "^2.6.0" commander@^2.9.0: - version "2.9.0" - resolved "https://registry.yarnpkg.com/commander/-/commander-2.9.0.tgz#9c99094176e12240cb22d6c5146098400fe0f7d4" + version "2.10.0" + resolved "https://registry.yarnpkg.com/commander/-/commander-2.10.0.tgz#e1f5d3245de246d1a5ca04702fa1ad1bd7e405fe" dependencies: graceful-readlink ">= 1.0.0" @@ -2925,10 +2925,6 @@ safe-buffer@^5.0.1, safe-buffer@~5.1.0: version "5.1.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853" -safe-buffer@~5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.0.1.tgz#d263ca54696cd8a306b5ca6551e92de57918fbe7" - sane@~1.6.0: version "1.6.0" resolved "https://registry.yarnpkg.com/sane/-/sane-1.6.0.tgz#9610c452307a135d29c1fdfe2547034180c46775" @@ -3115,10 +3111,10 @@ string_decoder@~0.10.x: resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94" string_decoder@~1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.0.2.tgz#b29e1f4e1125fa97a10382b8a533737b7491e179" + version "1.0.3" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.0.3.tgz#0fc67d7c141825de94282dd536bec6b9bce860ab" dependencies: - safe-buffer "~5.0.1" + safe-buffer "~5.1.0" stringstream@~0.0.4: version "0.0.5"