From 5853514bbde2cc2395609c7b0bee932c0d577e20 Mon Sep 17 00:00:00 2001 From: develar Date: Tue, 7 Jun 2016 08:14:19 +0200 Subject: [PATCH] fix: efficient implementation of copy extra files/resources --- .idea/dictionaries/develar.xml | 1 + .idea/runConfigurations/BuildTest.xml | 2 +- docs/Options.md | 25 ++++-- package.json | 7 +- src/globby.ts | 55 ------------ src/linuxPackager.ts | 2 +- src/metadata.ts | 11 +-- src/platformPackager.ts | 76 ++++++++++++---- test/src/globTest.ts | 2 +- test/tsconfig.json | 1 + tsconfig.json | 2 +- typings.json | 1 + typings/modules/minimatch/index.d.ts | 116 +++++++++++++++++++++++++ typings/modules/minimatch/typings.json | 11 +++ 14 files changed, 225 insertions(+), 87 deletions(-) delete mode 100644 src/globby.ts create mode 100644 typings/modules/minimatch/index.d.ts create mode 100644 typings/modules/minimatch/typings.json diff --git a/.idea/dictionaries/develar.xml b/.idea/dictionaries/develar.xml index fcc6c3bbde9..9dd44fdb82c 100644 --- a/.idea/dictionaries/develar.xml +++ b/.idea/dictionaries/develar.xml @@ -37,6 +37,7 @@ lzma lzop makedeb + minimatch mkdirp mpass multilib diff --git a/.idea/runConfigurations/BuildTest.xml b/.idea/runConfigurations/BuildTest.xml index 0af9c0cc35b..a8e2d4f3070 100644 --- a/.idea/runConfigurations/BuildTest.xml +++ b/.idea/runConfigurations/BuildTest.xml @@ -1,5 +1,5 @@ - + diff --git a/docs/Options.md b/docs/Options.md index 2d13a61bf04..9cb14152f73 100644 --- a/docs/Options.md +++ b/docs/Options.md @@ -54,7 +54,7 @@ Here documented only `electron-builder` specific options: | app-category-type |

*OS X-only.* The application category type, as shown in the Finder via *View -> Arrange by Application Category* when viewing the Applications directory.

For example, app-category-type=public.app-category.developer-tools will set the application category to *Developer Tools*.

Valid values are listed in [Apple’s documentation](https://developer.apple.com/library/ios/documentation/General/Reference/InfoPlistKeyReference/Articles/LaunchServicesKeys.html#//apple_ref/doc/uid/TP40009250-SW8).

| asar |

Whether to package the application’s source code into an archive, using [Electron’s archive format](https://github.com/electron/asar). Defaults to true. Reasons why you may want to disable this feature are described in [an application packaging tutorial in Electron’s documentation](http://electron.atom.io/docs/latest/tutorial/application-packaging/#limitations-on-node-api/).

Or you can pass object of any asar options.

| productName | See [AppMetadata.productName](#AppMetadata-productName). -| extraResources |

A [glob expression](https://www.npmjs.com/package/glob#glob-primer), when specified, copy the file or directory with matching names directly into the app’s resources directory (Contents/Resources for OS X, resources for Linux/Windows).

You can use ${os} (expanded to osx, linux or win according to current platform) and ${arch} in the pattern.

If directory matched, all contents are copied. So, you can just specify foo to copy <project_dir>/foo directory.

May be specified in the platform options (i.e. in the build.osx).

+| extraResources |

A [glob expression](https://www.npmjs.com/package/glob#glob-primer), when specified, copy the file or directory with matching names directly into the app’s resources directory (Contents/Resources for OS X, resources for Linux/Windows). [Multiple patterns](#multiple-glob-patterns) are supported.

You can use ${os} (expanded to osx, linux or win according to current platform) and ${arch} in the pattern.

If directory matched, all contents are copied. So, you can just specify foo to copy <project_dir>/foo directory.

May be specified in the platform options (e.g. in the build.osx).

| extraFiles | The same as [extraResources](#BuildMetadata-extraResources) but copy into the app's content directory (`Contents` for OS X, `` for Linux/Windows). | osx | See [.build.osx](#OsXBuildOptions). | mas | See [.build.mas](#MasBuildOptions). @@ -107,7 +107,7 @@ MAS (Mac Application Store) specific options (in addition to `build.osx`). | synopsis | *deb-only.* The [short description](https://www.debian.org/doc/debian-policy/ch-controlfields.html#s-f-Description). | maintainer | The maintainer. Defaults to [author](#AppMetadata-author). | vendor | The vendor. Defaults to [author](#AppMetadata-author). -| compression | *deb-only.* The compression type, one of `gz`, `bzip2`, `xz` (default: `xz`). +| compression | *deb-only.* The compression type, one of `gz`, `bzip2`, `xz`. Defaults to `xz`. | depends | Package dependencies. Defaults to `["libappindicator1", "libnotify-bin"]`. | target |

Target package type: list of default, deb, rpm, freebsd, pacman, p5p, apk, 7z, zip, tar.xz, tar.lz, tar.gz, tar.bz2. Defaults to default (deb).

Only deb is tested. Feel free to file issues for rpm and other package formats.

@@ -115,8 +115,23 @@ MAS (Mac Application Store) specific options (in addition to `build.osx`). ## `.directories` | Name | Description | --- | --- -| buildResources | The path to build resources, default `build`. -| output | The output directory, default `dist`. -| app | The application directory (containing the application package.json), default `app`, `www` or working directory. +| buildResources | The path to build resources, defaults to `build`. +| output | The output directory, defaults to `dist`. +| app | The application directory (containing the application package.json), defaults to `app`, `www` or working directory. + + +# Multiple Glob Patterns + ```js + [ + // match all files + "**/*", + + // except for js files in the foo/ directory + "!foo/*.js", + + // unless it's foo/bar.js + "foo/bar.js", + ] + ``` \ No newline at end of file diff --git a/package.json b/package.json index a9a6aaf37ff..979bfbd5b31 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,7 @@ "asar": "^0.11.0", "bluebird": "^3.4.0", "chalk": "^1.1.3", - "compare-versions": "^2.0.1", + "compare-versions": "^2.0.2", "debug": "^2.2.0", "deep-assign": "^2.0.0", "electron-osx-sign-tf": "0.6.0", @@ -73,6 +73,7 @@ "image-size": "^0.5.0", "lodash.template": "^4.2.5", "mime": "^1.3.4", + "minimatch": "^3.0.0", "progress": "^1.1.8", "progress-stream": "^1.2.0", "read-package-json": "^2.0.4", @@ -106,10 +107,10 @@ "pre-git": "^3.8.4", "semantic-release": "^6.3.0", "should": "^9.0.0", - "ts-babel": "^0.8.6", + "ts-babel": "^1.0.0", "tsconfig-glob": "^0.4.3", "tslint": "3.10.0-dev.2", - "typescript": "1.9.0-dev.20160520-1.0", + "typescript": "1.9.0-dev.20160607-1.0", "whitespace": "^2.0.0" }, "babel": { diff --git a/src/globby.ts b/src/globby.ts deleted file mode 100644 index aa0025796b0..00000000000 --- a/src/globby.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { Promise as BluebirdPromise } from "bluebird" -import { Glob, Options } from "glob" - -//noinspection JSUnusedLocalSymbols -const __awaiter = require("./awaiter") - -function isNegative(pattern: string): boolean { - return pattern[0] === "!" -} - -function generateGlobTasks(patterns: Array, opts: Options): Array { - opts = Object.assign({ignore: []}, opts) - - const globTasks: Array = [] - patterns.forEach(function (pattern, i) { - if (isNegative(pattern)) { - return - } - - const ignore = patterns.slice(i).filter(isNegative).map(it => it.slice(1)) - globTasks.push({ - pattern: pattern, - opts: Object.assign({}, opts, { - ignore: (>opts.ignore).concat(ignore) - }) - }) - }) - return globTasks -} - -export function globby(patterns: Array, opts: Options): Promise> { - let firstGlob: Glob | null = null - return BluebirdPromise - .map(generateGlobTasks(patterns, opts), task => new BluebirdPromise((resolve, reject) => { - let glob = new Glob(task.pattern, task.opts, (error, matches) => { - if (error == null) { - resolve(matches) - } - else { - reject(error) - } - }) - - if (firstGlob == null) { - firstGlob = glob - } - else { - glob.statCache = firstGlob.statCache - glob.symlinks = firstGlob.symlinks - glob.realpathCache = firstGlob.realpathCache - glob.cache = firstGlob.cache - } - })) - .then(it => new Set([].concat(...it))) -} \ No newline at end of file diff --git a/src/linuxPackager.ts b/src/linuxPackager.ts index 9db79b78140..1a7543ec8f9 100755 --- a/src/linuxPackager.ts +++ b/src/linuxPackager.ts @@ -66,7 +66,7 @@ export class LinuxPackager extends PlatformPackager { promises.push(this.computeDesktop(tempDir)) - return [].concat(...await BluebirdPromise.all(promises)) + return Array.prototype.concat.apply([], await BluebirdPromise.all(promises)) } async pack(outDir: string, arch: Arch, targets: Array, postAsyncTasks: Array>): Promise { diff --git a/src/metadata.ts b/src/metadata.ts index 6d7afb12815..2ec3367ca2f 100755 --- a/src/metadata.ts +++ b/src/metadata.ts @@ -109,12 +109,13 @@ export interface BuildMetadata { /** A [glob expression](https://www.npmjs.com/package/glob#glob-primer), when specified, copy the file or directory with matching names directly into the app's resources directory (`Contents/Resources` for OS X, `resources` for Linux/Windows). + [Multiple patterns](#multiple-glob-patterns) are supported. You can use `${os}` (expanded to osx, linux or win according to current platform) and `${arch}` in the pattern. If directory matched, all contents are copied. So, you can just specify `foo` to copy `/foo` directory. - May be specified in the platform options (i.e. in the `build.osx`). + May be specified in the platform options (e.g. in the `build.osx`). */ readonly extraResources?: Array | null @@ -307,7 +308,7 @@ export interface LinuxBuildOptions extends PlatformSpecificBuildOptions { afterRemove?: string | null /* - *deb-only.* The compression type, one of `gz`, `bzip2`, `xz` (default: `xz`). + *deb-only.* The compression type, one of `gz`, `bzip2`, `xz`. Defaults to `xz`. */ readonly compression?: string | null @@ -329,17 +330,17 @@ export interface LinuxBuildOptions extends PlatformSpecificBuildOptions { */ export interface MetadataDirectories { /* - The path to build resources, default `build`. + The path to build resources, defaults to `build`. */ readonly buildResources?: string | null /* - The output directory, default `dist`. + The output directory, defaults to `dist`. */ readonly output?: string | null /* - The application directory (containing the application package.json), default `app`, `www` or working directory. + The application directory (containing the application package.json), defaults to `app`, `www` or working directory. */ readonly app?: string | null } diff --git a/src/platformPackager.ts b/src/platformPackager.ts index d391927f25c..aee6f6b7b96 100644 --- a/src/platformPackager.ts +++ b/src/platformPackager.ts @@ -4,7 +4,6 @@ import EventEmitter = NodeJS.EventEmitter import { Promise as BluebirdPromise } from "bluebird" import * as path from "path" import { pack, ElectronPackagerOptions } from "electron-packager-tf" -import { globby } from "./globby" import { readdir, copy, unlink, lstat, remove } from "fs-extra-p" import { statOrNull, use, spawn, debug7zArgs, debug, warn, log } from "./util" import { Packager } from "./packager" @@ -12,6 +11,7 @@ import { listPackage, statFile, AsarFileMetadata, createPackageFromFiles, AsarOp import { path7za } from "7zip-bin" import deepAssign = require("deep-assign") import { Glob } from "glob" +import { Minimatch } from "minimatch" //noinspection JSUnusedLocalSymbols const __awaiter = require("./awaiter") @@ -234,15 +234,6 @@ export abstract class PlatformPackager return options } - private getExtraResources(isResources: boolean, arch: Arch, customBuildOptions: DC): Promise> { - let patterns: Array | n = (this.devMetadata.build)[isResources ? "extraResources" : "extraFiles"] - const platformSpecificPatterns = isResources ? customBuildOptions.extraResources : customBuildOptions.extraFiles - if (platformSpecificPatterns != null) { - patterns = patterns == null ? platformSpecificPatterns : patterns.concat(platformSpecificPatterns) - } - return patterns == null ? BluebirdPromise.resolve(new Set()) : globby(this.expandPatterns(patterns, arch), {cwd: this.projectDir}); - } - private computeAsarOptions(customBuildOptions: DC): AsarOptions | null { let result = this.devMetadata.build.asar let platformSpecific = customBuildOptions.asar @@ -313,10 +304,10 @@ export abstract class PlatformPackager await remove(src) } - private expandPatterns(list: Array, arch: Arch): Array { - return list.map(it => it + private expandPattern(pattern: string, arch: Arch): string { + return pattern .replace(/\$\{arch}/g, Arch[arch]) - .replace(/\$\{os}/g, this.platform.buildConfigurationKey)) + .replace(/\$\{os}/g, this.platform.buildConfigurationKey) } protected async copyExtraFiles(appOutDir: string, arch: Arch, customBuildOptions: DC): Promise { @@ -324,9 +315,39 @@ export abstract class PlatformPackager await this.doCopyExtraFiles(false, appOutDir, arch, customBuildOptions) } - private async doCopyExtraFiles(isResources: boolean, appOutDir: string, arch: Arch, customBuildOptions: DC): Promise> { + private async doCopyExtraFiles(isResources: boolean, appOutDir: string, arch: Arch, customBuildOptions: DC): Promise { const base = isResources ? this.getResourcesDir(appOutDir) : this.platform === Platform.OSX ? path.join(appOutDir, `${this.appName}.app`, "Contents") : appOutDir - return await BluebirdPromise.map(await this.getExtraResources(isResources, arch, customBuildOptions), it => copy(path.join(this.projectDir, it), path.join(base, it))) + + let patterns: Array | n = (this.devMetadata.build)[isResources ? "extraResources" : "extraFiles"] + const platformSpecificPatterns = isResources ? customBuildOptions.extraResources : customBuildOptions.extraFiles + if (platformSpecificPatterns != null) { + patterns = patterns == null ? platformSpecificPatterns : patterns.concat(platformSpecificPatterns) + } + + if (patterns == null) { + return + } + + const minimatchOptions = {} + const parsedPatterns: Array = [] + for (let i = 0; i < patterns.length; i++) { + parsedPatterns[i] = new Minimatch(this.expandPattern(patterns[i], arch), minimatchOptions) + } + + const src = this.projectDir + return copy(src, base, { + filter: (it) => { + if (src === it) { + return true + } + + let relative = it.substring(src.length + 1) + if (path.sep === "\\") { + relative = relative.replace(/\\/g, "/") + } + return minimatchAll(relative, parsedPatterns) + } + }) } protected async computePackageUrl(): Promise { @@ -501,3 +522,28 @@ export function smarten(s: string): string { s = s.replace(/"/g, "\u201d") return s } + +// https://github.com/joshwnj/minimatch-all/blob/master/index.js +function minimatchAll(path: string, patterns: Array): boolean { + let match = false + for (let pattern of patterns) { + // If we've got a match, only re-test for exclusions. + // if we don't have a match, only re-test for inclusions. + if (match !== pattern.negate) { + continue + } + + // partial match — pattern: foo/bar.txt path: foo — we must allow foo + match = pattern.match(path, true) + if (!match && !pattern.negate) { + const rawPattern = pattern.pattern + // 1 - slash + const patternLengthPlusSlash = rawPattern.length + 1 + if (path.length > patternLengthPlusSlash) { + // foo: include all directory content + match = path[rawPattern.length] === "/" && path.startsWith(rawPattern) + } + } + } + return match +} \ No newline at end of file diff --git a/test/src/globTest.ts b/test/src/globTest.ts index 56fbc96e218..f03c3ae31dc 100644 --- a/test/src/globTest.ts +++ b/test/src/globTest.ts @@ -29,7 +29,7 @@ test.ifDevOrLinuxCi("ignore build resources", () => { }) }) -test("copy extra content", async () => { +test("extraResources", async () => { for (let platform of getPossiblePlatforms().keys()) { const osName = platform.buildConfigurationKey diff --git a/test/tsconfig.json b/test/tsconfig.json index c98400d64c0..bd73d78875d 100755 --- a/test/tsconfig.json +++ b/test/tsconfig.json @@ -42,6 +42,7 @@ "../typings/main/definitions/debug/index.d.ts", "../typings/main/definitions/source-map-support/source-map-support.d.ts", "../typings/modules/glob/index.d.ts", + "../typings/modules/minimatch/index.d.ts", "../typings/node.d.ts", "../typings/progress-stream.d.ts", "../typings/read-package-json.d.ts", diff --git a/tsconfig.json b/tsconfig.json index acb8a3b8b6a..1d47214b44c 100755 --- a/tsconfig.json +++ b/tsconfig.json @@ -46,6 +46,7 @@ "typings/main/definitions/debug/index.d.ts", "typings/main/definitions/source-map-support/source-map-support.d.ts", "typings/modules/glob/index.d.ts", + "typings/modules/minimatch/index.d.ts", "typings/node.d.ts", "typings/progress-stream.d.ts", "typings/read-package-json.d.ts", @@ -65,7 +66,6 @@ "src/fpmDownload.ts", "src/gitHubPublisher.ts", "src/gitHubRequest.ts", - "src/globby.ts", "src/httpRequest.ts", "src/index.ts", "src/install-app-deps.ts", diff --git a/typings.json b/typings.json index ee848027787..621a51e7b24 100755 --- a/typings.json +++ b/typings.json @@ -5,6 +5,7 @@ }, "dependencies": { "glob": "registry:npm/glob#6.0.0+20160211003958", + "minimatch": "registry:npm/minimatch#3.0.0+20160211003958", "source-map-support": "github:typed-typings/npm-source-map-support#900ed4180a22285bce4bbabc0760427e71a59eca" } } diff --git a/typings/modules/minimatch/index.d.ts b/typings/modules/minimatch/index.d.ts new file mode 100644 index 00000000000..30a5fbae834 --- /dev/null +++ b/typings/modules/minimatch/index.d.ts @@ -0,0 +1,116 @@ +// Generated by typings +// Source: https://raw.githubusercontent.com/typed-typings/npm-minimatch/74f47de8acb42d668491987fc6bc144e7d9aa891/minimatch.d.ts +declare module '~minimatch/minimatch' { +function minimatch (target: string, pattern: string, options?: minimatch.Options): boolean; + +namespace minimatch { + export function match (list: string[], pattern: string, options?: Options): string[]; + export function filter (pattern: string, options?: Options): (element: string, indexed: number, array: string[]) => boolean; + export function makeRe (pattern: string, options?: Options): RegExp; + + /** + * All options are `false` by default. + */ + export interface Options { + /** + * Dump a ton of stuff to stderr. + */ + debug?: boolean; + /** + * Do not expand `{a,b}` and `{1..3}` brace sets. + */ + nobrace?: boolean; + /** + * Disable `**` matching against multiple folder names. + */ + noglobstar?: boolean; + /** + * Allow patterns to match filenames starting with a period, even if the pattern does not explicitly have a period in that spot. + * + * Note that by default, `a\/**\/b` will not match `a/.d/b`, unless `dot` is set. + */ + dot?: boolean; + /** + * Disable "extglob" style patterns like `+(a|b)`. + */ + noext?: boolean; + /** + * Perform a case-insensitive match. + */ + nocase?: boolean; + /** + * When a match is not found by `minimatch.match`, return a list containing the pattern itself if this option is set. When not set, an empty list is returned if there are no matches. + */ + nonull?: boolean; + /** + * If set, then patterns without slashes will be matched against the basename of the path if it contains slashes. For example, `a?b` would match the path `/xyz/123/acb`, but not `/xyz/acb/123`. + */ + matchBase?: boolean; + /** + * Suppress the behavior of treating `#` at the start of a pattern as a comment. + */ + nocomment?: boolean; + /** + * Suppress the behavior of treating a leading `!` character as negation. + */ + nonegate?: boolean; + /** + * Returns from negate expressions the same as if they were not negated. (Ie, true on a hit, false on a miss.) + */ + flipNegate?: boolean; + } + + export class Minimatch { + constructor (pattern: string, options?: Options); + + /** + * The original pattern the minimatch object represents. + */ + pattern: string; + /** + * The options supplied to the constructor. + */ + options: Options; + + /** + * Created by the `makeRe` method. A single regular expression expressing the entire pattern. This is useful in cases where you wish to use the pattern somewhat like `fnmatch(3)` with `FNM_PATH` enabled. + */ + regexp: RegExp; + /** + * True if the pattern is negated. + */ + negate: boolean; + /** + * True if the pattern is a comment. + */ + comment: boolean; + /** + * True if the pattern is `""`. + */ + empty: boolean; + + /** + * Generate the regexp member if necessary, and return it. Will return false if the pattern is invalid. + */ + makeRe (): RegExp | boolean; + /** + * Return true if the filename matches the pattern, or false otherwise. + */ + match (fname: string, partial?: boolean): boolean; + /** + * Take a `/-`split filename, and match it against a single row in the `regExpSet`. This method is mainly for internal use, but is exposed so that it can be used by a glob-walker that needs to avoid excessive filesystem calls. + */ + matchOne (fileArray: string[], patternArray: string[], partial: boolean): boolean; + } +} + +export = minimatch; +} +declare module 'minimatch/minimatch' { +import alias = require('~minimatch/minimatch'); +export = alias; +} +declare module 'minimatch' { +import alias = require('~minimatch/minimatch'); +export = alias; +} diff --git a/typings/modules/minimatch/typings.json b/typings/modules/minimatch/typings.json new file mode 100644 index 00000000000..22ff949ed9f --- /dev/null +++ b/typings/modules/minimatch/typings.json @@ -0,0 +1,11 @@ +{ + "resolution": "main", + "tree": { + "src": "https://raw.githubusercontent.com/typed-typings/npm-minimatch/74f47de8acb42d668491987fc6bc144e7d9aa891/typings.json", + "raw": "registry:npm/minimatch#3.0.0+20160211003958", + "main": "minimatch.d.ts", + "global": false, + "name": "minimatch", + "type": "typings" + } +}