From 51309bf2752bfe9f40713749b9ae06fb2504b612 Mon Sep 17 00:00:00 2001 From: develar Date: Sat, 27 Aug 2016 18:58:18 +0200 Subject: [PATCH] fix: pattern **/*.js still copies all files Closes #701 --- package.json | 2 +- src/asarUtil.ts | 41 ++++++----- src/fileMatcher.ts | 103 +++++++++++++++++++++++++++ src/platformPackager.ts | 149 +++++----------------------------------- src/util/filter.ts | 19 ++--- test/src/globTest.ts | 22 +++++- 6 files changed, 175 insertions(+), 161 deletions(-) create mode 100644 src/fileMatcher.ts diff --git a/package.json b/package.json index 8f6a220e061..27d4671fc58 100644 --- a/package.json +++ b/package.json @@ -73,7 +73,7 @@ "electron-download": "^2.1.2", "electron-osx-sign": "^0.4.0-beta4", "extract-zip": "^1.5.0", - "fs-extra-p": "^1.0.6", + "fs-extra-p": "^1.1.7", "hosted-git-info": "^2.1.5", "image-size": "^0.5.0", "isbinaryfile": "^3.0.1", diff --git a/src/asarUtil.ts b/src/asarUtil.ts index b7465ceeead..a1412ab3004 100644 --- a/src/asarUtil.ts +++ b/src/asarUtil.ts @@ -9,6 +9,7 @@ import * as path from "path" import { log } from "./util/log" import { Minimatch } from "minimatch" import { deepAssign } from "./util/deepAssign" +import { Filter } from "./util/filter" const isBinaryFile: any = BluebirdPromise.promisify(require("isbinaryfile")) const pickle = require ("chromium-pickle-js") @@ -22,27 +23,25 @@ const MAX_FILE_REQUESTS = 32 const concurrency = {concurrency: MAX_FILE_REQUESTS} const NODE_MODULES_PATTERN = path.sep + "node_modules" + path.sep -export function walk(dirPath: string, consumer?: (file: string, stat: Stats) => void, filter?: (file: string) => boolean, addRootToResult?: boolean): BluebirdPromise> { +export function walk(dirPath: string, consumer?: (file: string, stat: Stats) => void, filter?: Filter, addRootToResult?: boolean): BluebirdPromise> { return readdir(dirPath) - .then(names => { - return BluebirdPromise.map(names, name => { - const filePath = dirPath + path.sep + name - if (filter != null && !filter(filePath)) { - return null - } + .then(names => BluebirdPromise.map(names, name => { + const filePath = dirPath + path.sep + name + return lstat(filePath) + .then((stat): any => { + if (filter != null && !filter(filePath, stat)) { + return null + } - return lstat(filePath) - .then((stat): any => { - if (consumer != null) { - consumer(filePath, stat) - } - if (stat.isDirectory()) { - return walk(filePath, consumer, filter, true) - } - return filePath - }) - }, concurrency) - }) + if (consumer != null) { + consumer(filePath, stat) + } + if (stat.isDirectory()) { + return walk(filePath, consumer, filter, true) + } + return filePath + }) + }, concurrency)) .then(list => { list.sort((a, b) => { // files before directories @@ -75,7 +74,7 @@ export function walk(dirPath: string, consumer?: (file: string, stat: Stats) => }) } -export async function createAsarArchive(src: string, resourcesPath: string, options: AsarOptions, filter: (file: string) => boolean): Promise { +export async function createAsarArchive(src: string, resourcesPath: string, options: AsarOptions, filter: Filter): Promise { // sort files to minimize file change (i.e. asar file is not changed dramatically on small change) await new AsarPackager(src, resourcesPath, options).pack(filter) } @@ -96,7 +95,7 @@ class AsarPackager { this.outFile = path.join(this.resourcesPath, "app.asar") } - async pack(filter: (file: string) => boolean) { + async pack(filter: Filter) { const metadata = new Map() const files = await walk(this.src, (it, stat) => { metadata.set(it, stat) diff --git a/src/fileMatcher.ts b/src/fileMatcher.ts new file mode 100644 index 00000000000..626711082f9 --- /dev/null +++ b/src/fileMatcher.ts @@ -0,0 +1,103 @@ +import * as path from "path" +import { createFilter, hasMagic, Filter } from "./util/filter" +import { Minimatch } from "minimatch" +import { asArray } from "./util/util" + +export interface FilePattern { + from?: string + to?: string + filter?: Array | string +} + +export interface FileMatchOptions { + arch: string, + os: string +} + +export class FileMatcher { + readonly from: string + readonly to: string + + readonly patterns: Array + + constructor(from: string, to: string, private options: FileMatchOptions, patterns?: Array | string | n) { + this.from = this.expandPattern(from) + this.to = this.expandPattern(to) + this.patterns = asArray(patterns) + } + + addPattern(pattern: string) { + this.patterns.push(pattern) + } + + isEmpty() { + return this.patterns.length === 0 + } + + getParsedPatterns(fromDir?: string): Array { + const minimatchOptions = {} + + const parsedPatterns: Array = [] + const pathDifference = fromDir ? path.relative(fromDir, this.from) : null + + for (let i = 0; i < this.patterns.length; i++) { + let expandedPattern = this.expandPattern(this.patterns[i]) + if (pathDifference) { + expandedPattern = path.join(pathDifference, expandedPattern) + } + + const parsedPattern = new Minimatch(expandedPattern, minimatchOptions) + parsedPatterns.push(parsedPattern) + + if (!hasMagic(parsedPattern)) { + // https://github.com/electron-userland/electron-builder/issues/545 + // add **/* + parsedPatterns.push(new Minimatch(`${expandedPattern}/**/*`, minimatchOptions)) + } + } + + return parsedPatterns + } + + createFilter(ignoreFiles?: Set, rawFilter?: (file: string) => boolean, excludePatterns?: Array | n): Filter { + return createFilter(this.from, this.getParsedPatterns(), ignoreFiles, rawFilter, excludePatterns) + } + + private expandPattern(pattern: string): string { + return pattern + .replace(/\$\{arch}/g, this.options.arch) + .replace(/\$\{os}/g, this.options.os) + .replace(/\$\{\/\*}/g, "{,/**/*,/**/.*}") + } +} + +export function deprecatedUserIgnoreFilter(ignore: any, appDir: string) { + let ignoreFunc: any + if (typeof (ignore) === "function") { + ignoreFunc = function (file: string) { return !ignore(file) } + } + else { + if (!Array.isArray(ignore)) { + ignore = [ignore] + } + + ignoreFunc = function (file: string) { + for (let i = 0; i < ignore.length; i++) { + if (file.match(ignore[i])) { + return false + } + } + + return true + } + } + + return function filter(file: string) { + let name = file.split(path.resolve(appDir))[1] + if (path.sep === "\\") { + // convert slashes so unix-format ignores work + name = name.replace(/\\/g, "/") + } + return ignoreFunc(name) + } +} \ No newline at end of file diff --git a/src/platformPackager.ts b/src/platformPackager.ts index 60d54c62bc1..1aa85ebd32e 100644 --- a/src/platformPackager.ts +++ b/src/platformPackager.ts @@ -11,9 +11,10 @@ import { Minimatch } from "minimatch" import { checkFileInArchive, createAsarArchive } from "./asarUtil" import { warn, log, task } from "./util/log" import { AppInfo } from "./appInfo" -import { createFilter, copyFiltered, hasMagic, devDependencies } from "./util/filter" +import { copyFiltered, devDependencies } from "./util/filter" import { ElectronPackagerOptions, pack } from "./packager/dirPackager" import { TmpDir } from "./util/tmp" +import { FileMatchOptions, FileMatcher, FilePattern, deprecatedUserIgnoreFilter } from "./fileMatcher" //noinspection JSUnusedLocalSymbols const __awaiter = require("./util/awaiter") @@ -83,85 +84,6 @@ export abstract class TargetEx extends Target { abstract async build(appOutDir: string, arch: Arch): Promise } -export interface FilePattern { - from?: string - to?: string - filter?: Array | string -} - -export interface FileMatchOptions { - arch: string, - os: string -} - -export class FileMatcher { - readonly from: string - readonly to: string - readonly options: FileMatchOptions - - readonly patterns: Array - - constructor(from: string, to: string, options: FileMatchOptions, patterns?: Array | string | n) { - this.options = options - this.patterns = [] - - this.from = this.expandPattern(from) - this.to = this.expandPattern(to) - - if (patterns != null && !Array.isArray(patterns)) { - this.patterns = [patterns] - } - else if (patterns != null) { - this.patterns = patterns - } - } - - addPattern(pattern: string) { - this.patterns.push(pattern) - } - - isEmpty() { - return this.patterns.length === 0 - } - - getParsedPatterns(fromDir?: string): Array { - const minimatchOptions = {} - - const parsedPatterns: Array = [] - const pathDifference = fromDir ? path.relative(fromDir, this.from) : null - - for (let i = 0; i < this.patterns.length; i++) { - let expandedPattern = this.expandPattern(this.patterns[i]) - - if (pathDifference) { - expandedPattern = path.join(pathDifference, expandedPattern) - } - - const parsedPattern = new Minimatch(expandedPattern, minimatchOptions) - parsedPatterns.push(parsedPattern) - - if (!hasMagic(parsedPattern)) { - // https://github.com/electron-userland/electron-builder/issues/545 - // add **/* - parsedPatterns.push(new Minimatch(`${expandedPattern}/*/**`, minimatchOptions)) - } - } - - return parsedPatterns - } - - createFilter(ignoreFiles?: Set, rawFilter?: (file: string) => boolean, excludePatterns?: Array | n): (file: string) => boolean { - return createFilter(this.from, this.getParsedPatterns(), ignoreFiles, rawFilter, excludePatterns) - } - - private expandPattern(pattern: string): string { - return pattern - .replace(/\$\{arch}/g, this.options.arch) - .replace(/\$\{os}/g, this.options.os) - .replace(/\$\{\/\*}/g, "{,/**/*,/**/.*}") - } -} - export abstract class PlatformPackager { readonly options: PackagerOptions @@ -235,9 +157,9 @@ export abstract class PlatformPackager abstract pack(outDir: string, arch: Arch, targets: Array, postAsyncTasks: Array>): Promise - private getExtraFilePatterns(isResources: boolean, appOutDir: string, fileMatchOptions: FileMatchOptions, customBuildOptions: DC): Array | n { + private getExtraFileMatchers(isResources: boolean, appOutDir: string, fileMatchOptions: FileMatchOptions, customBuildOptions: DC): Array | n { const base = isResources ? this.getResourcesDir(appOutDir) : (this.platform === Platform.MAC ? path.join(appOutDir, `${this.appInfo.productFilename}.app`, "Contents") : appOutDir) - return this.getFilePatterns(isResources ? "extraResources" : "extraFiles", this.projectDir, base, true, fileMatchOptions, customBuildOptions) + return this.getFileMatchers(isResources ? "extraResources" : "extraFiles", this.projectDir, base, true, fileMatchOptions, customBuildOptions) } protected async doPack(options: ElectronPackagerOptions, outDir: string, appOutDir: string, platformName: string, arch: Arch, platformSpecificBuildOptions: DC) { @@ -247,8 +169,8 @@ export abstract class PlatformPackager os: this.platform.buildConfigurationKey } - const extraResourcePatterns = this.getExtraFilePatterns(true, appOutDir, fileMatchOptions, platformSpecificBuildOptions) - const extraFilePatterns = this.getExtraFilePatterns(false, appOutDir, fileMatchOptions, platformSpecificBuildOptions) + const extraResourceMatchers = this.getExtraFileMatchers(true, appOutDir, fileMatchOptions, platformSpecificBuildOptions) + const extraFileMatchers = this.getExtraFileMatchers(false, appOutDir, fileMatchOptions, platformSpecificBuildOptions) const resourcesPath = this.platform === Platform.MAC ? path.join(appOutDir, "Electron.app", "Contents", "Resources") : path.join(appOutDir, "resources") @@ -261,7 +183,7 @@ export abstract class PlatformPackager } } - let patterns = this.getFilePatterns("files", this.info.appDir, path.join(resourcesPath, "app"), false, fileMatchOptions, platformSpecificBuildOptions) + const patterns = this.getFileMatchers("files", this.info.appDir, path.join(resourcesPath, "app"), false, fileMatchOptions, platformSpecificBuildOptions) let defaultMatcher = patterns != null ? patterns[0] : new FileMatcher(this.info.appDir, path.join(resourcesPath, "app"), fileMatchOptions) if (defaultMatcher.isEmpty()) { @@ -271,26 +193,26 @@ export abstract class PlatformPackager let rawFilter: any = null const deprecatedIgnore = (this.devMetadata.build).ignore - if (deprecatedIgnore) { + if (deprecatedIgnore != null) { if (typeof deprecatedIgnore === "function") { log(`"ignore is specified as function, may be new "files" option will be suit your needs? Please see https://github.com/electron-userland/electron-builder/wiki/Options#BuildMetadata-files`) } else { warn(`"ignore is deprecated, please use "files", see https://github.com/electron-userland/electron-builder/wiki/Options#BuildMetadata-files`) } - rawFilter = deprecatedUserIgnoreFilter(options, this.info.appDir) + rawFilter = deprecatedUserIgnoreFilter(deprecatedIgnore, this.info.appDir) } let excludePatterns: Array = [] - if (extraResourcePatterns != null) { - for (let i = 0; i < extraResourcePatterns.length; i++) { - const patterns = extraResourcePatterns[i].getParsedPatterns(this.info.projectDir) + if (extraResourceMatchers != null) { + for (let i = 0; i < extraResourceMatchers.length; i++) { + const patterns = extraResourceMatchers[i].getParsedPatterns(this.info.projectDir) excludePatterns = excludePatterns.concat(patterns) } } - if (extraFilePatterns != null) { - for (let i = 0; i < extraFilePatterns.length; i++) { - const patterns = extraFilePatterns[i].getParsedPatterns(this.info.projectDir) + if (extraFileMatchers != null) { + for (let i = 0; i < extraFileMatchers.length; i++) { + const patterns = extraFileMatchers[i].getParsedPatterns(this.info.projectDir) excludePatterns = excludePatterns.concat(patterns) } } @@ -311,8 +233,8 @@ export abstract class PlatformPackager }) await task(`Packaging for platform ${platformName} ${Arch[arch]} using electron ${this.info.electronVersion} to ${path.relative(this.projectDir, appOutDir)}`, p) - await this.doCopyExtraFiles(extraResourcePatterns) - await this.doCopyExtraFiles(extraFilePatterns) + await this.doCopyExtraFiles(extraResourceMatchers) + await this.doCopyExtraFiles(extraFileMatchers) const afterPack = this.devMetadata.build.afterPack if (afterPack != null) { @@ -394,7 +316,7 @@ export abstract class PlatformPackager } } - private getFilePatterns(name: "files" | "extraFiles" | "extraResources", defaultSrc: string, defaultDest: string, allowAdvancedMatching: boolean, fileMatchOptions: FileMatchOptions, customBuildOptions: DC): Array | n { + private getFileMatchers(name: "files" | "extraFiles" | "extraResources", defaultSrc: string, defaultDest: string, allowAdvancedMatching: boolean, fileMatchOptions: FileMatchOptions, customBuildOptions: DC): Array | n { let globalPatterns: Array | string | n = (this.devMetadata.build)[name] let platformSpecificPatterns: Array | string | n = (customBuildOptions)[name] @@ -507,7 +429,7 @@ export abstract class PlatformPackager } } - async getTempFile(suffix: string): Promise { + getTempFile(suffix: string): Promise { return this.info.tempDirManager.getTempFile(suffix) } } @@ -535,37 +457,4 @@ export function smarten(s: string): string { // closing doubles s = s.replace(/"/g, "\u201d") return s -} - -export function deprecatedUserIgnoreFilter(opts: ElectronPackagerOptions, appDir: string) { - let ignore = opts.ignore || [] - let ignoreFunc: any - - if (typeof (ignore) === "function") { - ignoreFunc = function (file: string) { return !ignore(file) } - } - else { - if (!Array.isArray(ignore)) { - ignore = [ignore] - } - - ignoreFunc = function (file: string) { - for (let i = 0; i < ignore.length; i++) { - if (file.match(ignore[i])) { - return false - } - } - - return true - } - } - - return function filter(file: string) { - let name = file.split(path.resolve(appDir))[1] - if (path.sep === "\\") { - // convert slashes so unix-format ignores work - name = name.replace(/\\/g, "/") - } - return ignoreFunc(name) - } } \ No newline at end of file diff --git a/src/util/filter.ts b/src/util/filter.ts index 9fa113181f3..bc5cd3637fb 100644 --- a/src/util/filter.ts +++ b/src/util/filter.ts @@ -1,4 +1,4 @@ -import { copy } from "fs-extra-p" +import { copy, Stats } from "fs-extra-p" import { Minimatch } from "minimatch" import * as path from "path" import { Promise as BluebirdPromise } from "bluebird" @@ -8,10 +8,11 @@ const __awaiter = require("./awaiter") const readInstalled = require("read-installed") // we use relative path to avoid canonical path issue - e.g. /tmp vs /private/tmp -export function copyFiltered(src: string, destination: string, filter: (file: string) => boolean, dereference: boolean): Promise { +export function copyFiltered(src: string, destination: string, filter: Filter, dereference: boolean): Promise { return copy(src, destination, { dereference: dereference, - filter: filter + filter: filter, + passStats: true, }) } @@ -30,8 +31,10 @@ export function hasMagic(pattern: Minimatch) { return false } -export function createFilter(src: string, patterns: Array, ignoreFiles?: Set, rawFilter?: (file: string) => boolean, excludePatterns?: Array | null): (file: string) => boolean { - return function filter(it) { +export type Filter = (file: string, stat: Stats) => boolean + +export function createFilter(src: string, patterns: Array, ignoreFiles?: Set, rawFilter?: (file: string) => boolean, excludePatterns?: Array | null): Filter { + return function filter(it, stat) { if (src === it) { return true } @@ -51,7 +54,7 @@ export function createFilter(src: string, patterns: Array, ignoreFile relative = relative.replace(/\\/g, "/") } - return minimatchAll(relative, patterns) && (excludePatterns == null || !minimatchAll(relative, excludePatterns)) + return minimatchAll(relative, patterns, stat) && (excludePatterns == null || !minimatchAll(relative, excludePatterns, stat)) } } @@ -95,7 +98,7 @@ function flatDependencies(data: any, seen: Set): any { } // https://github.com/joshwnj/minimatch-all/blob/master/index.js -function minimatchAll(path: string, patterns: Array): boolean { +function minimatchAll(path: string, patterns: Array, stat: Stats): boolean { let match = false for (let pattern of patterns) { // If we've got a match, only re-test for exclusions. @@ -106,7 +109,7 @@ function minimatchAll(path: string, patterns: Array): boolean { // partial match — pattern: foo/bar.txt path: foo — we must allow foo // use it only for non-negate patterns: const m = new Minimatch("!node_modules/@(electron-download|electron)/**/*", {dot: true }); m.match("node_modules", true) will return false, but must be true - match = pattern.match(path, !pattern.negate) + match = pattern.match(path, stat.isDirectory() && !pattern.negate) } return match } \ No newline at end of file diff --git a/test/src/globTest.ts b/test/src/globTest.ts index 74e8a23dfb7..30e127f090b 100644 --- a/test/src/globTest.ts +++ b/test/src/globTest.ts @@ -1,7 +1,7 @@ import test from "./helpers/avaEx" import { expectedWinContents } from "./helpers/expectedContents" import { outputFile, symlink } from "fs-extra-p" -import { assertPack, modifyPackageJson, getPossiblePlatforms, app } from "./helpers/packTester" +import { assertPack, modifyPackageJson, getPossiblePlatforms, app, appThrows } from "./helpers/packTester" import { Promise as BluebirdPromise } from "bluebird" import * as path from "path" import { assertThat } from "./helpers/fileAssert" @@ -302,3 +302,23 @@ test("extraResources - one-package", async () => { }) } }) + +test.ifDevOrLinuxCi("copy only js files - no asar", appThrows(/Application "package.json" does not exist/, { + targets: Platform.LINUX.createTarget(DIR_TARGET), + devMetadata: { + build: { + "files": ["**/*.js"], + asar: false, + } + } +})) + +test.ifDevOrLinuxCi("copy only js files - asar", appThrows(/Application "package.json" in the /, { + targets: Platform.LINUX.createTarget(DIR_TARGET), + devMetadata: { + build: { + "files": ["**/*.js"], + asar: true, + } + } +})) \ No newline at end of file