feat: ignore dll/exe files from node_modules if target platform not w…

…indows

Close electron-userland#1738

docs/Options.md

@@ -76,7 +76,7 @@ You can use macros in the file patterns, artifact file name patterns and publish
 * `**/*`
 * `!**/node_modules/*/{CHANGELOG.md,README.md,README,readme.md,readme,test,__tests__,tests,powered-test,example,examples,*.d.ts}`
 * `!**/node_modules/.bin`
-* `!**/*.{o,hprof,orig,pyc,pyo,rbc}`
+* `!**/*.{iml,o,hprof,orig,pyc,pyo,rbc,swp,csproj,sln,xproj}`
 * `!**/._*`
 * `!**/{.DS_Store,.git,.hg,.svn,CVS,RCS,SCCS,__pycache__,thumbs.db,.gitignore,.gitattributes,.editorconfig,.flowconfig,.yarn-metadata.json,.idea,.vs,appveyor.yml,.travis.yml,circle.yml,npm-debug.log,.nyc_output,yarn.lock,.yarn-integrity}`
 

package.json

@@ -66,10 +66,10 @@
   },
   "devDependencies": {
     "@types/ini": "^1.3.29",
-    "@types/jest": "^20.0.1",
+    "@types/jest": "^20.0.2",
     "@types/js-yaml": "^3.5.31",
     "@types/source-map-support": "^0.4.0",
-    "@types/xml2js": "^0.0.33",
+    "@types/xml2js": "^0.4.0",
     "babel-plugin-array-includes": "^2.0.3",
     "babel-plugin-transform-async-to-module-method": "^6.24.1",
     "babel-plugin-transform-es2015-destructuring": "^6.23.0",

packages/electron-builder/src/fileMatcher.ts

@@ -4,16 +4,17 @@ import { copyDir, copyOrLinkFile, Filter, statOrNull } from "electron-builder-ut
 import { mkdirs } from "fs-extra-p"
 import { Minimatch } from "minimatch"
 import * as path from "path"
+import { Platform } from "./core"
 import { Config, FilePattern, PlatformSpecificBuildOptions } from "./metadata"
-import { BuildInfo } from "./packagerApi"
+import { PlatformPackager } from "./platformPackager"
 import { createFilter, hasMagic } from "./util/filter"
 
 /** @internal */
 export class FileMatcher {
   readonly from: string
   readonly to: string
 
-  private readonly patterns: Array<string>
+  readonly patterns: Array<string>
 
   constructor(from: string, to: string, private readonly macroExpander: (pattern: string) => string, patterns?: Array<string> | string | n) {
     this.from = macroExpander(from)
@@ -86,31 +87,49 @@ export class FileMatcher {
 }
 
 /** @internal */
-export function createFileMatcher(info: BuildInfo, appDir: string, resourcesPath: string, macroExpander: (pattern: string) => string, platformSpecificBuildOptions: PlatformSpecificBuildOptions, buildResourceDir: string) {
-  const patterns = info.isPrepackedAppAsar ? null : getFileMatchers(info.config, "files", appDir, path.join(resourcesPath, "app"), false, macroExpander, platformSpecificBuildOptions)
+export function createFileMatcher(appDir: string, resourcesPath: string, macroExpander: (pattern: string) => string, platformSpecificBuildOptions: PlatformSpecificBuildOptions, packager: PlatformPackager<any>) {
+  const buildResourceDir = path.resolve(packager.info.projectDir, packager.buildResourcesDir)
+
+  const patterns = packager.info.isPrepackedAppAsar ? null : getFileMatchers(packager.info.config, "files", appDir, path.join(resourcesPath, "app"), false, macroExpander, platformSpecificBuildOptions)
   const matcher = patterns == null ? new FileMatcher(appDir, path.join(resourcesPath, "app"), macroExpander) : patterns[0]
 
   const relativeBuildResourceDir = path.relative(matcher.from, buildResourceDir)
   const ignoreBuildResourceDirPattern = (relativeBuildResourceDir.length !== 0 && !relativeBuildResourceDir.startsWith(".")) ? `!${relativeBuildResourceDir}{,/**/*}` : null
+  const customFirstPatterns: Array<string> = []
   if (matcher.isEmpty() || matcher.containsOnlyIgnore()) {
     if (ignoreBuildResourceDirPattern != null) {
       matcher.addPattern(ignoreBuildResourceDirPattern)
     }
-    matcher.prependPattern("**/*")
+    customFirstPatterns.push("**/*")
   }
   else {
     if (ignoreBuildResourceDirPattern != null) {
-      matcher.prependPattern(ignoreBuildResourceDirPattern)
+      customFirstPatterns.push(ignoreBuildResourceDirPattern)
     }
+
     // prependPattern - user pattern should be after to be able to override
-    matcher.prependPattern("**/node_modules/**/*")
+    customFirstPatterns.push("**/node_modules/**/*")
     matcher.addPattern("package.json")
   }
+
+  if (packager.platform !== Platform.WINDOWS) {
+    // https://github.com/electron-userland/electron-builder/issues/1738
+    customFirstPatterns.push("!**/node_modules/**/*.{dll,exe}")
+  }
+
+  matcher.patterns.unshift(...customFirstPatterns)
+
+  // https://github.com/electron-userland/electron-builder/issues/1738#issuecomment-310729208
+  // must be before common ignore patterns (to ignore common ignores like .svn)
+  matcher.addPattern("!**/node_modules/lzma-native/build/**/*")
+  matcher.addPattern("**/node_modules/lzma-native/build/{Release,Debug}")
+  matcher.addPattern("!**/node_modules/lzma-native/deps/xz-*")
+  matcher.addPattern("!**/node_modules/lzma-native/deps/doc{,/**/*}")
+
   matcher.addPattern("!**/node_modules/*/{CHANGELOG.md,ChangeLog,changelog.md,README.md,README,readme.md,readme,test,__tests__,tests,powered-test,example,examples,*.d.ts}")
   matcher.addPattern("!**/node_modules/.bin")
-  matcher.addPattern("!**/*.{o,hprof,orig,pyc,pyo,rbc,swp}")
+  matcher.addPattern(`!**/*.{iml,o,hprof,orig,pyc,pyo,rbc,swp,csproj,sln,xproj}`)
   matcher.addPattern("!**/._*")
-  matcher.addPattern("!*.iml")
   //noinspection SpellCheckingInspection
   matcher.addPattern("!**/{.git,.hg,.svn,CVS,RCS,SCCS," +
     "__pycache__,.DS_Store,thumbs.db,.gitignore,.gitattributes," +

packages/electron-builder/src/platformPackager.ts

@@ -165,7 +165,13 @@ export abstract class PlatformPackager<DC extends PlatformSpecificBuildOptions>
       }
     }
 
-    const defaultMatcher = createFileMatcher(this.info, appDir, resourcesPath, macroExpander, platformSpecificBuildOptions, path.resolve(this.info.projectDir, this.buildResourcesDir))
+    const packContext: AfterPackContext = {
+      appOutDir, outDir, arch, targets,
+      packager: this,
+      electronPlatformName: platformName,
+    }
+
+    const defaultMatcher = createFileMatcher(appDir, resourcesPath, macroExpander, platformSpecificBuildOptions, this)
     const isElectronCompile = asarOptions != null && isElectronCompileUsed(this.info)
     if (isElectronCompile) {
       defaultMatcher.addPattern("!.cache{,/**/*}")
@@ -190,12 +196,6 @@ export abstract class PlatformPackager<DC extends PlatformSpecificBuildOptions>
       promise = new AsarPackager(appDir, resourcesPath, asarOptions, fileMatcher == null ? null : fileMatcher.createFilter(), transformer).pack(filter, isElectronCompile, this)
     }
 
-    const packContext: AfterPackContext = {
-      appOutDir, outDir, arch, targets,
-      packager: this,
-      electronPlatformName: platformName,
-    }
-
     //noinspection ES6MissingAwait
     const promises = [promise, unlinkIfExists(path.join(resourcesPath, "default_app.asar")), unlinkIfExists(path.join(appOutDir, "version")), this.postInitApp(packContext)]
     if (this.platform !== Platform.MAC) {

test/out/__snapshots__/BuildTest.js.snap

@@ -270,19 +270,32 @@ Object {
 }
 `;
 
-exports[`relative index 1`] = `
+exports[`posix smart unpack 1`] = `
 Object {
   "linux": Array [],
 }
 `;
 
-exports[`smart unpack 1`] = `
+exports[`posix smart unpack 2`] = `
+Array [
+  "app.asar",
+  "electron.asar",
+]
+`;
+
+exports[`relative index 1`] = `
 Object {
   "linux": Array [],
 }
 `;
 
-exports[`smart unpack 2`] = `
+exports[`win smart unpack 1`] = `
+Object {
+  "win": Array [],
+}
+`;
+
+exports[`win smart unpack 2`] = `
 Array [
   "app.asar",
   "electron.asar",
@@ -296,14 +309,10 @@ Array [
   "app.asar.unpacked/node_modules/edge-cs/src",
   "app.asar.unpacked/node_modules/edge-cs/src/edge-cs",
   "app.asar.unpacked/node_modules/edge-cs/src/edge-cs/EdgeCompiler.cs",
-  "app.asar.unpacked/node_modules/edge-cs/src/edge-cs/edge-cs.csproj",
-  "app.asar.unpacked/node_modules/edge-cs/src/edge-cs/edge-cs.sln",
   "app.asar.unpacked/node_modules/edge-cs/src/edge-cs/Properties",
   "app.asar.unpacked/node_modules/edge-cs/src/edge-cs/Properties/AssemblyInfo.cs",
   "app.asar.unpacked/node_modules/edge-cs/src/Edge.js.CSharp",
   "app.asar.unpacked/node_modules/edge-cs/src/Edge.js.CSharp/EdgeCompiler.cs",
-  "app.asar.unpacked/node_modules/edge-cs/src/Edge.js.CSharp/edge-cs-coreclr.sln",
-  "app.asar.unpacked/node_modules/edge-cs/src/Edge.js.CSharp/edge-cs-coreclr.xproj",
   "app.asar.unpacked/node_modules/edge-cs/src/Edge.js.CSharp/gulpfile.js",
   "app.asar.unpacked/node_modules/edge-cs/src/Edge.js.CSharp/package.json",
   "app.asar.unpacked/node_modules/edge-cs/src/Edge.js.CSharp/project.json",

test/src/BuildTest.ts

@@ -170,8 +170,9 @@ test.ifLinuxOrDevMac("beforeBuild", () => {
   })
 })
 
-test.ifDevOrLinuxCi("smart unpack", app({
-  targets: linuxDirTarget,
+// https://github.com/electron-userland/electron-builder/issues/1738
+test.ifAll.ifDevOrLinuxCi("win smart unpack", app({
+  targets: Platform.WINDOWS.createTarget(DIR_TARGET),
 }, {
   installDepsBefore: true,
   projectDirCreated: packageJson(it => {
@@ -182,16 +183,31 @@ test.ifDevOrLinuxCi("smart unpack", app({
       "@electron-builder/test-smart-unpack-empty": "1.0.0",
     }
   }),
-  packed: async context => {
-    const resourceDir = context.getResources(Platform.LINUX)
-    expect(await readAsarJson(path.join(resourceDir, "app.asar"), "node_modules/debug/package.json")).toMatchObject({
-      name: "debug"
-    })
-
-    expect((await walk(resourceDir, file => !path.basename(file).startsWith("."))).map(it => it.substring(resourceDir.length + 1))).toMatchSnapshot()
-  }
+  packed: context => verifySmartUnpack(context.getResources(Platform.WINDOWS))
 }))
 
+async function verifySmartUnpack(resourceDir: string) {
+  expect(await readAsarJson(path.join(resourceDir, "app.asar"), "node_modules/debug/package.json")).toMatchObject({
+    name: "debug"
+  })
+
+  expect((await walk(resourceDir, file => !path.basename(file).startsWith("."))).map(it => it.substring(resourceDir.length + 1))).toMatchSnapshot()
+}
+
+// https://github.com/electron-userland/electron-builder/issues/1738
+test.ifAll.ifDevOrLinuxCi("posix smart unpack", app({
+  targets: linuxDirTarget,
+}, {
+  installDepsBefore: true,
+  projectDirCreated: packageJson(it => {
+    it.dependencies = {
+      "debug": "^2.2.0",
+      "edge-cs": "1.2.1",
+      "lzma-native": "2.0.3",
+    }
+  }),
+  packed: context => verifySmartUnpack(context.getResources(Platform.LINUX))}))
+
 test("wine version", async () => {
   await checkWineVersion(BluebirdPromise.resolve("1.9.23 (Staging)"))
   await checkWineVersion(BluebirdPromise.resolve("2.0-rc2"))

test/src/helpers/packTester.ts

@@ -97,7 +97,8 @@ export async function assertPack(fixtureName: string, packagerOptions: PackagerO
     if (projectDirCreated != null) {
       await projectDirCreated(projectDir)
       if (checkOptions.installDepsBefore) {
-        await spawn("node", [path.join(__dirname, "..", "..", "vendor", "yarn.js"), "install", "--production", "--no-bin-links"], {
+        // bin links required (e.g. for node-pre-gyp - if package refers to it in the install script)
+        await spawn("node", [path.join(__dirname, "..", "..", "vendor", "yarn.js"), "install", "--production", "--no-lockfile"], {
           cwd: projectDir,
         })
       }

yarn.lock

@@ -26,7 +26,7 @@
   version "1.3.29"
   resolved "https://registry.yarnpkg.com/@types/ini/-/ini-1.3.29.tgz#1325e981e047d40d13ce0359b821475b97741d2f"
 
-"@types/jest@^20.0.1":
+"@types/jest@^20.0.2":
   version "20.0.2"
   resolved "https://registry.yarnpkg.com/@types/jest/-/jest-20.0.2.tgz#86c751121fb53dbd39bb1a08c45083da13f2dc67"
 
@@ -44,9 +44,11 @@
   dependencies:
     "@types/node" "*"
 
-"@types/xml2js@^0.0.33":
-  version "0.0.33"
-  resolved "https://registry.yarnpkg.com/@types/xml2js/-/xml2js-0.0.33.tgz#20c5dd6460245284d64a55690015b95e409fb7de"
+"@types/xml2js@^0.4.0":
+  version "0.4.0"
+  resolved "https://registry.yarnpkg.com/@types/xml2js/-/xml2js-0.4.0.tgz#e54a89a0055d5ed69305b2610f970909bf363e45"
+  dependencies:
+    "@types/node" "*"
 
 abab@^1.0.3:
   version "1.0.3"
@@ -526,10 +528,14 @@ balanced-match@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767"
 
-[email protected], base64-js@^1.0.2:
+[email protected]:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.2.0.tgz#a39992d723584811982be5e290bb6a53d86700f1"
 
+base64-js@^1.0.2:
+  version "1.2.1"
+  resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.2.1.tgz#a91947da1f4a516ea38e5b4ec0ec3773675e0886"
+
 bcrypt-pbkdf@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.1.tgz#63bc5dcb61331b92bc05fd528953c33462a06f8d"