feat: Allow usage of environmental variables in extraFiles and extraR…

…esources

Close electron-userland#1307

package.json

@@ -18,12 +18,13 @@
     "update-deps": "lerna exec -- npm-check-updates --reject 'electron-builder-http,electron-builder-util,electron-builder-core,electron-publish' -a",
     "set-versions": "node test/out/helpers/setVersions.js",
     "npm-publish": "yarn compile && ./packages/npm-publish.sh && conventional-changelog -p angular -i CHANGELOG.md -s",
-    "schema": "typescript-json-schema packages/electron-builder/tsconfig.json Config --out packages/electron-builder/scheme.json --noExtraProps",
+    "schema": "typescript-json-schema packages/electron-builder/tsconfig.json Config --out packages/electron-builder/scheme.json --noExtraProps --useTypeOfKeyword --strictNullChecks",
     "jsdoc": "ts2jsdoc packages/electron-updater packages/electron-builder-http",
     "docs": "rm -rf jsdoc/out/docs && jsdoc -R jsdoc/readme.md -c jsdoc/jsdoc.json -d jsdoc/out/docs"
   },
   "//": "repository must be specified otherwise conventional-changelog will use forked repo (currently cloned)",
   "repository": "electron-userland/electron-builder",
+  "///": "all dependencies for all packages (hoisted)",
   "dependencies": {
     "7zip-bin": "^2.0.4",
     "ajv": "^5.0.2-beta",
@@ -61,7 +62,7 @@
     "yargs": "^6.6.0"
   },
   "devDependencies": {
-    "@develar/typescript-json-schema": "0.9.1",
+    "@develar/typescript-json-schema": "0.9.3",
     "@types/electron": "^1.4.32",
     "@types/ini": "^1.3.29",
     "@types/jest": "^18.1.1",
@@ -90,7 +91,6 @@
     "ts-babel": "^1.4.4",
     "tslint": "^4.4.2",
     "typescript": "^2.2.1",
-    "typescript-json-schema": "^0.9.0",
     "whitespace": "^2.1.0",
     "xml2js": "^0.4.17"
   },

packages/electron-builder/src/fileMatcher.ts

@@ -5,7 +5,6 @@ import { warn } from "electron-builder-util/out/log"
 import { mkdirs } from "fs-extra-p"
 import { Minimatch } from "minimatch"
 import * as path from "path"
-import { Macros } from "./metadata"
 import { createFilter, hasMagic } from "./util/filter"
 
 export class FileMatcher {
@@ -14,14 +13,14 @@ export class FileMatcher {
 
   readonly patterns: Array<string>
 
-  constructor(from: string, to: string, private options: Macros, patterns?: Array<string> | string | n) {
-    this.from = this.expandPattern(from)
-    this.to = this.expandPattern(to)
-    this.patterns = asArray(patterns).map(it => path.posix.normalize(it))
+  constructor(from: string, to: string, private readonly macroExpander: (pattern: string) => string, patterns?: Array<string> | string | n) {
+    this.from = macroExpander(from)
+    this.to = macroExpander(to)
+    this.patterns = asArray(patterns).map(it => path.posix.normalize(macroExpander(it)))
   }
 
   addPattern(pattern: string) {
-    this.patterns.push(path.posix.normalize((pattern)))
+    this.patterns.push(path.posix.normalize(this.macroExpander(pattern)))
   }
 
   addAllPattern() {
@@ -49,19 +48,18 @@ export class FileMatcher {
       return
     }
 
-    for (const p of this.patterns) {
-      let expandedPattern = this.expandPattern(p)
+    for (let pattern of this.patterns) {
       if (relativeFrom != null) {
-        expandedPattern = path.join(relativeFrom, expandedPattern)
+        pattern = path.join(relativeFrom, pattern)
       }
 
-      const parsedPattern = new Minimatch(expandedPattern, minimatchOptions)
+      const parsedPattern = new Minimatch(pattern, minimatchOptions)
       result.push(parsedPattern)
 
       if (!hasMagic(parsedPattern)) {
         // https://github.com/electron-userland/electron-builder/issues/545
         // add **/*
-        result.push(new Minimatch(`${expandedPattern}/**/*`, minimatchOptions))
+        result.push(new Minimatch(`${pattern}/**/*`, minimatchOptions))
       }
     }
   }
@@ -71,13 +69,6 @@ export class FileMatcher {
     this.computeParsedPatterns(parsedPatterns)
     return createFilter(this.from, parsedPatterns, ignoreFiles, rawFilter, excludePatterns)
   }
-
-  private expandPattern(pattern: string): string {
-    return pattern
-      .replace(/\$\{arch}/g, this.options.arch!)
-      .replace(/\$\{os}/g, this.options.os)
-      .replace(/\$\{\/\*}/g, "{,/**/*}")
-  }
 }
 
 export function copyFiles(patterns: Array<FileMatcher> | null): Promise<any> {

packages/electron-builder/src/metadata.ts

@@ -341,11 +341,6 @@ export interface PlatformSpecificBuildOptions extends TargetSpecificOptions {
   readonly publish?: Publish
 }
 
-export interface Macros {
-  os: string
-  arch: string | null
-}
-
 export function getPlatformIconFileName(value: string | null | undefined, isMac: boolean) {
   if (value === undefined) {
     return undefined

packages/electron-builder/src/platformPackager.ts

@@ -10,7 +10,7 @@ import * as path from "path"
 import { AppInfo } from "./appInfo"
 import { checkFileInArchive, createAsarArchive } from "./asarUtil"
 import { copyFiles, FileMatcher } from "./fileMatcher"
-import { AsarOptions, Config, FileAssociation, FilePattern, Macros, PlatformSpecificBuildOptions } from "./metadata"
+import { AsarOptions, Config, FileAssociation, FilePattern, PlatformSpecificBuildOptions } from "./metadata"
 import { unpackElectron } from "./packager/dirPackager"
 import { BuildInfo, PackagerOptions } from "./packagerApi"
 import { readInstalled } from "./readInstalled"
@@ -104,14 +104,14 @@ export abstract class PlatformPackager<DC extends PlatformSpecificBuildOptions>
       .then(() => BluebirdPromise.each(targets, it => it.isAsyncSupported ? null : it.build(appOutDir, arch))))
   }
 
-  private getExtraFileMatchers(isResources: boolean, appOutDir: string, fileMatchOptions: Macros, customBuildOptions: DC): Array<FileMatcher> | null {
+  private getExtraFileMatchers(isResources: boolean, appOutDir: string, macroExpander: (pattern: string) => string, customBuildOptions: DC): Array<FileMatcher> | null {
     const base = isResources ? this.getResourcesDir(appOutDir) : (this.platform === Platform.MAC ? path.join(appOutDir, `${this.appInfo.productFilename}.app`, "Contents") : appOutDir)
-    return this.getFileMatchers(isResources ? "extraResources" : "extraFiles", this.projectDir, base, true, fileMatchOptions, customBuildOptions)
+    return this.getFileMatchers(isResources ? "extraResources" : "extraFiles", this.projectDir, base, true, macroExpander, customBuildOptions)
   }
 
-  private createFileMatcher(appDir: string, resourcesPath: string, fileMatchOptions: Macros, platformSpecificBuildOptions: DC) {
-    const patterns = this.info.isPrepackedAppAsar ? null : this.getFileMatchers("files", appDir, path.join(resourcesPath, "app"), false, fileMatchOptions, platformSpecificBuildOptions)
-    const matcher = patterns == null ? new FileMatcher(appDir, path.join(resourcesPath, "app"), fileMatchOptions) : patterns[0]
+  private createFileMatcher(appDir: string, resourcesPath: string, macroExpander: (pattern: string) => string, platformSpecificBuildOptions: DC) {
+    const patterns = this.info.isPrepackedAppAsar ? null : this.getFileMatchers("files", appDir, path.join(resourcesPath, "app"), false, macroExpander, platformSpecificBuildOptions)
+    const matcher = patterns == null ? new FileMatcher(appDir, path.join(resourcesPath, "app"), macroExpander) : patterns[0]
     if (matcher.isEmpty() || matcher.containsOnlyIgnore()) {
       matcher.addAllPattern()
     }
@@ -141,13 +141,9 @@ export abstract class PlatformPackager<DC extends PlatformSpecificBuildOptions>
     }
 
     const asarOptions = await this.computeAsarOptions(platformSpecificBuildOptions)
-    const fileMatchOptions: Macros = {
-      arch: Arch[arch],
-      os: this.platform.buildConfigurationKey
-    }
-
-    const extraResourceMatchers = this.getExtraFileMatchers(true, appOutDir, fileMatchOptions, platformSpecificBuildOptions)
-    const extraFileMatchers = this.getExtraFileMatchers(false, appOutDir, fileMatchOptions, platformSpecificBuildOptions)
+    const macroExpander = (it: string) => this.expandMacro(it, arch, {"/*": "{,/**/*}"})
+    const extraResourceMatchers = this.getExtraFileMatchers(true, appOutDir, macroExpander, platformSpecificBuildOptions)
+    const extraFileMatchers = this.getExtraFileMatchers(false, appOutDir, macroExpander, platformSpecificBuildOptions)
 
     const resourcesPath = this.platform === Platform.MAC ? path.join(appOutDir, "Electron.app", "Contents", "Resources") : path.join(appOutDir, "resources")
 
@@ -188,7 +184,7 @@ export abstract class PlatformPackager<DC extends PlatformSpecificBuildOptions>
       }
     }
 
-    const defaultMatcher = this.createFileMatcher(appDir, resourcesPath, fileMatchOptions, platformSpecificBuildOptions)
+    const defaultMatcher = this.createFileMatcher(appDir, resourcesPath, macroExpander, platformSpecificBuildOptions)
     const filter = defaultMatcher.createFilter(ignoreFiles, rawFilter, excludePatterns.length > 0 ? excludePatterns : null)
     let promise
     if (this.info.isPrepackedAppAsar) {
@@ -198,7 +194,7 @@ export abstract class PlatformPackager<DC extends PlatformSpecificBuildOptions>
       promise = copyDir(appDir, path.join(resourcesPath, "app"), filter)
     }
     else {
-      const unpackPattern = this.getFileMatchers("asarUnpack", appDir, path.join(resourcesPath, "app"), false, fileMatchOptions, platformSpecificBuildOptions)
+      const unpackPattern = this.getFileMatchers("asarUnpack", appDir, path.join(resourcesPath, "app"), false, macroExpander, platformSpecificBuildOptions)
       const fileMatcher = unpackPattern == null ? null : unpackPattern[0]
       promise = createAsarArchive(appDir, resourcesPath, asarOptions, filter, fileMatcher == null ? null : fileMatcher.createFilter())
     }
@@ -279,11 +275,11 @@ export abstract class PlatformPackager<DC extends PlatformSpecificBuildOptions>
     return deepAssign({}, result, defaultOptions)
   }
 
-  private getFileMatchers(name: "files" | "extraFiles" | "extraResources" | "asarUnpack", defaultSrc: string, defaultDest: string, allowAdvancedMatching: boolean, fileMatchOptions: Macros, customBuildOptions: DC): Array<FileMatcher> | null {
+  private getFileMatchers(name: "files" | "extraFiles" | "extraResources" | "asarUnpack", defaultSrc: string, defaultDest: string, allowAdvancedMatching: boolean, macroExpander: (pattern: string) => string, customBuildOptions: DC): Array<FileMatcher> | null {
     const globalPatterns: Array<string | FilePattern> | string | n | FilePattern = (<any>this.config)[name]
     const platformSpecificPatterns: Array<string | FilePattern> | string | n = (<any>customBuildOptions)[name]
 
-    const defaultMatcher = new FileMatcher(defaultSrc, defaultDest, fileMatchOptions)
+    const defaultMatcher = new FileMatcher(defaultSrc, defaultDest, macroExpander)
     const fileMatchers: Array<FileMatcher> = []
 
     function addPatterns(patterns: Array<string | FilePattern> | string | n | FilePattern) {
@@ -306,7 +302,7 @@ export abstract class PlatformPackager<DC extends PlatformSpecificBuildOptions>
         else if (allowAdvancedMatching) {
           const from = pattern.from == null ? defaultSrc : path.resolve(defaultSrc, pattern.from)
           const to = pattern.to == null ? defaultDest : path.resolve(defaultDest, pattern.to)
-          fileMatchers.push(new FileMatcher(from, to, fileMatchOptions, pattern.filter))
+          fileMatchers.push(new FileMatcher(from, to, macroExpander, pattern.filter))
         }
         else {
           throw new Error(`Advanced file copying not supported for "${name}"`)
@@ -393,16 +389,22 @@ export abstract class PlatformPackager<DC extends PlatformSpecificBuildOptions>
     if (pattern == null) {
       pattern = this.platformSpecificBuildOptions.artifactName || this.config.artifactName || defaultPattern || "${productName}-${version}.${ext}"
     }
+    return this.expandMacro(pattern, arch, {
+      ext: ext
+    })
+  }
 
+  expandMacro(pattern: string, arch: Arch | n, extra: any = {}): string {
     if (arch == null) {
       pattern = pattern
         .replace("-${arch}", "")
         .replace(" ${arch}", "")
         .replace("_${arch}", "")
+        .replace("/${arch}", "")
     }
 
     const appInfo = this.appInfo
-    return pattern.replace(/\$\{([a-zA-Z]+)\}/g, (match, p1): string => {
+    return pattern.replace(/\$\{([_a-zA-Z./*]+)\}/g, (match, p1): string => {
       switch (p1) {
         case "name":
           return appInfo.name
@@ -423,11 +425,23 @@ export abstract class PlatformPackager<DC extends PlatformSpecificBuildOptions>
         case "os":
           return this.platform.buildConfigurationKey
 
-        case "ext":
-          return ext
-
         default:
-          throw new Error(`Macro ${p1} is not defined`)
+          if (p1.startsWith("env.")) {
+            const envName = p1.substring("env.".length)
+            const envValue = process.env[envName]
+            if (envValue == null) {
+              throw new Error(`Env ${envName} is not defined`)
+            }
+            return envValue
+          }
+
+          const value = extra[p1]
+          if (value == null) {
+            throw new Error(`Macro ${p1} is not defined`)
+          }
+          else {
+            return value
+          }
       }
     })
   }

packages/electron-builder/src/publish/PublishManager.ts

@@ -15,7 +15,7 @@ import isCi from "is-ci"
 import { safeDump } from "js-yaml"
 import * as path from "path"
 import * as url from "url"
-import { Macros, PlatformSpecificBuildOptions } from "../metadata"
+import { PlatformSpecificBuildOptions } from "../metadata"
 import { Packager } from "../packager"
 import { ArtifactCreated, BuildInfo } from "../packagerApi"
 import { PlatformPackager } from "../platformPackager"
@@ -28,7 +28,6 @@ export class PublishManager implements PublishContext {
   readonly publishTasks: Array<Promise<any>> = []
   private readonly errors: Array<Error> = []
 
-  private isPublishOptionGuessed = false
   private isPublish = false
 
   readonly progress = (<NodeJS.WritableStream>process.stdout).isTTY ? new MultiProgress() : null
@@ -44,12 +43,10 @@ export class PublishManager implements PublishContext {
           if (tag != null) {
             log(`Tag ${tag} is defined, so artifacts will be published`)
             publishOptions.publish = "onTag"
-            this.isPublishOptionGuessed = true
           }
           else if (isCi) {
             log("CI detected, so artifacts will be published if draft release exists")
             publishOptions.publish = "onTagOrDraft"
-            this.isPublishOptionGuessed = true
           }
         }
       }
@@ -68,14 +65,16 @@ export class PublishManager implements PublishContext {
       }
 
       const packager = event.packager
-      const publishConfigs = await getPublishConfigsForUpdateInfo(packager, await getPublishConfigs(packager, null, false))
+      const publishConfigs = await getPublishConfigsForUpdateInfo(packager, await getPublishConfigs(packager, null))
       if (publishConfigs == null || publishConfigs.length === 0) {
         return
       }
 
       let publishConfig = publishConfigs[0]
       if ((<GenericServerOptions>publishConfig).url != null) {
-        publishConfig = Object.assign({}, publishConfig, {url: expandPattern((<GenericServerOptions>publishConfig).url, {os: packager.platform.buildConfigurationKey, arch: packager.platform === Platform.WINDOWS ? null : Arch[Arch.x64]})})
+        publishConfig = Object.assign({}, publishConfig, {
+          url: packager.expandMacro((<GenericServerOptions>publishConfig).url, packager.platform === Platform.WINDOWS ? null : Arch.x64)
+        })
       }
 
       if (packager.platform === Platform.WINDOWS) {
@@ -94,7 +93,7 @@ export class PublishManager implements PublishContext {
   private async artifactCreated(event: ArtifactCreated) {
     const packager = event.packager
     const target = event.target
-    const publishConfigs = event.publishConfig == null ? await getPublishConfigs(packager, target == null ? null : (<any>packager.config)[target.name], !this.isPublishOptionGuessed) : [event.publishConfig]
+    const publishConfigs = event.publishConfig == null ? await getPublishConfigs(packager, target == null ? null : (<any>packager.config)[target.name]) : [event.publishConfig]
 
     const eventFile = event.file
     if (publishConfigs == null) {
@@ -228,10 +227,7 @@ async function writeUpdateInfo(event: ArtifactCreated, _publishConfigs: Array<Pu
       await (<any>outputJson)(updateInfoFile, <VersionInfo>{
         version: version,
         releaseDate: new Date().toISOString(),
-        url: computeDownloadUrl(publishConfig, packager.generateName2("zip", "mac", isGitHub), version, {
-          os: Platform.MAC.buildConfigurationKey,
-          arch: Arch[Arch.x64]
-        }),
+        url: computeDownloadUrl(publishConfig, packager.generateName2("zip", "mac", isGitHub), packager, null),
       }, {spaces: 2})
 
       packager.info.dispatchArtifactCreated({
@@ -301,9 +297,9 @@ export function createPublisher(context: PublishContext, version: string, publis
   return null
 }
 
-export function computeDownloadUrl(publishConfig: PublishConfiguration, fileName: string | null, version: string, macros: Macros) {
+export function computeDownloadUrl(publishConfig: PublishConfiguration, fileName: string | null, packager: PlatformPackager<any>, arch: Arch | null) {
   if (publishConfig.provider === "generic") {
-    const baseUrlString = expandPattern((<GenericServerOptions>publishConfig).url, macros)
+    const baseUrlString = packager.expandMacro((<GenericServerOptions>publishConfig).url, arch)
     if (fileName == null) {
       return baseUrlString
     }
@@ -318,7 +314,7 @@ export function computeDownloadUrl(publishConfig: PublishConfiguration, fileName
   }
   else {
     const gh = <GithubOptions>publishConfig
-    baseUrl = `${githubUrl(gh)}/${gh.owner}/${gh.repo}/releases/download/v${version}`
+    baseUrl = `${githubUrl(gh)}/${gh.owner}/${gh.repo}/releases/download/v${packager.appInfo.version}`
   }
 
   if (fileName == null) {
@@ -327,25 +323,7 @@ export function computeDownloadUrl(publishConfig: PublishConfiguration, fileName
   return `${baseUrl}/${encodeURI(fileName)}`
 }
 
-function expandPattern(pattern: string, macros: Macros): string {
-  const arch = macros.arch
-  if (arch == null) {
-    pattern = pattern
-      .replace("-${arch}", "")
-      .replace(" ${arch}", "")
-      .replace("_${arch}", "")
-      .replace("/${arch}", "")
-  }
-
-  pattern = pattern.replace(/\$\{os}/g, macros.os)
-  if (arch != null) {
-    pattern = pattern.replace(/\$\{arch}/g, arch)
-  }
-
-  return pattern
-}
-
-export async function getPublishConfigs(packager: PlatformPackager<any>, targetSpecificOptions: PlatformSpecificBuildOptions | null | undefined, errorIfCannot: boolean = true): Promise<Array<PublishConfiguration> | null> {
+export async function getPublishConfigs(packager: PlatformPackager<any>, targetSpecificOptions: PlatformSpecificBuildOptions | null | undefined): Promise<Array<PublishConfiguration> | null> {
   let publishers
 
   // check build.nsis (target)
@@ -419,4 +397,4 @@ function isPullRequest() {
   }
 
   return isSet(process.env.TRAVIS_PULL_REQUEST) || isSet(process.env.CI_PULL_REQUEST) || isSet(process.env.CI_PULL_REQUESTS)
-}
+}

packages/electron-builder/src/targets/WebInstaller.ts

@@ -1,4 +1,3 @@
-import { Platform } from "electron-builder-core"
 import { NsisWebOptions } from "../options/winOptions"
 import { computeDownloadUrl, getPublishConfigs, getPublishConfigsForUpdateInfo } from "../publish/PublishManager"
 import { WinPackager } from "../winPackager"
@@ -27,10 +26,7 @@ export default class WebInstallerTarget extends NsisTarget {
         throw new Error("Cannot compute app package download URL")
       }
 
-      appPackageUrl = computeDownloadUrl(publishConfigs[0], null, packager.appInfo.version, {
-        os: Platform.WINDOWS.buildConfigurationKey,
-        arch: ""
-      })
+      appPackageUrl = computeDownloadUrl(publishConfigs[0], null, packager, null)
 
       defines.APP_PACKAGE_URL_IS_INCOMLETE = null
     }

test/out/__snapshots__/filesTest.js.snap

@@ -1,5 +1,7 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
+exports[`expand not defined env 1`] = `"Env FOO_NOT_DEFINED is not defined"`;
+
 exports[`extraResources on Linux and Windows 1`] = `
 Array [
   "RELEASES",