feat(config): add support for varFiles

This allows users to include a `garden.env` file for project-global variables, and `garden.<env-name>.env` files for environment-specific variables. You may also override the file paths for each of those. These variables are made available in `${var.<key>}` template keys, same as variables specified directly in the project garden.yml. A full documentation guide for variables and templating will follow in a separate PR.
garden-io · Sep 9, 2019 · e2ade31 · e2ade31
1 parent 4f5fabc
commit e2ade31
Show file tree

Hide file tree

Showing 14 changed files with 582 additions and 22 deletions.
diff --git a/docs/reference/config.md b/docs/reference/config.md
@@ -132,6 +132,21 @@ environmentDefaults:
       - stage
 ```
 
+### `environmentDefaults.varFile`
+
+[environmentDefaults](#environmentdefaults) > varFile
+
+Specify a path (relative to the project root) to a file containing variables, that we apply on top of the
+_environment-specific_ `variables` field. The file should be in a standard "dotenv" format, specified
+[here](https://github.com/motdotla/dotenv#rules).
+
+If you don't set the field and the `garden.<env-name>.env` file does not exist,
+we simply ignore it. If you do override the default value and the file doesn't exist, an error will be thrown.
+
+| Type     | Required | Default                   |
+| -------- | -------- | ------------------------- |
+| `string` | No       | `"garden.<env-name>.env"` |
+
 ### `environmentDefaults.variables`
 
 [environmentDefaults](#environmentdefaults) > variables
@@ -142,20 +157,6 @@ A key/value map of variables that modules can reference when using this environm
 | -------- | -------- | ------- |
 | `object` | No       | `{}`    |
 
-### `environments`
-
-A list of environments to configure for the project.
-
-| Type                            | Required |
-| ------------------------------- | -------- |
-| `array[object] | array[string]` | No       |
-
-Example:
-
-```yaml
-environments: [{"name":"local","providers":[{"name":"local-kubernetes","environments":[]}],"variables":{}}]
-```
-
 ### `modules`
 
 | Type     | Required |
@@ -287,6 +288,22 @@ sources:
   - repositoryUrl: "git+https://github.com/org/repo.git#v2.0"
 ```
 
+### `varFile`
+
+Specify a path (relative to the project root) to a file containing variables, that we apply on top of the
+project-wide `variables` field. The file should be in a standard "dotenv" format, specified
+[here](https://github.com/motdotla/dotenv#rules).
+
+If you don't set the field and the `garden.env` file does not exist, we simply ignore it.
+If you do override the default value and the file doesn't exist, an error will be thrown.
+
+_Note that in many cases it is advisable to only use environment-specific var files, instead of combining
+multiple ones. See the `environments[].varFile` field for this option._
+
+| Type     | Required | Default        |
+| -------- | -------- | -------------- |
+| `string` | No       | `"garden.env"` |
+
 ### `variables`
 
 Variables to configure for all environments.
@@ -295,6 +312,95 @@ Variables to configure for all environments.
 | -------- | -------- | ------- |
 | `object` | No       | `{}`    |
 
+### `environments`
+
+| Type            | Required |
+| --------------- | -------- |
+| `array[object]` | No       |
+
+### `environments[].providers[]`
+
+[environments](#environments) > providers
+
+DEPRECATED - Please use the top-level `providers` field instead, and if needed use the `environments` key on the provider configurations to limit them to specific environments.
+
+| Type            | Required | Default |
+| --------------- | -------- | ------- |
+| `array[object]` | No       | `[]`    |
+
+### `environments[].providers[].name`
+
+[environments](#environments) > [providers](#environments[].providers[]) > name
+
+The name of the provider plugin to use.
+
+| Type     | Required |
+| -------- | -------- |
+| `string` | Yes      |
+
+Example:
+
+```yaml
+environments:
+  - providers:
+      - name: "local-kubernetes"
+```
+
+### `environments[].providers[].environments[]`
+
+[environments](#environments) > [providers](#environments[].providers[]) > environments
+
+If specified, this provider will only be used in the listed environments. Note that an empty array effectively disables the provider. To use a provider in all environments, omit this field.
+
+| Type            | Required |
+| --------------- | -------- |
+| `array[string]` | No       |
+
+Example:
+
+```yaml
+environments:
+  - providers:
+      - environments:
+        - dev
+        - stage
+```
+
+### `environments[].varFile`
+
+[environments](#environments) > varFile
+
+Specify a path (relative to the project root) to a file containing variables, that we apply on top of the
+_environment-specific_ `variables` field. The file should be in a standard "dotenv" format, specified
+[here](https://github.com/motdotla/dotenv#rules).
+
+If you don't set the field and the `garden.<env-name>.env` file does not exist,
+we simply ignore it. If you do override the default value and the file doesn't exist, an error will be thrown.
+
+| Type     | Required | Default                   |
+| -------- | -------- | ------------------------- |
+| `string` | No       | `"garden.<env-name>.env"` |
+
+### `environments[].variables`
+
+[environments](#environments) > variables
+
+A key/value map of variables that modules can reference when using this environment. These take precedence over variables defined in the top-level `variables` field.
+
+| Type     | Required | Default |
+| -------- | -------- | ------- |
+| `object` | No       | `{}`    |
+
+### `environments[].name`
+
+[environments](#environments) > name
+
+The name of the environment.
+
+| Type     | Required |
+| -------- | -------- |
+| `string` | Yes      |
+
 
 ## Project YAML schema
 ```yaml
@@ -309,8 +415,8 @@ environmentDefaults:
   providers:
     - name:
       environments:
+  varFile: garden.<env-name>.env
   variables: {}
-environments:
 modules:
   include:
   exclude:
@@ -320,7 +426,15 @@ providers:
 sources:
   - name:
     repositoryUrl:
+varFile: garden.env
 variables: {}
+environments:
+  - providers:
+      - name:
+        environments:
+  - varFile: garden.<env-name>.env
+    variables: {}
+    name:
 ```
 
 ## Module configuration keys

diff --git a/garden-service/package-lock.json b/garden-service/package-lock.json
diff --git a/garden-service/package.json b/garden-service/package.json
@@ -52,6 +52,7 @@
     "deline": "^1.0.4",
     "directory-tree": "^2.2.4",
     "dockerode": "^2.5.8",
+    "dotenv": "^8.1.0",
     "elegant-spinner": "^2.0.0",
     "escape-string-regexp": "^2.0.0",
     "eventemitter2": "^5.0.1",
@@ -125,6 +126,7 @@
     "@types/dedent": "^0.7.0",
     "@types/deep-diff": "1.0.0",
     "@types/dockerode": "^2.5.20",
+    "@types/dotenv": "^6.1.1",
     "@types/fs-extra": "^8.0.0",
     "@types/hapi__joi": "^15.0.4",
     "@types/has-ansi": "^3.0.0",

diff --git a/garden-service/src/config/project.ts b/garden-service/src/config/project.ts
@@ -6,6 +6,7 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  */
 
+import dotenv = require("dotenv")
 import { safeDump } from "js-yaml"
 import { apply, merge } from "json-merge-patch"
 import { deline, dedent } from "../util/string"
@@ -30,6 +31,11 @@ import { cloneDeep, omit } from "lodash"
 import { providerConfigBaseSchema, Provider, ProviderConfig } from "./provider"
 import { DEFAULT_API_VERSION } from "../constants"
 import { defaultDotIgnoreFiles } from "../util/fs"
+import { pathExists, readFile } from "fs-extra"
+import { resolve } from "path"
+
+export const defaultVarFilePath = "garden.env"
+export const defaultEnvVarFilePath = (environmentName: string) => `garden.${environmentName}.env`
 
 export interface CommonEnvironmentConfig {
   providers?: ProviderConfig[]  // further validated by each plugin
@@ -45,6 +51,19 @@ export const environmentConfigSchema = joi.object()
         DEPRECATED - Please use the top-level \`providers\` field instead, and if needed use the \`environments\` key
         on the provider configurations to limit them to specific environments.
       `),
+    varFile: joi.string()
+      .posixPath()
+      .default(
+        (context: any) => defaultEnvVarFilePath(context.name || "<env-name>"), defaultEnvVarFilePath("<env-name>"),
+      )
+      .description(dedent`
+        Specify a path (relative to the project root) to a file containing variables, that we apply on top of the
+        _environment-specific_ \`variables\` field. The file should be in a standard "dotenv" format, specified
+        [here](https://github.com/motdotla/dotenv#rules).
+
+        If you don't set the field and the \`${defaultEnvVarFilePath("<env-name>")}\` file does not exist,
+        we simply ignore it. If you do override the default value and the file doesn't exist, an error will be thrown.
+      `),
     variables: joiVariables()
       .description(deline`
         A key/value map of variables that modules can reference when using this environment. These take precedence
@@ -54,6 +73,7 @@ export const environmentConfigSchema = joi.object()
 
 export interface EnvironmentConfig extends CommonEnvironmentConfig {
   name: string
+  varFile?: string
 }
 
 export interface Environment extends EnvironmentConfig {
@@ -64,7 +84,7 @@ export const environmentNameSchema = joiUserIdentifier()
   .required()
   .description("The name of the environment.")
 
-const environmentSchema = environmentConfigSchema
+export const environmentSchema = environmentConfigSchema
   .keys({
     name: environmentNameSchema,
   })
@@ -109,6 +129,7 @@ export interface ProjectConfig {
   },
   providers: ProviderConfig[]
   sources?: SourceConfig[]
+  varFile?: string
   variables: PrimitiveMap
 }
 
@@ -125,6 +146,7 @@ export const defaultEnvironments: EnvironmentConfig[] = [
         environments: [],
       },
     ],
+    varFile: defaultEnvVarFilePath("local"),
     variables: {},
   },
 ]
@@ -214,6 +236,20 @@ export const projectSchema = joi.object()
         "Please refer to individual plugins/providers for details on how to configure them.",
       ),
     sources: projectSourcesSchema,
+    varFile: joi.string()
+      .posixPath()
+      .default(defaultVarFilePath)
+      .description(dedent`
+        Specify a path (relative to the project root) to a file containing variables, that we apply on top of the
+        project-wide \`variables\` field. The file should be in a standard "dotenv" format, specified
+        [here](https://github.com/motdotla/dotenv#rules).
+
+        If you don't set the field and the \`garden.env\` file does not exist, we simply ignore it.
+        If you do override the default value and the file doesn't exist, an error will be thrown.
+
+        _Note that in many cases it is advisable to only use environment-specific var files, instead of combining
+        multiple ones. See the \`environments[].varFile\` field for this option._
+      `),
     variables: joiVariables()
       .description("Variables to configure for all environments."),
   })
@@ -240,6 +276,7 @@ export async function resolveProjectConfig(config: ProjectConfig): Promise<Proje
       environmentDefaults: { variables: {}, ...environmentDefaults || {}, providers: <ProviderConfig[]>[] },
       name: config.name,
       sources: config.sources,
+      varFile: config.varFile,
       variables: config.variables,
       environments: environments.map(e => omit(e, ["providers"])),
     },
@@ -317,6 +354,8 @@ export async function resolveProjectConfig(config: ProjectConfig): Promise<Proje
  *
  * For project variables, we apply the variables specified to the selected environment on the global variables
  * specified on the top-level `variables` key using a JSON Merge Patch (https://tools.ietf.org/html/rfc7396).
+ * We also attempt to load the configured varFiles, and include those in the merge. The precedence order is as follows:
+ *   environment.varFile > environment.variables > project.varFile > project.variables
  *
  * For provider configuration, we filter down to the providers that are enabled for all environments (no `environments`
  * key specified) and those that explicitly list the specified environments. Then we merge any provider configs with
@@ -331,7 +370,7 @@ export async function resolveProjectConfig(config: ProjectConfig): Promise<Proje
  * @param config a resolved project config (as returned by `resolveProjectConfig()`)
  * @param environmentName the name of the environment to use
  */
-export function pickEnvironment(config: ProjectConfig, environmentName: string) {
+export async function pickEnvironment(config: ProjectConfig, environmentName: string) {
   const { environments, name: projectName } = config
 
   const environmentConfig = findByName(environments, environmentName)
@@ -361,10 +400,45 @@ export function pickEnvironment(config: ProjectConfig, environmentName: string)
     }
   }
 
-  const variables: PrimitiveMap = <any>merge(config.variables, environmentConfig.variables)
+  const projectVarFileVars = await loadVarFile(
+    config.path, config.varFile, defaultVarFilePath,
+  )
+  const envVarFileVars = await loadVarFile(
+    config.path, environmentConfig.varFile, defaultEnvVarFilePath(environmentName),
+  )
+
+  const variables: PrimitiveMap = <any>merge(
+    merge(config.variables, projectVarFileVars),
+    merge(environmentConfig.variables, envVarFileVars),
+  )
 
   return {
     providers: Object.values(mergedProviders),
     variables,
   }
 }
+
+async function loadVarFile(projectRoot: string, path: string | undefined, defaultPath: string): Promise<PrimitiveMap> {
+  const resolvedPath = resolve(projectRoot, path || defaultPath)
+  const exists = await pathExists(resolvedPath)
+
+  if (!exists && path && path !== defaultPath) {
+    throw new ConfigurationError(`Could not find varFile at path '${path}'`, {
+      path,
+      resolvedPath,
+    })
+  }
+
+  if (!exists) {
+    return {}
+  }
+
+  try {
+    return dotenv.parse(await readFile(resolvedPath))
+  } catch (error) {
+    throw new ConfigurationError(`Unable to load varFile at '${path}': ${error}`, {
+      error,
+      path,
+    })
+  }
+}