feat(config): allow any objects and arrays in project variables

This lifts a limitation that is no longer necessary, and adds quite a bit of flexibility to our variables mechanism. You can now put any object or array in your project variables (both global and environment-specific), and template strings can also output any object or array. See the updated docs on variables and templating for more details, as well as the tests.
garden-io · Mar 23, 2020 · 6c2df1b · 6c2df1b
1 parent 038328a
commit 6c2df1b
Show file tree

Hide file tree

Showing 10 changed files with 193 additions and 53 deletions.
diff --git a/docs/guides/variables-and-templating.md b/docs/guides/variables-and-templating.md
@@ -117,9 +117,11 @@ This is useful when you don't want to provide _any_ value unless one is explicit
 
 A common use case for templating is to define variables in the project/environment configuration, and to use template strings to propagate values to modules in the project.
 
-You can define them in your project configuration using the [`variables` key](../reference/config.md#variables), as well as the [`environment[].variables` key](../reference/config.md#environmentsvariables) for environment-specific values. You might, for example, define project defaults using the `variables` key, and then provide environment-specific overrides in the `environment[].variables` key for each environment.
+You can define them in your project configuration using the [`variables` key](../reference/config.md#variables), as well as the [`environment[].variables` key](../reference/config.md#environmentsvariables) for environment-specific values.
 
-The variables can then be configured via `${var.<key>}` template string keys. For example:
+You might, for example, define project defaults using the `variables` key, and then provide environment-specific overrides in the `environment[].variables` key for each environment. When merging the environment-specific variables and project-wide variables, we use a [JSON Merge Patch](https://tools.ietf.org/html/rfc7396).
+
+The variables can then be referenced via `${var.<key>}` template string keys. For example:
 
 ```yaml
 kind: Project
@@ -143,6 +145,28 @@ services:
       LOG_LEVEL: ${var.log-level}   # <- resolves to "debug" for the "local" environment, "info" for the "remote" env
 ```
 
+Variable values can be any valid JSON/YAML values (strings, numbers, nulls, nested objects, and arrays of any of those). When referencing a nested key, simply use a standard dot delimiter, e.g. `${var.my.nested.key}`.
+
+You can also output objects or arrays from template strings. For example:
+
+```yaml
+kind: Project
+...
+variables:
+  dockerBuildArgs: [--no-cache, --squash]   # (this is just an example, not suggesting you actually do this :)
+  envVars:
+    LOG_LEVEL: debug
+    SOME_OTHER_VAR: something
+---
+kind: Module
+...
+buildArgs: ${var.dockerBuildArgs}  # <- resolves to the whole dockerBuildArgs list
+services:
+  - name: my-service
+    ...
+    env: ${var.envVars}            # <- resolves to the whole envVars object
+```
+
 ### Variable files (varfiles)
 
 You can also provide variables using "variable files" or _varfiles_. These work mostly like "dotenv" files or envfiles. However, they don't implicitly affect the environment of the Garden process and the configured services, but rather are added on top of the `variables` you define in your project `garden.yml`.

diff --git a/docs/reference/config.md b/docs/reference/config.md
@@ -152,7 +152,8 @@ sources:
 # multiple ones. See the `environments[].varfile` field for this option._
 varfile: garden.env
 
-# Variables to configure for all environments.
+# Key/value map of variables to configure for all environments. Keys may contain letters and numbers. Any values are
+# permitted, including arrays and objects of any nesting.
 variables: {}
 ```
 
@@ -558,7 +559,7 @@ varfile: "custom.env"
 
 ### `variables`
 
-Variables to configure for all environments.
+Key/value map of variables to configure for all environments. Keys may contain letters and numbers. Any values are permitted, including arrays and objects of any nesting.
 
 | Type     | Default | Required |
 | -------- | ------- | -------- |

diff --git a/garden-service/src/config/common.ts b/garden-service/src/config/common.ts
@@ -393,13 +393,19 @@ export const joiIdentifierMap = (valueSchema: Joi.Schema) =>
     .default(() => ({}))
     .description("Key/value map. Keys must be valid identifiers.")
 
+export const joiVariablesDescription =
+  "Keys may contain letters and numbers. Any values are permitted, including arrays and objects of any nesting."
+
 export const joiVariables = () =>
   joi
     .object()
-    .pattern(/[a-zA-Z][a-zA-Z0-9_\-]+/i, joiPrimitive())
+    .pattern(
+      /[a-zA-Z][a-zA-Z0-9_\-]+/i,
+      joi.alternatives(joiPrimitive(), joi.link("..."), joi.array().items(joi.link("...")))
+    )
     .default(() => ({}))
     .unknown(false)
-    .description("Key/value map. Keys may contain letters and numbers, and values must be primitives.")
+    .description("Key/value map. " + joiVariablesDescription)
 
 export const joiEnvVars = () =>
   joi

diff --git a/garden-service/src/config/config-context.ts b/garden-service/src/config/config-context.ts
@@ -10,7 +10,7 @@ import Joi from "@hapi/joi"
 import chalk from "chalk"
 import username from "username"
 import { isString } from "lodash"
-import { PrimitiveMap, joiIdentifierMap, joiStringMap, joiPrimitive } from "./common"
+import { PrimitiveMap, joiIdentifierMap, joiStringMap, joiPrimitive, DeepPrimitiveMap, joiVariables } from "./common"
 import { Provider, ProviderConfig } from "./provider"
 import { ModuleConfig } from "./module"
 import { ConfigurationError } from "../exceptions"
@@ -350,16 +350,16 @@ export class ProviderConfigContext extends ProjectConfigContext {
   public providers: Map<string, ProviderContext>
 
   @schema(
-    joiIdentifierMap(joiPrimitive().description("The value of the variable."))
+    joiVariables()
       .description("A map of all variables defined in the project configuration.")
       .meta({ keyPlaceholder: "<variable-name>" })
   )
-  public variables: PrimitiveMap
+  public variables: DeepPrimitiveMap
 
   @schema(joiIdentifierMap(joiPrimitive()).description("Alias for the variables field."))
-  public var: PrimitiveMap
+  public var: DeepPrimitiveMap
 
-  constructor(garden: Garden, resolvedProviders: Provider[], variables: PrimitiveMap) {
+  constructor(garden: Garden, resolvedProviders: Provider[], variables: DeepPrimitiveMap) {
     super(garden.artifactsPath)
     const _this = this
 
@@ -562,7 +562,7 @@ export class ModuleConfigContext extends ProviderConfigContext {
   constructor(
     garden: Garden,
     resolvedProviders: Provider[],
-    variables: PrimitiveMap,
+    variables: DeepPrimitiveMap,
     moduleConfigs: ModuleConfig[],
     // We only supply this when resolving configuration in dependency order.
     // Otherwise we pass `${runtime.*} template strings through for later resolution.
@@ -605,7 +605,7 @@ export class OutputConfigContext extends ModuleConfigContext {
   constructor(
     garden: Garden,
     resolvedProviders: Provider[],
-    variables: PrimitiveMap,
+    variables: DeepPrimitiveMap,
     moduleConfigs: ModuleConfig[],
     runtimeContext: RuntimeContext
   ) {

diff --git a/garden-service/src/config/project.ts b/garden-service/src/config/project.ts
@@ -19,6 +19,8 @@ import {
   joi,
   includeGuideLink,
   joiPrimitive,
+  DeepPrimitiveMap,
+  joiVariablesDescription,
 } from "./common"
 import { validateWithPath } from "./validation"
 import { resolveTemplateStrings } from "../template-string"
@@ -41,7 +43,7 @@ export const fixedPlugins = ["exec", "container"]
 
 export interface CommonEnvironmentConfig {
   providers?: ProviderConfig[] // further validated by each plugin
-  variables: { [key: string]: Primitive }
+  variables: DeepPrimitiveMap
 }
 
 const environmentConfigKeys = {
@@ -160,7 +162,7 @@ export interface ProjectConfig {
   providers: ProviderConfig[]
   sources?: SourceConfig[]
   varfile?: string
-  variables: PrimitiveMap
+  variables: DeepPrimitiveMap
 }
 
 export interface ProjectResource extends ProjectConfig {
@@ -343,7 +345,9 @@ export const projectDocsSchema = () =>
       `
         )
         .example("custom.env"),
-      variables: joiVariables().description("Variables to configure for all environments."),
+      variables: joiVariables().description(
+        "Key/value map of variables to configure for all environments. " + joiVariablesDescription
+      ),
     })
     .required()
     .description(
@@ -491,7 +495,7 @@ export async function pickEnvironment(config: ProjectConfig, environmentName: st
     defaultEnvVarfilePath(environmentName)
   )
 
-  const variables: PrimitiveMap = <any>(
+  const variables: DeepPrimitiveMap = <any>(
     merge(merge(config.variables, projectVarfileVars), merge(environmentConfig.variables, envVarfileVars))
   )
 

diff --git a/garden-service/src/garden.ts b/garden-service/src/garden.ts
@@ -30,7 +30,7 @@ import { TaskGraph, TaskResults, ProcessTasksOpts } from "./task-graph"
 import { getLogger } from "./logger/logger"
 import { PluginActionHandlers, GardenPlugin } from "./types/plugin/plugin"
 import { loadConfig, findProjectConfig, prepareModuleResource } from "./config/base"
-import { PrimitiveMap } from "./config/common"
+import { DeepPrimitiveMap } from "./config/common"
 import { validateSchema } from "./config/validation"
 import { BaseTask } from "./tasks/base"
 import { LocalConfigStore, ConfigStore, GlobalConfigStore } from "./config-store"
@@ -111,7 +111,7 @@ export interface GardenParams {
   projectRoot: string
   projectSources?: SourceConfig[]
   providerConfigs: ProviderConfig[]
-  variables: PrimitiveMap
+  variables: DeepPrimitiveMap
   vcs: VcsHandler
   workingCopyId: string
 }
@@ -140,7 +140,7 @@ export class Garden {
   public readonly projectRoot: string
   public readonly projectName: string
   public readonly environmentName: string
-  public readonly variables: PrimitiveMap
+  public readonly variables: DeepPrimitiveMap
   public readonly projectSources: SourceConfig[]
   public readonly buildDir: BuildDir
   public readonly gardenDirPath: string
@@ -950,7 +950,7 @@ export class Garden {
 export interface ConfigDump {
   environmentName: string
   providers: Provider[]
-  variables: PrimitiveMap
+  variables: DeepPrimitiveMap
   moduleConfigs: ModuleConfig[]
   projectRoot: string
 }
diff --git a/garden-service/src/template-string.ts b/garden-service/src/template-string.ts
@@ -12,7 +12,7 @@ import { asyncDeepMap } from "./util/util"
 import { GardenBaseError, ConfigurationError } from "./exceptions"
 import { ConfigContext, ContextResolveOpts, ScanContext, ContextResolveOutput } from "./config/config-context"
 import { uniq, isPlainObject, isNumber } from "lodash"
-import { Primitive, isPrimitive } from "./config/common"
+import { Primitive } from "./config/common"
 import { profileAsync } from "./util/profiling"
 
 export type StringOrStringPromise = Promise<string> | string
@@ -49,7 +49,7 @@ export async function resolveTemplateString(
   string: string,
   context: ConfigContext,
   opts: ContextResolveOpts = {}
-): Promise<Primitive | undefined> {
+): Promise<any> {
   if (!string) {
     return string
   }
@@ -85,7 +85,7 @@ export async function resolveTemplateString(
     }
 
     // Use value directly if there is only one (or no) value in the output.
-    let resolved: Primitive | undefined = outputs[0]?.resolved
+    let resolved: any = outputs[0]?.resolved
 
     if (outputs.length > 1) {
       resolved = outputs
@@ -96,17 +96,7 @@ export async function resolveTemplateString(
         .join("")
     }
 
-    if (resolved !== undefined && !isPrimitive(resolved)) {
-      throw new ConfigurationError(
-        `Template string doesn't resolve to a primitive (string, number, boolean or null).`,
-        {
-          string,
-          resolved,
-        }
-      )
-    }
-
-    return <Primitive | undefined>resolved
+    return resolved
   } catch (err) {
     const prefix = `Invalid template string ${string}: `
     const message = err.message.startsWith(prefix) ? err.message : prefix + err.message

diff --git a/garden-service/test/unit/src/config/project.ts b/garden-service/test/unit/src/config/project.ts
@@ -483,24 +483,40 @@ describe("pickEnvironment", () => {
         {
           name: "default",
           variables: {
-            b: "B",
-            c: "c",
+            b: "env value B",
+            c: "env value C",
+            array: [{ envArrayKey: "env array value" }],
+            nested: {
+              nestedB: "nested env value B",
+              nestedC: "nested env value C",
+            },
           },
         },
       ],
       providers: [],
       variables: {
-        a: "a",
-        b: "b",
+        a: "project value A",
+        b: "project value B",
+        array: [{ projectArrayKey: "project array value" }],
+        nested: {
+          nestedA: "nested project value A",
+          nestedB: "nested project value B",
+        },
       },
     }
 
     const result = await pickEnvironment(config, "default")
 
     expect(result.variables).to.eql({
-      a: "a",
-      b: "B",
-      c: "c",
+      a: "project value A",
+      b: "env value B",
+      c: "env value C",
+      array: [{ envArrayKey: "env array value", projectArrayKey: "project array value" }],
+      nested: {
+        nestedA: "nested project value A",
+        nestedB: "nested env value B",
+        nestedC: "nested env value C",
+      },
     })
   })