diff --git a/docs/guides/variables-and-templating.md b/docs/guides/variables-and-templating.md index 49c5e75a8c..e0a3dfbbda 100644 --- a/docs/guides/variables-and-templating.md +++ b/docs/guides/variables-and-templating.md @@ -117,9 +117,11 @@ This is useful when you don't want to provide _any_ value unless one is explicit A common use case for templating is to define variables in the project/environment configuration, and to use template strings to propagate values to modules in the project. -You can define them in your project configuration using the [`variables` key](../reference/config.md#variables), as well as the [`environment[].variables` key](../reference/config.md#environmentsvariables) for environment-specific values. You might, for example, define project defaults using the `variables` key, and then provide environment-specific overrides in the `environment[].variables` key for each environment. +You can define them in your project configuration using the [`variables` key](../reference/config.md#variables), as well as the [`environment[].variables` key](../reference/config.md#environmentsvariables) for environment-specific values. -The variables can then be configured via `${var.}` template string keys. For example: +You might, for example, define project defaults using the `variables` key, and then provide environment-specific overrides in the `environment[].variables` key for each environment. When merging the environment-specific variables and project-wide variables, we use a [JSON Merge Patch](https://tools.ietf.org/html/rfc7396). + +The variables can then be referenced via `${var.}` template string keys. For example: ```yaml kind: Project @@ -143,6 +145,28 @@ services: LOG_LEVEL: ${var.log-level} # <- resolves to "debug" for the "local" environment, "info" for the "remote" env ``` +Variable values can be any valid JSON/YAML values (strings, numbers, nulls, nested objects, and arrays of any of those). When referencing a nested key, simply use a standard dot delimiter, e.g. `${var.my.nested.key}`. + +You can also output objects or arrays from template strings. For example: + +```yaml +kind: Project +... +variables: + dockerBuildArgs: [--no-cache, --squash] # (this is just an example, not suggesting you actually do this :) + envVars: + LOG_LEVEL: debug + SOME_OTHER_VAR: something +--- +kind: Module +... +buildArgs: ${var.dockerBuildArgs} # <- resolves to the whole dockerBuildArgs list +services: + - name: my-service + ... + env: ${var.envVars} # <- resolves to the whole envVars object +``` + ### Variable files (varfiles) You can also provide variables using "variable files" or _varfiles_. These work mostly like "dotenv" files or envfiles. However, they don't implicitly affect the environment of the Garden process and the configured services, but rather are added on top of the `variables` you define in your project `garden.yml`. diff --git a/docs/reference/config.md b/docs/reference/config.md index 725a978349..e34915535a 100644 --- a/docs/reference/config.md +++ b/docs/reference/config.md @@ -152,7 +152,8 @@ sources: # multiple ones. See the `environments[].varfile` field for this option._ varfile: garden.env -# Variables to configure for all environments. +# Key/value map of variables to configure for all environments. Keys may contain letters and numbers. Any values are +# permitted, including arrays and objects of any nesting. variables: {} ``` @@ -558,7 +559,7 @@ varfile: "custom.env" ### `variables` -Variables to configure for all environments. +Key/value map of variables to configure for all environments. Keys may contain letters and numbers. Any values are permitted, including arrays and objects of any nesting. | Type | Default | Required | | -------- | ------- | -------- | diff --git a/garden-service/src/config/common.ts b/garden-service/src/config/common.ts index 1b3f77b148..ad8adea5b5 100644 --- a/garden-service/src/config/common.ts +++ b/garden-service/src/config/common.ts @@ -393,13 +393,19 @@ export const joiIdentifierMap = (valueSchema: Joi.Schema) => .default(() => ({})) .description("Key/value map. Keys must be valid identifiers.") +export const joiVariablesDescription = + "Keys may contain letters and numbers. Any values are permitted, including arrays and objects of any nesting." + export const joiVariables = () => joi .object() - .pattern(/[a-zA-Z][a-zA-Z0-9_\-]+/i, joiPrimitive()) + .pattern( + /[a-zA-Z][a-zA-Z0-9_\-]+/i, + joi.alternatives(joiPrimitive(), joi.link("..."), joi.array().items(joi.link("..."))) + ) .default(() => ({})) .unknown(false) - .description("Key/value map. Keys may contain letters and numbers, and values must be primitives.") + .description("Key/value map. " + joiVariablesDescription) export const joiEnvVars = () => joi diff --git a/garden-service/src/config/config-context.ts b/garden-service/src/config/config-context.ts index 189ddf8073..de35714d2f 100644 --- a/garden-service/src/config/config-context.ts +++ b/garden-service/src/config/config-context.ts @@ -10,7 +10,7 @@ import Joi from "@hapi/joi" import chalk from "chalk" import username from "username" import { isString } from "lodash" -import { PrimitiveMap, joiIdentifierMap, joiStringMap, joiPrimitive } from "./common" +import { PrimitiveMap, joiIdentifierMap, joiStringMap, joiPrimitive, DeepPrimitiveMap, joiVariables } from "./common" import { Provider, ProviderConfig } from "./provider" import { ModuleConfig } from "./module" import { ConfigurationError } from "../exceptions" @@ -350,16 +350,16 @@ export class ProviderConfigContext extends ProjectConfigContext { public providers: Map @schema( - joiIdentifierMap(joiPrimitive().description("The value of the variable.")) + joiVariables() .description("A map of all variables defined in the project configuration.") .meta({ keyPlaceholder: "" }) ) - public variables: PrimitiveMap + public variables: DeepPrimitiveMap @schema(joiIdentifierMap(joiPrimitive()).description("Alias for the variables field.")) - public var: PrimitiveMap + public var: DeepPrimitiveMap - constructor(garden: Garden, resolvedProviders: Provider[], variables: PrimitiveMap) { + constructor(garden: Garden, resolvedProviders: Provider[], variables: DeepPrimitiveMap) { super(garden.artifactsPath) const _this = this @@ -562,7 +562,7 @@ export class ModuleConfigContext extends ProviderConfigContext { constructor( garden: Garden, resolvedProviders: Provider[], - variables: PrimitiveMap, + variables: DeepPrimitiveMap, moduleConfigs: ModuleConfig[], // We only supply this when resolving configuration in dependency order. // Otherwise we pass `${runtime.*} template strings through for later resolution. @@ -605,7 +605,7 @@ export class OutputConfigContext extends ModuleConfigContext { constructor( garden: Garden, resolvedProviders: Provider[], - variables: PrimitiveMap, + variables: DeepPrimitiveMap, moduleConfigs: ModuleConfig[], runtimeContext: RuntimeContext ) { diff --git a/garden-service/src/config/project.ts b/garden-service/src/config/project.ts index 37cf24d873..ff1aeb067f 100644 --- a/garden-service/src/config/project.ts +++ b/garden-service/src/config/project.ts @@ -19,6 +19,8 @@ import { joi, includeGuideLink, joiPrimitive, + DeepPrimitiveMap, + joiVariablesDescription, } from "./common" import { validateWithPath } from "./validation" import { resolveTemplateStrings } from "../template-string" @@ -41,7 +43,7 @@ export const fixedPlugins = ["exec", "container"] export interface CommonEnvironmentConfig { providers?: ProviderConfig[] // further validated by each plugin - variables: { [key: string]: Primitive } + variables: DeepPrimitiveMap } const environmentConfigKeys = { @@ -160,7 +162,7 @@ export interface ProjectConfig { providers: ProviderConfig[] sources?: SourceConfig[] varfile?: string - variables: PrimitiveMap + variables: DeepPrimitiveMap } export interface ProjectResource extends ProjectConfig { @@ -343,7 +345,9 @@ export const projectDocsSchema = () => ` ) .example("custom.env"), - variables: joiVariables().description("Variables to configure for all environments."), + variables: joiVariables().description( + "Key/value map of variables to configure for all environments. " + joiVariablesDescription + ), }) .required() .description( @@ -491,7 +495,7 @@ export async function pickEnvironment(config: ProjectConfig, environmentName: st defaultEnvVarfilePath(environmentName) ) - const variables: PrimitiveMap = ( + const variables: DeepPrimitiveMap = ( merge(merge(config.variables, projectVarfileVars), merge(environmentConfig.variables, envVarfileVars)) ) diff --git a/garden-service/src/garden.ts b/garden-service/src/garden.ts index 29f913d706..416625420a 100644 --- a/garden-service/src/garden.ts +++ b/garden-service/src/garden.ts @@ -30,7 +30,7 @@ import { TaskGraph, TaskResults, ProcessTasksOpts } from "./task-graph" import { getLogger } from "./logger/logger" import { PluginActionHandlers, GardenPlugin } from "./types/plugin/plugin" import { loadConfig, findProjectConfig, prepareModuleResource } from "./config/base" -import { PrimitiveMap } from "./config/common" +import { DeepPrimitiveMap } from "./config/common" import { validateSchema } from "./config/validation" import { BaseTask } from "./tasks/base" import { LocalConfigStore, ConfigStore, GlobalConfigStore } from "./config-store" @@ -111,7 +111,7 @@ export interface GardenParams { projectRoot: string projectSources?: SourceConfig[] providerConfigs: ProviderConfig[] - variables: PrimitiveMap + variables: DeepPrimitiveMap vcs: VcsHandler workingCopyId: string } @@ -140,7 +140,7 @@ export class Garden { public readonly projectRoot: string public readonly projectName: string public readonly environmentName: string - public readonly variables: PrimitiveMap + public readonly variables: DeepPrimitiveMap public readonly projectSources: SourceConfig[] public readonly buildDir: BuildDir public readonly gardenDirPath: string @@ -950,7 +950,7 @@ export class Garden { export interface ConfigDump { environmentName: string providers: Provider[] - variables: PrimitiveMap + variables: DeepPrimitiveMap moduleConfigs: ModuleConfig[] projectRoot: string } diff --git a/garden-service/src/template-string.ts b/garden-service/src/template-string.ts index f91bddee56..2a153ebd1b 100644 --- a/garden-service/src/template-string.ts +++ b/garden-service/src/template-string.ts @@ -12,7 +12,7 @@ import { asyncDeepMap } from "./util/util" import { GardenBaseError, ConfigurationError } from "./exceptions" import { ConfigContext, ContextResolveOpts, ScanContext, ContextResolveOutput } from "./config/config-context" import { uniq, isPlainObject, isNumber } from "lodash" -import { Primitive, isPrimitive } from "./config/common" +import { Primitive } from "./config/common" import { profileAsync } from "./util/profiling" export type StringOrStringPromise = Promise | string @@ -49,7 +49,7 @@ export async function resolveTemplateString( string: string, context: ConfigContext, opts: ContextResolveOpts = {} -): Promise { +): Promise { if (!string) { return string } @@ -85,7 +85,7 @@ export async function resolveTemplateString( } // Use value directly if there is only one (or no) value in the output. - let resolved: Primitive | undefined = outputs[0]?.resolved + let resolved: any = outputs[0]?.resolved if (outputs.length > 1) { resolved = outputs @@ -96,17 +96,7 @@ export async function resolveTemplateString( .join("") } - if (resolved !== undefined && !isPrimitive(resolved)) { - throw new ConfigurationError( - `Template string doesn't resolve to a primitive (string, number, boolean or null).`, - { - string, - resolved, - } - ) - } - - return resolved + return resolved } catch (err) { const prefix = `Invalid template string ${string}: ` const message = err.message.startsWith(prefix) ? err.message : prefix + err.message diff --git a/garden-service/test/unit/src/config/project.ts b/garden-service/test/unit/src/config/project.ts index 6bf8904873..8f4c57fe95 100644 --- a/garden-service/test/unit/src/config/project.ts +++ b/garden-service/test/unit/src/config/project.ts @@ -483,24 +483,40 @@ describe("pickEnvironment", () => { { name: "default", variables: { - b: "B", - c: "c", + b: "env value B", + c: "env value C", + array: [{ envArrayKey: "env array value" }], + nested: { + nestedB: "nested env value B", + nestedC: "nested env value C", + }, }, }, ], providers: [], variables: { - a: "a", - b: "b", + a: "project value A", + b: "project value B", + array: [{ projectArrayKey: "project array value" }], + nested: { + nestedA: "nested project value A", + nestedB: "nested project value B", + }, }, } const result = await pickEnvironment(config, "default") expect(result.variables).to.eql({ - a: "a", - b: "B", - c: "c", + a: "project value A", + b: "env value B", + c: "env value C", + array: [{ envArrayKey: "env array value", projectArrayKey: "project array value" }], + nested: { + nestedA: "nested project value A", + nestedB: "nested env value B", + nestedC: "nested env value C", + }, }) }) diff --git a/garden-service/test/unit/src/garden.ts b/garden-service/test/unit/src/garden.ts index 903b1e1b52..cd4e57772e 100644 --- a/garden-service/test/unit/src/garden.ts +++ b/garden-service/test/unit/src/garden.ts @@ -2156,6 +2156,106 @@ describe("Garden", () => { expect(module.allowPublish).to.equal(false) }) + it("should correctly resolve template strings referencing nested variable", async () => { + const test = createGardenPlugin({ + name: "test", + createModuleTypes: [ + { + name: "test", + docs: "test", + schema: joi.object().keys({ bla: joi.string() }), + handlers: {}, + }, + ], + }) + + const garden = await TestGarden.factory(pathFoo, { + plugins: [test], + config: { + apiVersion: DEFAULT_API_VERSION, + kind: "Project", + name: "test", + path: pathFoo, + defaultEnvironment: "default", + dotIgnoreFiles: [], + environments: [{ name: "default", variables: { some: { nested: { key: "my value" } } } }], + providers: [{ name: "test" }], + variables: {}, + }, + }) + + garden.setModuleConfigs([ + { + apiVersion: DEFAULT_API_VERSION, + name: "module-a", + type: "test", + allowPublish: false, + build: { dependencies: [] }, + disabled: false, + outputs: {}, + path: pathFoo, + serviceConfigs: [], + taskConfigs: [], + testConfigs: [], + spec: { bla: "${var.some.nested.key}" }, + }, + ]) + + const module = await garden.resolveModuleConfig(garden.log, "module-a") + + expect(module.spec.bla).to.equal("my value") + }) + + it("should correctly resolve template strings referencing objects", async () => { + const test = createGardenPlugin({ + name: "test", + createModuleTypes: [ + { + name: "test", + docs: "test", + schema: joi.object().keys({ bla: joi.object() }), + handlers: {}, + }, + ], + }) + + const garden = await TestGarden.factory(pathFoo, { + plugins: [test], + config: { + apiVersion: DEFAULT_API_VERSION, + kind: "Project", + name: "test", + path: pathFoo, + defaultEnvironment: "default", + dotIgnoreFiles: [], + environments: [{ name: "default", variables: { some: { nested: { key: "my value" } } } }], + providers: [{ name: "test" }], + variables: {}, + }, + }) + + garden.setModuleConfigs([ + { + apiVersion: DEFAULT_API_VERSION, + name: "module-a", + type: "test", + allowPublish: false, + build: { dependencies: [] }, + disabled: false, + outputs: {}, + path: pathFoo, + serviceConfigs: [], + taskConfigs: [], + testConfigs: [], + spec: { bla: "${var.some}" }, + }, + ]) + + const module = await garden.resolveModuleConfig(garden.log, "module-a") + + expect(module.spec.bla).to.eql({ nested: { key: "my value" } }) + }) + it("should handle module references within single file", async () => { const projectRoot = getDataDir("test-projects", "1067-module-ref-within-file") const garden = await makeTestGarden(projectRoot) diff --git a/garden-service/test/unit/src/template-string.ts b/garden-service/test/unit/src/template-string.ts index 0442494e62..b0ae082d55 100644 --- a/garden-service/test/unit/src/template-string.ts +++ b/garden-service/test/unit/src/template-string.ts @@ -121,17 +121,6 @@ describe("resolveTemplateString", async () => { throw new Error("Expected error") }) - it("should throw when a found key is not a primitive", async () => { - return expectError( - () => resolveTemplateString("${some}", new TestContext({ some: {} })), - (err) => - expect(err.message).to.equal( - "Invalid template string ${some}: " + - "Template string doesn't resolve to a primitive (string, number, boolean or null)." - ) - ) - }) - it("should throw with an incomplete template string", async () => { try { await resolveTemplateString("${some", new TestContext({ some: {} })) @@ -514,6 +503,16 @@ describe("resolveTemplateString", async () => { const res = await resolveTemplateString("${a}", new TestContext({ a: null })) expect(res).to.equal(null) }) + + it("should return a resolved object directly", async () => { + const res = await resolveTemplateString("${a}", new TestContext({ a: { b: 123 } })) + expect(res).to.eql({ b: 123 }) + }) + + it("should return a resolved array directly", async () => { + const res = await resolveTemplateString("${a}", new TestContext({ a: [123] })) + expect(res).to.eql([123]) + }) }) context("when the template string is a part of a string", () => {