diff --git a/garden-service/src/config/config-context.ts b/garden-service/src/config/config-context.ts index dbb0b227ccb..1c344b56132 100644 --- a/garden-service/src/config/config-context.ts +++ b/garden-service/src/config/config-context.ts @@ -307,6 +307,46 @@ export class ProjectConfigContext extends ConfigContext { } } +/** + * This context is available for template strings for all `environments[]` fields (except name) + */ +export class EnvironmentConfigContext extends ProjectConfigContext { + @schema( + LocalContext.getSchema().description( + "Context variables that are specific to the currently running environment/machine." + ) + ) + public local: LocalContext + + @schema(ProjectContext.getSchema().description("Information about the Garden project.")) + public project: ProjectContext + + @schema( + joiVariables() + .description("A map of all variables defined in the project configuration.") + .meta({ keyPlaceholder: "" }) + ) + public variables: DeepPrimitiveMap + + @schema(joiIdentifierMap(joiPrimitive()).description("Alias for the variables field.")) + public var: DeepPrimitiveMap + + constructor({ + projectName, + artifactsPath, + username, + variables, + }: { + projectName: string + artifactsPath: string + username?: string + variables: DeepPrimitiveMap + }) { + super({ projectName, artifactsPath, username }) + this.variables = this.var = variables + } +} + class EnvironmentContext extends ConfigContext { @schema( joi @@ -342,22 +382,22 @@ class EnvironmentContext extends ConfigContext { } } -export class WorkflowConfigContext extends ProjectConfigContext { +export class WorkflowConfigContext extends EnvironmentConfigContext { @schema( EnvironmentContext.getSchema().description("Information about the environment that Garden is running against.") ) public environment: EnvironmentContext + // Overriding to update the description. Same schema as base. @schema( joiVariables() - .description("A map of all variables defined in the project configuration.") + .description( + "A map of all variables defined in the project configuration, including environment-specific variables." + ) .meta({ keyPlaceholder: "" }) ) public variables: DeepPrimitiveMap - @schema(joiIdentifierMap(joiPrimitive()).description("Alias for the variables field.")) - public var: DeepPrimitiveMap - @schema( joiStringMap(joi.string().description("The secret's value.")) .description("A map of all secrets for this project in the current environment.") @@ -372,14 +412,18 @@ export class WorkflowConfigContext extends ProjectConfigContext { public steps: Map | PassthroughContext constructor(garden: Garden) { - super({ projectName: garden.projectName, artifactsPath: garden.artifactsPath, username: garden.username }) + super({ + projectName: garden.projectName, + artifactsPath: garden.artifactsPath, + username: garden.username, + variables: garden.variables, + }) const fullEnvName = garden.namespace ? `${garden.namespace}.${garden.environmentName}` : garden.environmentName this.environment = new EnvironmentContext(this, garden.environmentName, fullEnvName, garden.namespace) this.project = new ProjectContext(this, garden.projectName) - this.var = this.variables = garden.variables this.secrets = garden.secrets this.steps = new PassthroughContext() diff --git a/garden-service/src/config/project.ts b/garden-service/src/config/project.ts index 40fa183f336..c09f7e75c55 100644 --- a/garden-service/src/config/project.ts +++ b/garden-service/src/config/project.ts @@ -24,7 +24,7 @@ import { } from "./common" import { validateWithPath } from "./validation" import { resolveTemplateStrings } from "../template-string" -import { ProjectConfigContext } from "./config-context" +import { ProjectConfigContext, EnvironmentConfigContext } from "./config-context" import { findByName, getNames } from "../util/util" import { ConfigurationError, ParameterError, ValidationError } from "../exceptions" import { PrimitiveMap } from "./common" @@ -117,7 +117,8 @@ export const environmentSchema = () => .example("custom.env"), variables: joiVariables().description(deline` A key/value map of variables that modules can reference when using this environment. These take precedence - over variables defined in the top-level \`variables\` field. + over variables defined in the top-level \`variables\` field, but may also reference the top-level variables in + template strings. `), }) @@ -405,7 +406,7 @@ export const projectSchema = () => /** * Resolves and validates the given raw project configuration, and returns it in a canonical form. * - * Note: Does _not_ resolve template strings on providers (this needs to happen later in the process). + * Note: Does _not_ resolve template strings on environments and providers (this needs to happen later in the process). * * @param config raw project configuration */ @@ -420,14 +421,19 @@ export function resolveProjectConfig(config: ProjectConfig, artifactsPath: strin sources: config.sources, varfile: config.varfile, variables: config.variables, - environments: environments.map((e) => omit(e, ["providers"])), + environments: [], }, new ProjectConfigContext({ projectName: name, artifactsPath, username }) ) // Validate after resolving global fields config = validateWithPath({ - config: { ...config, ...globalConfig, name }, + config: { + ...config, + ...globalConfig, + name, + environments: environments.map((e) => omit(e, ["providers"])), + }, schema: projectSchema(), configType: "project", path: config.path, @@ -480,7 +486,7 @@ export function resolveProjectConfig(config: ProjectConfig, artifactsPath: strin /** * Given an environment name, pulls the relevant environment-specific configuration from the specified project - * config, and merges values appropriately. + * config, and merges values appropriately. Also resolves template strings in the picked environment. * * For project variables, we apply the variables specified to the selected environment on the global variables * specified on the top-level `variables` key using a JSON Merge Patch (https://tools.ietf.org/html/rfc7396). @@ -500,12 +506,22 @@ export function resolveProjectConfig(config: ProjectConfig, artifactsPath: strin * @param config a resolved project config (as returned by `resolveProjectConfig()`) * @param envString the name of the environment to use */ -export async function pickEnvironment(config: ProjectConfig, envString: string) { +export async function pickEnvironment({ + config, + envString, + artifactsPath, + username, +}: { + config: ProjectConfig + envString: string + artifactsPath: string + username: string +}) { const { environments, name: projectName } = config let { environment, namespace } = parseEnvironment(envString) - const environmentConfig = findByName(environments, environment) + let environmentConfig = findByName(environments, environment) if (!environmentConfig) { throw new ParameterError(`Project ${projectName} does not specify environment ${environment}`, { @@ -516,12 +532,24 @@ export async function pickEnvironment(config: ProjectConfig, envString: string) }) } + const projectVarfileVars = await loadVarfile(config.path, config.varfile, defaultVarfilePath) + const projectVariables: DeepPrimitiveMap = merge(config.variables, projectVarfileVars) + + const envProviders = environmentConfig.providers || [] + + // Resolve template strings in the environment config, except providers + environmentConfig = resolveTemplateStrings( + { ...environmentConfig, providers: [] }, + new EnvironmentConfigContext({ projectName, artifactsPath, username, variables: projectVariables }) + ) + namespace = getNamespace(environmentConfig, namespace) const fixedProviders = fixedPlugins.map((name) => ({ name })) const allProviders = [ ...fixedProviders, ...config.providers.filter((p) => !p.environments || p.environments.includes(environment)), + ...envProviders, ] const mergedProviders: { [name: string]: ProviderConfig } = {} @@ -535,12 +563,9 @@ export async function pickEnvironment(config: ProjectConfig, envString: string) } } - const projectVarfileVars = await loadVarfile(config.path, config.varfile, defaultVarfilePath) const envVarfileVars = await loadVarfile(config.path, environmentConfig.varfile, defaultEnvVarfilePath(environment)) - const variables: DeepPrimitiveMap = ( - merge(merge(config.variables, projectVarfileVars), merge(environmentConfig.variables, envVarfileVars)) - ) + const variables: DeepPrimitiveMap = merge(projectVariables, merge(environmentConfig.variables, envVarfileVars)) return { environmentName: environment, diff --git a/garden-service/src/garden.ts b/garden-service/src/garden.ts index 438c3382cd6..c6cb18a403c 100644 --- a/garden-service/src/garden.ts +++ b/garden-service/src/garden.ts @@ -302,10 +302,12 @@ export class Garden { environmentStr = defaultEnvironment } - const { environmentName, namespace, providers, variables, production } = await pickEnvironment( + const { environmentName, namespace, providers, variables, production } = await pickEnvironment({ config, - environmentStr - ) + envString: environmentStr, + artifactsPath, + username: _username, + }) const buildDir = await BuildDir.factory(projectRoot, gardenDirPath) const workingCopyId = await getWorkingCopyId(gardenDirPath) diff --git a/garden-service/static/docs/templates/template-strings.hbs b/garden-service/static/docs/templates/template-strings.hbs index 757e576cbea..ee585943013 100644 --- a/garden-service/static/docs/templates/template-strings.hbs +++ b/garden-service/static/docs/templates/template-strings.hbs @@ -11,10 +11,16 @@ Note that there are four sections below, since different configuration sections ## Project configuration context -The following keys are available in any template strings within project definitions in `garden.yml` config files, except the `name` field (which cannot be templated). See the [Provider](#provider-configuration-context) section below for additional keys available when configuring `providers`: +The following keys are available in any template strings within project definitions in `garden.yml` config files, except the `name` field (which cannot be templated). See the [Environment](#environment-configuration-context) and [Provider](#provider-configuration-context) sections below for additional keys available when configuring `environments` and `providers`, respectively. {{{projectContext}}} +## Environment configuration context + +The following keys are available in template strings under the `environments` key in project configs. Additional keys are available for the `environments[].providers` field, see the [Provider](#provider-configuration-context) section below for those. + +{{{environmentContext}}} + ## Provider configuration context The following keys are available in template strings under the `providers` key (or `environments[].providers) in project configs. diff --git a/garden-service/test/unit/src/config/project.ts b/garden-service/test/unit/src/config/project.ts index 0871308c693..5dfa12bf2e4 100644 --- a/garden-service/test/unit/src/config/project.ts +++ b/garden-service/test/unit/src/config/project.ts @@ -24,8 +24,9 @@ import { expectError } from "../../../helpers" import { defaultDotIgnoreFiles } from "../../../../src/util/fs" import { realpath, writeFile } from "fs-extra" import { dedent } from "../../../../src/util/string" -import { resolve } from "path" +import { resolve, join } from "path" import stripAnsi from "strip-ansi" +import { keyBy } from "lodash" describe("resolveProjectConfig", () => { it("should pass through a canonical project config", async () => { @@ -80,7 +81,7 @@ describe("resolveProjectConfig", () => { }) }) - it("should resolve template strings on fields other than provider configs", async () => { + it("should resolve template strings on fields other than environment and provider configs", async () => { const repositoryUrl = "git://github.com/foo/bar.git#boo" const config: ProjectConfig = { @@ -122,7 +123,7 @@ describe("resolveProjectConfig", () => { production: false, providers: [], variables: { - envVar: "foo", + envVar: "${local.env.TEST_ENV_VAR}", }, }, ], @@ -209,6 +210,33 @@ describe("resolveProjectConfig", () => { delete process.env.TEST_ENV_VAR_B }) + it("should pass through templated fields on environment configs", async () => { + const config: ProjectConfig = { + apiVersion: DEFAULT_API_VERSION, + kind: "Project", + name: "my-project", + path: "/tmp/foo", + defaultEnvironment: "default", + dotIgnoreFiles: defaultDotIgnoreFiles, + environments: [ + { + name: "default", + defaultNamespace, + production: false, + variables: { + envVar: "${var.foo}", + }, + }, + ], + providers: [], + variables: {}, + } + + const result = resolveProjectConfig(config, "/tmp", "some-user") + + expect(result.environments).to.eql(config.environments) + }) + it("should set defaultEnvironment to first environment if not configured", async () => { const config: ProjectConfig = { apiVersion: DEFAULT_API_VERSION, @@ -255,7 +283,7 @@ describe("resolveProjectConfig", () => { }) }) - it("should include providers in correct precedency order from all possible config keys", async () => { + it("should include providers in correct precedence order from all possible config keys", async () => { const config: ProjectConfig = { apiVersion: DEFAULT_API_VERSION, kind: "Project", @@ -387,10 +415,13 @@ describe("resolveProjectConfig", () => { describe("pickEnvironment", () => { let tmpDir: tmp.DirectoryResult let tmpPath: string + let artifactsPath: string + const username = "test" beforeEach(async () => { tmpDir = await tmp.dir({ unsafeCleanup: true }) tmpPath = await realpath(tmpDir.path) + artifactsPath = join(tmpPath, ".garden", "artifacts") }) afterEach(async () => { @@ -410,7 +441,7 @@ describe("pickEnvironment", () => { variables: {}, } - await expectError(() => pickEnvironment(config, "foo"), "parameter") + await expectError(() => pickEnvironment({ config, envString: "foo", artifactsPath, username }), "parameter") }) it("should include fixed providers in output", async () => { @@ -426,7 +457,7 @@ describe("pickEnvironment", () => { variables: {}, } - expect(await pickEnvironment(config, "default")).to.eql({ + expect(await pickEnvironment({ config, envString: "default", artifactsPath, username })).to.eql({ environmentName: "default", namespace: "default", providers: [{ name: "exec" }, { name: "container" }], @@ -443,7 +474,14 @@ describe("pickEnvironment", () => { path: "/tmp/foo", defaultEnvironment: "default", dotIgnoreFiles: defaultDotIgnoreFiles, - environments: [{ name: "default", defaultNamespace, variables: {} }], + environments: [ + { + name: "default", + defaultNamespace, + variables: {}, + providers: [{ name: "my-provider", b: "d" }, { name: "env-provider" }], + }, + ], providers: [ { name: "container", newKey: "foo" }, { name: "my-provider", a: "a" }, @@ -453,10 +491,15 @@ describe("pickEnvironment", () => { variables: {}, } - expect(await pickEnvironment(config, "default")).to.eql({ + expect(await pickEnvironment({ config, envString: "default", artifactsPath, username })).to.eql({ environmentName: "default", namespace: "default", - providers: [{ name: "exec" }, { name: "container", newKey: "foo" }, { name: "my-provider", a: "c", b: "b" }], + providers: [ + { name: "exec" }, + { name: "container", newKey: "foo" }, + { name: "my-provider", a: "c", b: "d" }, + { name: "env-provider" }, + ], production: false, variables: {}, }) @@ -480,7 +523,7 @@ describe("pickEnvironment", () => { variables: {}, } - expect(await pickEnvironment(config, "default")).to.eql({ + expect(await pickEnvironment({ config, envString: "default", artifactsPath, username })).to.eql({ environmentName: "default", namespace: "default", providers: [{ name: "exec" }, { name: "container", newKey: "foo" }, { name: "my-provider", b: "b" }], @@ -524,7 +567,7 @@ describe("pickEnvironment", () => { }, } - const result = await pickEnvironment(config, "default") + const result = await pickEnvironment({ config, envString: "default", artifactsPath, username }) expect(result.variables).to.eql({ a: "project value A", @@ -570,7 +613,7 @@ describe("pickEnvironment", () => { variables: {}, } - const result = await pickEnvironment(config, "default") + const result = await pickEnvironment({ config, envString: "default", artifactsPath, username }) expect(result.variables).to.eql({ a: "a", @@ -610,7 +653,7 @@ describe("pickEnvironment", () => { }, } - const result = await pickEnvironment(config, "default") + const result = await pickEnvironment({ config, envString: "default", artifactsPath, username }) expect(result.variables).to.eql({ a: "a", @@ -651,7 +694,7 @@ describe("pickEnvironment", () => { variables: {}, } - const result = await pickEnvironment(config, "default") + const result = await pickEnvironment({ config, envString: "default", artifactsPath, username }) expect(result.variables).to.eql({ a: "a", @@ -692,7 +735,7 @@ describe("pickEnvironment", () => { }, } - const result = await pickEnvironment(config, "default") + const result = await pickEnvironment({ config, envString: "default", artifactsPath, username }) expect(result.variables).to.eql({ a: "a", @@ -701,6 +744,85 @@ describe("pickEnvironment", () => { }) }) + it("should resolve template strings in the picked environment", async () => { + const config: ProjectConfig = { + apiVersion: DEFAULT_API_VERSION, + kind: "Project", + name: "my-project", + path: "/tmp/foo", + defaultEnvironment: "default", + dotIgnoreFiles: defaultDotIgnoreFiles, + environments: [{ name: "default", defaultNamespace, variables: { foo: "${local.username}" } }], + providers: [], + variables: {}, + } + + const result = await pickEnvironment({ config, envString: "default", artifactsPath, username }) + + expect(result.variables).to.eql({ + foo: username, + }) + }) + + it("should ignore template strings in other environments", async () => { + const config: ProjectConfig = { + apiVersion: DEFAULT_API_VERSION, + kind: "Project", + name: "my-project", + path: "/tmp/foo", + defaultEnvironment: "default", + dotIgnoreFiles: defaultDotIgnoreFiles, + environments: [ + { name: "default", defaultNamespace, variables: {} }, + { name: "other", defaultNamespace, variables: { foo: "${var.missing}" } }, + ], + providers: [], + variables: {}, + } + + await pickEnvironment({ config, envString: "default", artifactsPath, username }) + }) + + it("should pass through template strings in the providers field on environments", async () => { + const config: ProjectConfig = { + apiVersion: DEFAULT_API_VERSION, + kind: "Project", + name: "my-project", + path: "/tmp/foo", + defaultEnvironment: "default", + dotIgnoreFiles: defaultDotIgnoreFiles, + environments: [ + { name: "default", defaultNamespace, variables: {}, providers: [{ name: "my-provider", a: "${var.missing}" }] }, + ], + providers: [], + variables: {}, + } + + const result = await pickEnvironment({ config, envString: "default", artifactsPath, username }) + + expect(keyBy(result.providers, "name")["my-provider"].a).to.equal("${var.missing}") + }) + + it("should allow referencing top-level variables", async () => { + const config: ProjectConfig = { + apiVersion: DEFAULT_API_VERSION, + kind: "Project", + name: "my-project", + path: "/tmp/foo", + defaultEnvironment: "default", + dotIgnoreFiles: defaultDotIgnoreFiles, + environments: [{ name: "default", defaultNamespace, variables: { foo: "${var.foo}" } }], + providers: [], + variables: { foo: "value" }, + } + + const result = await pickEnvironment({ config, envString: "default", artifactsPath, username }) + + expect(result.variables).to.eql({ + foo: "value", + }) + }) + it("should correctly merge all variable sources in precedence order (variables fields and varfiles)", async () => { // Precedence 1/4 (highest) await writeFile( @@ -746,7 +868,7 @@ describe("pickEnvironment", () => { }, } - const result = await pickEnvironment(config, "default") + const result = await pickEnvironment({ config, envString: "default", artifactsPath, username }) expect(result.variables).to.eql({ a: "a", @@ -778,7 +900,7 @@ describe("pickEnvironment", () => { } await expectError( - () => pickEnvironment(config, "default"), + () => pickEnvironment({ config, envString: "default", artifactsPath, username }), (err) => expect(err.message).to.equal("Could not find varfile at path 'foo.env'") ) }) @@ -804,7 +926,7 @@ describe("pickEnvironment", () => { } await expectError( - () => pickEnvironment(config, "default"), + () => pickEnvironment({ config, envString: "default", artifactsPath, username }), (err) => expect(err.message).to.equal("Could not find varfile at path 'foo.env'") ) }) @@ -822,7 +944,7 @@ describe("pickEnvironment", () => { variables: {}, } - expect(await pickEnvironment(config, "foo.default")).to.eql({ + expect(await pickEnvironment({ config, envString: "foo.default", artifactsPath, username })).to.eql({ environmentName: "default", namespace: "foo", providers: [{ name: "exec" }, { name: "container" }], @@ -844,7 +966,7 @@ describe("pickEnvironment", () => { variables: {}, } - expect(await pickEnvironment(config, "foo.default")).to.eql({ + expect(await pickEnvironment({ config, envString: "foo.default", artifactsPath, username })).to.eql({ environmentName: "default", namespace: "foo", providers: [{ name: "exec" }, { name: "container" }], @@ -866,7 +988,7 @@ describe("pickEnvironment", () => { variables: {}, } - expect(await pickEnvironment(config, "default")).to.eql({ + expect(await pickEnvironment({ config, envString: "default", artifactsPath, username })).to.eql({ environmentName: "default", namespace: "default", providers: [{ name: "exec" }, { name: "container" }], @@ -889,7 +1011,7 @@ describe("pickEnvironment", () => { } await expectError( - () => pickEnvironment(config, "$.%"), + () => pickEnvironment({ config, envString: "$.%", artifactsPath, username }), (err) => expect(err.message).to.equal( "Invalid environment specified ($.%): must be a valid environment name or ." @@ -911,7 +1033,7 @@ describe("pickEnvironment", () => { } await expectError( - () => pickEnvironment(config, "default"), + () => pickEnvironment({ config, envString: "default", artifactsPath, username }), (err) => expect(stripAnsi(err.message)).to.equal( "Environment default has defaultNamespace set to null, and no explicit namespace was specified. Please either set a defaultNamespace or explicitly set a namespace at runtime (e.g. --env=some-namespace.default)."