From 4d65cb2d000c076c3d6140778ce64a5bb6fdf95f Mon Sep 17 00:00:00 2001 From: Thorarinn Sigurdsson Date: Tue, 15 Jun 2021 17:08:34 +0200 Subject: [PATCH] feat(core): allow variables in remote sources Template strings in remote sources now have access to the same variables as provider configs. This adds flexibility when using remote sources in conjunction with automation (e.g. when using dynamic Git branches / tags for remote sources, depending on project- and environment-level variables). --- core/src/config/project.ts | 8 +- core/src/config/template-contexts/project.ts | 38 ++- core/src/config/template-contexts/workflow.ts | 39 +-- core/src/docs/template-strings.ts | 15 +- core/src/garden.ts | 11 +- core/test/unit/src/config/project.ts | 71 ++++- core/test/unit/src/garden.ts | 64 ++++ docs/reference/template-strings.md | 283 +++++++++++++++++- static/docs/templates/template-strings.hbs | 7 +- 9 files changed, 486 insertions(+), 50 deletions(-) diff --git a/core/src/config/project.ts b/core/src/config/project.ts index fc6859f310..c54c037a62 100644 --- a/core/src/config/project.ts +++ b/core/src/config/project.ts @@ -418,16 +418,16 @@ export function resolveProjectConfig({ secrets: PrimitiveMap commandInfo: CommandInfo }): ProjectConfig { - // Resolve template strings for non-environment-specific fields - const { environments = [], name } = config + // Resolve template strings for non-environment-specific fields (apart from `sources`). + const { environments = [], name, sources = [] } = config const globalConfig = resolveTemplateStrings( { apiVersion: config.apiVersion, - sources: config.sources, varfile: config.varfile, variables: config.variables, environments: [], + sources: [], }, new ProjectConfigContext({ projectName: name, @@ -450,6 +450,7 @@ export function resolveProjectConfig({ name, defaultEnvironment, environments: [], + sources: [], }, schema: projectSchema(), configType: "project", @@ -477,6 +478,7 @@ export function resolveProjectConfig({ ...config, environments: config.environments || [], providers, + sources, } config.defaultEnvironment = getDefaultEnvironmentName(defaultEnvironment, config) diff --git a/core/src/config/template-contexts/project.ts b/core/src/config/template-contexts/project.ts index 269e99ca5b..26d330428c 100644 --- a/core/src/config/template-contexts/project.ts +++ b/core/src/config/template-contexts/project.ts @@ -11,8 +11,9 @@ import chalk from "chalk" import { PrimitiveMap, joiIdentifierMap, joiStringMap, joiPrimitive, DeepPrimitiveMap, joiVariables } from "../common" import { joi } from "../common" import { deline, dedent } from "../../util/string" -import { schema, ConfigContext, ContextKeySegment } from "./base" +import { schema, ConfigContext, ContextKeySegment, EnvironmentContext } from "./base" import { CommandInfo } from "../../plugin-context" +import { Garden } from "../../garden" class LocalContext extends ConfigContext { @schema( @@ -298,3 +299,38 @@ export class EnvironmentConfigContext extends ProjectConfigContext { this.variables = this.var = params.variables } } + +export class RemoteSourceConfigContext extends EnvironmentConfigContext { + @schema( + EnvironmentContext.getSchema().description("Information about the environment that Garden is running against.") + ) + public environment: EnvironmentContext + + // Overriding to update the description. Same schema as base. + @schema( + joiVariables() + .description( + "A map of all variables defined in the project configuration, including environment-specific variables." + ) + .meta({ keyPlaceholder: "" }) + ) + public variables: DeepPrimitiveMap + + constructor(garden: Garden) { + super({ + projectName: garden.projectName, + projectRoot: garden.projectRoot, + artifactsPath: garden.artifactsPath, + branch: garden.vcsBranch, + username: garden.username, + variables: garden.variables, + loggedIn: !!garden.enterpriseApi, + enterpriseDomain: garden.enterpriseApi?.domain, + secrets: garden.secrets, + commandInfo: garden.commandInfo, + }) + + const fullEnvName = garden.namespace ? `${garden.namespace}.${garden.environmentName}` : garden.environmentName + this.environment = new EnvironmentContext(this, garden.environmentName, fullEnvName, garden.namespace) + } +} diff --git a/core/src/config/template-contexts/workflow.ts b/core/src/config/template-contexts/workflow.ts index 74ee300ac2..b79c33efe4 100644 --- a/core/src/config/template-contexts/workflow.ts +++ b/core/src/config/template-contexts/workflow.ts @@ -10,46 +10,13 @@ import { joiIdentifierMap, DeepPrimitiveMap, joiVariables } from "../common" import { Garden } from "../../garden" import { joi } from "../common" import { dedent } from "../../util/string" -import { EnvironmentConfigContext } from "./project" -import { schema, EnvironmentContext, ConfigContext, ErrorContext } from "./base" +import { RemoteSourceConfigContext } from "./project" +import { schema, ConfigContext, ErrorContext } from "./base" /** * This context is available for template strings in all workflow config fields except `name` and `triggers[]`. */ -export class WorkflowConfigContext extends EnvironmentConfigContext { - @schema( - EnvironmentContext.getSchema().description("Information about the environment that Garden is running against.") - ) - public environment: EnvironmentContext - - // Overriding to update the description. Same schema as base. - @schema( - joiVariables() - .description( - "A map of all variables defined in the project configuration, including environment-specific variables." - ) - .meta({ keyPlaceholder: "" }) - ) - public variables: DeepPrimitiveMap - - constructor(garden: Garden) { - super({ - projectName: garden.projectName, - projectRoot: garden.projectRoot, - artifactsPath: garden.artifactsPath, - branch: garden.vcsBranch, - username: garden.username, - variables: garden.variables, - loggedIn: !!garden.enterpriseApi, - enterpriseDomain: garden.enterpriseApi?.domain, - secrets: garden.secrets, - commandInfo: garden.commandInfo, - }) - - const fullEnvName = garden.namespace ? `${garden.namespace}.${garden.environmentName}` : garden.environmentName - this.environment = new EnvironmentContext(this, garden.environmentName, fullEnvName, garden.namespace) - } -} +export class WorkflowConfigContext extends RemoteSourceConfigContext {} class WorkflowStepContext extends ConfigContext { @schema(joi.string().description("The full output log from the step.")) diff --git a/core/src/docs/template-strings.ts b/core/src/docs/template-strings.ts index 37a515256b..ee80280f6d 100644 --- a/core/src/docs/template-strings.ts +++ b/core/src/docs/template-strings.ts @@ -11,7 +11,11 @@ import { TEMPLATES_DIR, renderTemplateStringReference } from "./config" import { readFileSync, writeFileSync } from "fs" import handlebars from "handlebars" import { GARDEN_CORE_ROOT } from "../constants" -import { ProjectConfigContext, EnvironmentConfigContext } from "../config/template-contexts/project" +import { + ProjectConfigContext, + EnvironmentConfigContext, + RemoteSourceConfigContext, +} from "../config/template-contexts/project" import { ProviderConfigContext } from "../config/template-contexts/provider" import { ModuleConfigContext, OutputConfigContext } from "../config/template-contexts/module" import { WorkflowStepConfigContext } from "../config/template-contexts/workflow" @@ -26,14 +30,18 @@ export function writeTemplateStringReferenceDocs(docsRoot: string) { schema: ProjectConfigContext.getSchema().required(), }) - const providerContext = renderTemplateStringReference({ - schema: ProviderConfigContext.getSchema().required(), + const remoteSourceContext = renderTemplateStringReference({ + schema: RemoteSourceConfigContext.getSchema().required(), }) const environmentContext = renderTemplateStringReference({ schema: EnvironmentConfigContext.getSchema().required(), }) + const providerContext = renderTemplateStringReference({ + schema: ProviderConfigContext.getSchema().required(), + }) + const moduleContext = renderTemplateStringReference({ schema: ModuleConfigContext.getSchema().required(), }) @@ -51,6 +59,7 @@ export function writeTemplateStringReferenceDocs(docsRoot: string) { const markdown = template({ helperFunctions: sortBy(Object.values(helperFunctions), "name"), projectContext, + remoteSourceContext, environmentContext, providerContext, moduleContext, diff --git a/core/src/garden.ts b/core/src/garden.ts index ae8d1d31ad..b362d78577 100644 --- a/core/src/garden.ts +++ b/core/src/garden.ts @@ -28,6 +28,7 @@ import { EnvironmentConfig, parseEnvironment, getDefaultEnvironmentName, + projectSourcesSchema, } from "./config/project" import { findByName, pickKeys, getPackageVersion, getNames, findByNames, duplicatesByKey, uuidv4 } from "./util/util" import { ConfigurationError, PluginError, RuntimeError } from "./exceptions" @@ -92,12 +93,13 @@ import { import { TemplatedModuleConfig } from "./plugins/templated" import { BuildDirRsync } from "./build-staging/rsync" import { EnterpriseApi } from "./enterprise/api" -import { DefaultEnvironmentContext } from "./config/template-contexts/project" +import { DefaultEnvironmentContext, RemoteSourceConfigContext } from "./config/template-contexts/project" import { OutputConfigContext } from "./config/template-contexts/module" import { ProviderConfigContext } from "./config/template-contexts/provider" import { getSecrets } from "./enterprise/get-secrets" import { killSyncDaemon } from "./plugins/kubernetes/mutagen" import { ConfigContext } from "./config/template-contexts/base" +import { validateSchema } from "./config/validation" export interface ActionHandlerMap { [actionName: string]: PluginActionHandlers[T] @@ -1023,8 +1025,11 @@ export class Garden { * Returns the configured project sources, and resolves any template strings on them. */ public getProjectSources() { - const context = new ProviderConfigContext(this, {}) - return resolveTemplateStrings(this.projectSources, context) + const context = new RemoteSourceConfigContext(this) + const resolved = validateSchema(resolveTemplateStrings(this.projectSources, context), projectSourcesSchema(), { + context: "remote source", + }) + return resolved } /** diff --git a/core/test/unit/src/config/project.ts b/core/test/unit/src/config/project.ts index fdd392edb6..c2efc9aa54 100644 --- a/core/test/unit/src/config/project.ts +++ b/core/test/unit/src/config/project.ts @@ -109,7 +109,7 @@ describe("resolveProjectConfig", () => { }) }) - it("should resolve template strings on fields other than environment and provider configs", async () => { + it("should resolve template strings on fields other than environments, providers and remote sources", async () => { const repositoryUrl = "git://github.com/foo/bar.git#boo" const defaultEnvironment = "default" @@ -141,6 +141,7 @@ describe("resolveProjectConfig", () => { platform: "${local.platform}", secret: "${secrets.foo}", projectPath: "${local.projectPath}", + envVar: "${local.env.TEST_ENV_VAR}", }, } @@ -173,7 +174,7 @@ describe("resolveProjectConfig", () => { outputs: [], sources: [ { - name: "foo", + name: "${local.env.TEST_ENV_VAR}", repositoryUrl, }, ], @@ -182,6 +183,7 @@ describe("resolveProjectConfig", () => { platform: platform(), secret: "banana", projectPath: config.path, + envVar: "foo", }, }) @@ -305,6 +307,71 @@ describe("resolveProjectConfig", () => { expect(result.environments[0].variables).to.eql(config.environments[0].variables) }) + it("should pass through templated fields on remote source configs", async () => { + const repositoryUrl = "git://github.com/foo/bar.git#boo" + const defaultEnvironment = "default" + + const config: ProjectConfig = { + apiVersion: DEFAULT_API_VERSION, + kind: "Project", + name: "my-project", + path: "/tmp/foo", + defaultEnvironment, + dotIgnoreFiles: defaultDotIgnoreFiles, + environments: [ + { + name: "default", + defaultNamespace, + variables: {}, + }, + ], + providers: [], + sources: [ + { + name: "${local.env.TEST_ENV_VAR}", + repositoryUrl, + }, + ], + variables: {}, + } + + process.env.TEST_ENV_VAR = "foo" + + expect( + resolveProjectConfig({ + defaultEnvironment, + config, + artifactsPath: "/tmp", + branch: "main", + username: "some-user", + loggedIn: true, + enterpriseDomain, + secrets: {}, + commandInfo, + }) + ).to.eql({ + ...config, + environments: [ + { + name: "default", + defaultNamespace, + variables: {}, + }, + ], + outputs: [], + sources: [ + { + name: "${local.env.TEST_ENV_VAR}", + repositoryUrl, + }, + ], + varfile: defaultVarfilePath, + variables: {}, + }) + + delete process.env.TEST_ENV_VAR + }) + it("should set defaultEnvironment to first environment if not configured", async () => { const defaultEnvironment = "" const config: ProjectConfig = { diff --git a/core/test/unit/src/garden.ts b/core/test/unit/src/garden.ts index 2d764e4ee2..642fbc4e87 100644 --- a/core/test/unit/src/garden.ts +++ b/core/test/unit/src/garden.ts @@ -2158,6 +2158,70 @@ describe("Garden", () => { }) }) + describe("getProjectSources", () => { + it("should correctly resolve template strings in remote source configs", async () => { + const remoteTag = "feature-branch" + process.env.TEST_ENV_VAR = "foo" + const garden = await makeTestGarden(pathFoo, { + config: { + apiVersion: DEFAULT_API_VERSION, + kind: "Project", + name: "test", + path: pathFoo, + defaultEnvironment: "default", + dotIgnoreFiles: [], + environments: [{ name: "default", defaultNamespace, variables: { remoteTag } }], + providers: [{ name: "test-plugin" }], + variables: { sourceName: "${local.env.TEST_ENV_VAR}" }, + sources: [ + { + name: "${var.sourceName}", + repositoryUrl: "git://github.com/foo/bar.git#${var.remoteTag}", + }, + ], + }, + }) + + const sources = garden.getProjectSources() + + expect(sources).to.eql([{ name: "foo", repositoryUrl: "git://github.com/foo/bar.git#feature-branch" }]) + + delete process.env.TEST_ENV_VAR + }) + + it("should validate the resolved remote sources", async () => { + const remoteTag = "feature-branch" + process.env.TEST_ENV_VAR = "foo" + const garden = await makeTestGarden(pathFoo, { + config: { + apiVersion: DEFAULT_API_VERSION, + kind: "Project", + name: "test", + path: pathFoo, + defaultEnvironment: "default", + dotIgnoreFiles: [], + environments: [{ name: "default", defaultNamespace, variables: { remoteTag } }], + providers: [{ name: "test-plugin" }], + variables: { sourceName: 123 }, + sources: [ + { + name: "${var.sourceName}", + repositoryUrl: "git://github.com/foo/bar.git#${var.remoteTag}", + }, + ], + }, + }) + + expectError( + () => garden.getProjectSources(), + (err) => + expect(stripAnsi(err.message)).to.equal("Error validating remote source: key [0][name] must be a string") + ) + + delete process.env.TEST_ENV_VAR + }) + }) + describe("scanForConfigs", () => { it("should find all garden configs in the project directory", async () => { const garden = await makeTestGardenA() diff --git a/docs/reference/template-strings.md b/docs/reference/template-strings.md index 714686af3e..ca5b8026d6 100644 --- a/docs/reference/template-strings.md +++ b/docs/reference/template-strings.md @@ -374,6 +374,287 @@ The secret's value. | `string` | +## Remote source configuration context + +The following keys are available in template strings under the `sources` key in project configs. + +### `${local.*}` + +Context variables that are specific to the currently running environment/machine. + +| Type | +| -------- | +| `object` | + +### `${local.artifactsPath}` + +The absolute path to the directory where exported artifacts from test and task runs are stored. + +| Type | +| -------- | +| `string` | + +Example: + +```yaml +my-variable: ${local.artifactsPath} +``` + +### `${local.env.*}` + +A map of all local environment variables (see https://nodejs.org/api/process.html#process_process_env). + +| Type | +| -------- | +| `object` | + +### `${local.env.}` + +The environment variable value. + +| Type | +| -------- | +| `string` | + +### `${local.platform}` + +A string indicating the platform that the framework is running on (see https://nodejs.org/api/process.html#process_process_platform) + +| Type | +| -------- | +| `string` | + +Example: + +```yaml +my-variable: ${local.platform} +``` + +### `${local.projectPath}` + +The absolute path to the project root directory. + +| Type | +| -------- | +| `string` | + +Example: + +```yaml +my-variable: ${local.projectPath} +``` + +### `${local.username}` + +The current username (as resolved by https://github.com/sindresorhus/username). + +| Type | +| -------- | +| `string` | + +Example: + +```yaml +my-variable: ${local.username} +``` + +### `${local.usernameLowerCase}` + +The current username (as resolved by https://github.com/sindresorhus/username), with any upper case characters converted to lower case. + +| Type | +| -------- | +| `string` | + +Example: + +```yaml +my-variable: ${local.usernameLowerCase} +``` + +### `${command.*}` + +Information about the currently running command and its arguments. + +| Type | +| -------- | +| `object` | + +### `${command.name}` + +The currently running Garden CLI command, without positional arguments or option flags. This can be handy to e.g. change some variables based on whether you're running `garden test` or some other specific command. + +Note that this will currently always resolve to `"run workflow"` when running Workflows, as opposed to individual workflow step commands. This may be revisited at a later time, but currently all configuration is resolved once for all workflow steps. + +| Type | +| -------- | +| `string` | + +Example: + +```yaml +my-variable: ${command.name} +``` + +### `${command.params.*}` + +A map of all parameters set when calling the current command. This includes both positional arguments and option flags, and includes any default values set by the framework or specific command. This can be powerful if used right, but do take care since different parameters are only available in certain commands, some have array values etc. + +For example, to see if a service is in hot-reload mode, you might do something like `${command.params contains 'hot-reload' && command.params.hot-reload contains 'my-service'}`. Notice that you currently need to check both for the existence of the parameter, and also to correctly handle the array value. + +Option values can be referenced by the option's default name (e.g. `dev-mode`) or its alias (e.g. `dev`) if one is defined for that option. + +| Type | +| -------- | +| `object` | + +### `${command.params.}` + +| Type | +| ----- | +| `any` | + +### `${project.*}` + +Information about the Garden project. + +| Type | +| -------- | +| `object` | + +### `${project.name}` + +The name of the Garden project. + +| Type | +| -------- | +| `string` | + +Example: + +```yaml +my-variable: ${project.name} +``` + +### `${git.*}` + +Information about the current state of the project's local git repository. + +| Type | +| -------- | +| `object` | + +### `${git.branch}` + +The current Git branch, if available. Resolves to an empty string if HEAD is in a detached state +(e.g. when rebasing), or if the repository has no commits. + +When using remote sources, the branch used is that of the project/top-level repository (the one that contains +the project configuration). + +The branch is computed at the start of the Garden command's execution, and is not updated if the current +branch changes during the command's execution (which could happen, for example, when using watch-mode +commands). + +| Type | +| -------- | +| `string` | + +Example: + +```yaml +my-variable: ${git.branch} +``` + +### `${secrets.}` + +The secret's value. + +| Type | +| -------- | +| `string` | + +### `${variables.*}` + +A map of all variables defined in the project configuration, including environment-specific variables. + +| Type | Default | +| -------- | ------- | +| `object` | `{}` | + +### `${variables.}` + +| Type | +| ------------------------------------------------ | +| `string | number | boolean | link | array[link]` | + +### `${var.*}` + +Alias for the variables field. + +| Type | Default | +| -------- | ------- | +| `object` | `{}` | + +### `${var.}` + +Number, string or boolean + +| Type | +| --------------------------- | +| `string | number | boolean` | + +### `${environment.*}` + +Information about the environment that Garden is running against. + +| Type | +| -------- | +| `object` | + +### `${environment.name}` + +The name of the environment Garden is running against, excluding the namespace. + +| Type | +| -------- | +| `string` | + +Example: + +```yaml +my-variable: ${environment.name} +``` + +### `${environment.fullName}` + +The full name of the environment Garden is running against, including the namespace. + +| Type | +| -------- | +| `string` | + +Example: + +```yaml +my-variable: ${environment.fullName} +``` + +### `${environment.namespace}` + +The currently active namespace (if any). + +| Type | +| -------- | +| `string` | + +Example: + +```yaml +my-variable: ${environment.namespace} +``` + + ## Environment configuration context The following keys are available in template strings under the `environments` key in project configs. Additional keys are available for the `environments[].providers` field, see the [Provider](#provider-configuration-context) section below for those. @@ -607,7 +888,7 @@ Number, string or boolean ## Provider configuration context -The following keys are available in template strings under the `providers` key (or `environments[].providers) in project configs. +The following keys are available in template strings under the `providers` key (or `environments[].providers`) in project configs. Providers can also reference outputs defined by other providers, via the `${providers..outputs}` key. For details on which outputs are available for a given provider, please refer to the [reference](https://docs.garden.io/reference/providers) docs for the provider in question, and look for the _Outputs_ section. diff --git a/static/docs/templates/template-strings.hbs b/static/docs/templates/template-strings.hbs index f2d73cc837..b0472ac571 100644 --- a/static/docs/templates/template-strings.hbs +++ b/static/docs/templates/template-strings.hbs @@ -29,6 +29,11 @@ Examples: The following keys are available in any template strings within project definitions in `garden.yml` config files, except the `name` field (which cannot be templated). See the [Environment](#environment-configuration-context) and [Provider](#provider-configuration-context) sections below for additional keys available when configuring `environments` and `providers`, respectively. {{{projectContext}}} +## Remote source configuration context + +The following keys are available in template strings under the `sources` key in project configs. +{{{remoteSourceContext}}} + ## Environment configuration context The following keys are available in template strings under the `environments` key in project configs. Additional keys are available for the `environments[].providers` field, see the [Provider](#provider-configuration-context) section below for those. @@ -36,7 +41,7 @@ The following keys are available in template strings under the `environments` k ## Provider configuration context -The following keys are available in template strings under the `providers` key (or `environments[].providers) in project configs. +The following keys are available in template strings under the `providers` key (or `environments[].providers`) in project configs. Providers can also reference outputs defined by other providers, via the `${providers..outputs}` key. For details on which outputs are available for a given provider, please refer to the [reference](https://docs.garden.io/reference/providers) docs for the provider in question, and look for the _Outputs_ section. {{{providerContext}}}