feat(core): allow variables in remote sources

Template strings in remote sources now have access to the same variables
as provider configs.

This adds flexibility when using remote sources in conjunction with
automation (e.g. when using dynamic Git branches / tags for remote
sources, depending on project- and environment-level variables).

core/src/config/project.ts

@@ -418,16 +418,16 @@ export function resolveProjectConfig({
   secrets: PrimitiveMap
   commandInfo: CommandInfo
 }): ProjectConfig {
-  // Resolve template strings for non-environment-specific fields
-  const { environments = [], name } = config
+  // Resolve template strings for non-environment-specific fields (apart from `sources`).
+  const { environments = [], name, sources = [] } = config
 
   const globalConfig = resolveTemplateStrings(
     {
       apiVersion: config.apiVersion,
-      sources: config.sources,
       varfile: config.varfile,
       variables: config.variables,
       environments: [],
+      sources: [],
     },
     new ProjectConfigContext({
       projectName: name,
@@ -450,6 +450,7 @@ export function resolveProjectConfig({
       name,
       defaultEnvironment,
       environments: [],
+      sources: [],
     },
     schema: projectSchema(),
     configType: "project",
@@ -477,6 +478,7 @@ export function resolveProjectConfig({
     ...config,
     environments: config.environments || [],
     providers,
+    sources,
   }
 
   config.defaultEnvironment = getDefaultEnvironmentName(defaultEnvironment, config)

core/src/config/template-contexts/project.ts

@@ -11,8 +11,9 @@ import chalk from "chalk"
 import { PrimitiveMap, joiIdentifierMap, joiStringMap, joiPrimitive, DeepPrimitiveMap, joiVariables } from "../common"
 import { joi } from "../common"
 import { deline, dedent } from "../../util/string"
-import { schema, ConfigContext, ContextKeySegment } from "./base"
+import { schema, ConfigContext, ContextKeySegment, EnvironmentContext } from "./base"
 import { CommandInfo } from "../../plugin-context"
+import { Garden } from "../../garden"
 
 class LocalContext extends ConfigContext {
   @schema(
@@ -298,3 +299,38 @@ export class EnvironmentConfigContext extends ProjectConfigContext {
     this.variables = this.var = params.variables
   }
 }
+
+export class RemoteSourceConfigContext extends EnvironmentConfigContext {
+  @schema(
+    EnvironmentContext.getSchema().description("Information about the environment that Garden is running against.")
+  )
+  public environment: EnvironmentContext
+
+  // Overriding to update the description. Same schema as base.
+  @schema(
+    joiVariables()
+      .description(
+        "A map of all variables defined in the project configuration, including environment-specific variables."
+      )
+      .meta({ keyPlaceholder: "<variable-name>" })
+  )
+  public variables: DeepPrimitiveMap
+
+  constructor(garden: Garden) {
+    super({
+      projectName: garden.projectName,
+      projectRoot: garden.projectRoot,
+      artifactsPath: garden.artifactsPath,
+      branch: garden.vcsBranch,
+      username: garden.username,
+      variables: garden.variables,
+      loggedIn: !!garden.enterpriseApi,
+      enterpriseDomain: garden.enterpriseApi?.domain,
+      secrets: garden.secrets,
+      commandInfo: garden.commandInfo,
+    })
+
+    const fullEnvName = garden.namespace ? `${garden.namespace}.${garden.environmentName}` : garden.environmentName
+    this.environment = new EnvironmentContext(this, garden.environmentName, fullEnvName, garden.namespace)
+  }
+}

core/src/config/template-contexts/workflow.ts

@@ -10,46 +10,13 @@ import { joiIdentifierMap, DeepPrimitiveMap, joiVariables } from "../common"
 import { Garden } from "../../garden"
 import { joi } from "../common"
 import { dedent } from "../../util/string"
-import { EnvironmentConfigContext } from "./project"
-import { schema, EnvironmentContext, ConfigContext, ErrorContext } from "./base"
+import { RemoteSourceConfigContext } from "./project"
+import { schema, ConfigContext, ErrorContext } from "./base"
 
 /**
  * This context is available for template strings in all workflow config fields except `name` and `triggers[]`.
  */
-export class WorkflowConfigContext extends EnvironmentConfigContext {
-  @schema(
-    EnvironmentContext.getSchema().description("Information about the environment that Garden is running against.")
-  )
-  public environment: EnvironmentContext
-
-  // Overriding to update the description. Same schema as base.
-  @schema(
-    joiVariables()
-      .description(
-        "A map of all variables defined in the project configuration, including environment-specific variables."
-      )
-      .meta({ keyPlaceholder: "<variable-name>" })
-  )
-  public variables: DeepPrimitiveMap
-
-  constructor(garden: Garden) {
-    super({
-      projectName: garden.projectName,
-      projectRoot: garden.projectRoot,
-      artifactsPath: garden.artifactsPath,
-      branch: garden.vcsBranch,
-      username: garden.username,
-      variables: garden.variables,
-      loggedIn: !!garden.enterpriseApi,
-      enterpriseDomain: garden.enterpriseApi?.domain,
-      secrets: garden.secrets,
-      commandInfo: garden.commandInfo,
-    })
-
-    const fullEnvName = garden.namespace ? `${garden.namespace}.${garden.environmentName}` : garden.environmentName
-    this.environment = new EnvironmentContext(this, garden.environmentName, fullEnvName, garden.namespace)
-  }
-}
+export class WorkflowConfigContext extends RemoteSourceConfigContext {}
 
 class WorkflowStepContext extends ConfigContext {
   @schema(joi.string().description("The full output log from the step."))

core/src/docs/template-strings.ts

@@ -11,7 +11,11 @@ import { TEMPLATES_DIR, renderTemplateStringReference } from "./config"
 import { readFileSync, writeFileSync } from "fs"
 import handlebars from "handlebars"
 import { GARDEN_CORE_ROOT } from "../constants"
-import { ProjectConfigContext, EnvironmentConfigContext } from "../config/template-contexts/project"
+import {
+  ProjectConfigContext,
+  EnvironmentConfigContext,
+  RemoteSourceConfigContext,
+} from "../config/template-contexts/project"
 import { ProviderConfigContext } from "../config/template-contexts/provider"
 import { ModuleConfigContext, OutputConfigContext } from "../config/template-contexts/module"
 import { WorkflowStepConfigContext } from "../config/template-contexts/workflow"
@@ -26,14 +30,18 @@ export function writeTemplateStringReferenceDocs(docsRoot: string) {
     schema: ProjectConfigContext.getSchema().required(),
   })
 
-  const providerContext = renderTemplateStringReference({
-    schema: ProviderConfigContext.getSchema().required(),
+  const remoteSourceContext = renderTemplateStringReference({
+    schema: RemoteSourceConfigContext.getSchema().required(),
   })
 
   const environmentContext = renderTemplateStringReference({
     schema: EnvironmentConfigContext.getSchema().required(),
   })
 
+  const providerContext = renderTemplateStringReference({
+    schema: ProviderConfigContext.getSchema().required(),
+  })
+
   const moduleContext = renderTemplateStringReference({
     schema: ModuleConfigContext.getSchema().required(),
   })
@@ -51,6 +59,7 @@ export function writeTemplateStringReferenceDocs(docsRoot: string) {
   const markdown = template({
     helperFunctions: sortBy(Object.values(helperFunctions), "name"),
     projectContext,
+    remoteSourceContext,
     environmentContext,
     providerContext,
     moduleContext,

core/src/garden.ts

@@ -28,6 +28,7 @@ import {
   EnvironmentConfig,
   parseEnvironment,
   getDefaultEnvironmentName,
+  projectSourcesSchema,
 } from "./config/project"
 import { findByName, pickKeys, getPackageVersion, getNames, findByNames, duplicatesByKey, uuidv4 } from "./util/util"
 import { ConfigurationError, PluginError, RuntimeError } from "./exceptions"
@@ -92,12 +93,13 @@ import {
 import { TemplatedModuleConfig } from "./plugins/templated"
 import { BuildDirRsync } from "./build-staging/rsync"
 import { EnterpriseApi } from "./enterprise/api"
-import { DefaultEnvironmentContext } from "./config/template-contexts/project"
+import { DefaultEnvironmentContext, RemoteSourceConfigContext } from "./config/template-contexts/project"
 import { OutputConfigContext } from "./config/template-contexts/module"
 import { ProviderConfigContext } from "./config/template-contexts/provider"
 import { getSecrets } from "./enterprise/get-secrets"
 import { killSyncDaemon } from "./plugins/kubernetes/mutagen"
 import { ConfigContext } from "./config/template-contexts/base"
+import { validateSchema } from "./config/validation"
 
 export interface ActionHandlerMap<T extends keyof PluginActionHandlers> {
   [actionName: string]: PluginActionHandlers[T]
@@ -1023,8 +1025,11 @@ export class Garden {
    * Returns the configured project sources, and resolves any template strings on them.
    */
   public getProjectSources() {
-    const context = new ProviderConfigContext(this, {})
-    return resolveTemplateStrings(this.projectSources, context)
+    const context = new RemoteSourceConfigContext(this)
+    const resolved = validateSchema(resolveTemplateStrings(this.projectSources, context), projectSourcesSchema(), {
+      context: "remote source",
+    })
+    return resolved
   }
 
   /**

core/test/unit/src/config/project.ts

@@ -109,7 +109,7 @@ describe("resolveProjectConfig", () => {
     })
   })
 
-  it("should resolve template strings on fields other than environment and provider configs", async () => {
+  it("should resolve template strings on fields other than environments, providers and remote sources", async () => {
     const repositoryUrl = "git://github.com/foo/bar.git#boo"
     const defaultEnvironment = "default"
 
@@ -141,6 +141,7 @@ describe("resolveProjectConfig", () => {
         platform: "${local.platform}",
         secret: "${secrets.foo}",
         projectPath: "${local.projectPath}",
+        envVar: "${local.env.TEST_ENV_VAR}",
       },
     }
 
@@ -173,7 +174,7 @@ describe("resolveProjectConfig", () => {
       outputs: [],
       sources: [
         {
-          name: "foo",
+          name: "${local.env.TEST_ENV_VAR}",
           repositoryUrl,
         },
       ],
@@ -182,6 +183,7 @@ describe("resolveProjectConfig", () => {
         platform: platform(),
         secret: "banana",
         projectPath: config.path,
+        envVar: "foo",
       },
     })
 
@@ -305,6 +307,71 @@ describe("resolveProjectConfig", () => {
     expect(result.environments[0].variables).to.eql(config.environments[0].variables)
   })
 
+  it("should pass through templated fields on remote source configs", async () => {
+    const repositoryUrl = "git://github.com/foo/bar.git#boo"
+    const defaultEnvironment = "default"
+
+    const config: ProjectConfig = {
+      apiVersion: DEFAULT_API_VERSION,
+      kind: "Project",
+      name: "my-project",
+      path: "/tmp/foo",
+      defaultEnvironment,
+      dotIgnoreFiles: defaultDotIgnoreFiles,
+      environments: [
+        {
+          name: "default",
+          defaultNamespace,
+          variables: {},
+        },
+      ],
+      providers: [],
+      sources: [
+        {
+          name: "${local.env.TEST_ENV_VAR}",
+          repositoryUrl,
+        },
+      ],
+      variables: {},
+    }
+
+    process.env.TEST_ENV_VAR = "foo"
+
+    expect(
+      resolveProjectConfig({
+        defaultEnvironment,
+        config,
+        artifactsPath: "/tmp",
+        branch: "main",
+        username: "some-user",
+        loggedIn: true,
+        enterpriseDomain,
+        secrets: {},
+        commandInfo,
+      })
+    ).to.eql({
+      ...config,
+      environments: [
+        {
+          name: "default",
+          defaultNamespace,
+          variables: {},
+        },
+      ],
+      outputs: [],
+      sources: [
+        {
+          name: "${local.env.TEST_ENV_VAR}",
+          repositoryUrl,
+        },
+      ],
+      varfile: defaultVarfilePath,
+      variables: {},
+    })
+
+    delete process.env.TEST_ENV_VAR
+  })
+
   it("should set defaultEnvironment to first environment if not configured", async () => {
     const defaultEnvironment = ""
     const config: ProjectConfig = {

core/test/unit/src/garden.ts

@@ -2158,6 +2158,70 @@ describe("Garden", () => {
     })
   })
 
+  describe("getProjectSources", () => {
+    it("should correctly resolve template strings in remote source configs", async () => {
+      const remoteTag = "feature-branch"
+      process.env.TEST_ENV_VAR = "foo"
+      const garden = await makeTestGarden(pathFoo, {
+        config: {
+          apiVersion: DEFAULT_API_VERSION,
+          kind: "Project",
+          name: "test",
+          path: pathFoo,
+          defaultEnvironment: "default",
+          dotIgnoreFiles: [],
+          environments: [{ name: "default", defaultNamespace, variables: { remoteTag } }],
+          providers: [{ name: "test-plugin" }],
+          variables: { sourceName: "${local.env.TEST_ENV_VAR}" },
+          sources: [
+            {
+              name: "${var.sourceName}",
+              repositoryUrl: "git://github.com/foo/bar.git#${var.remoteTag}",
+            },
+          ],
+        },
+      })
+
+      const sources = garden.getProjectSources()
+
+      expect(sources).to.eql([{ name: "foo", repositoryUrl: "git://github.com/foo/bar.git#feature-branch" }])
+
+      delete process.env.TEST_ENV_VAR
+    })
+
+    it("should validate the resolved remote sources", async () => {
+      const remoteTag = "feature-branch"
+      process.env.TEST_ENV_VAR = "foo"
+      const garden = await makeTestGarden(pathFoo, {
+        config: {
+          apiVersion: DEFAULT_API_VERSION,
+          kind: "Project",
+          name: "test",
+          path: pathFoo,
+          defaultEnvironment: "default",
+          dotIgnoreFiles: [],
+          environments: [{ name: "default", defaultNamespace, variables: { remoteTag } }],
+          providers: [{ name: "test-plugin" }],
+          variables: { sourceName: 123 },
+          sources: [
+            {
+              name: "${var.sourceName}",
+              repositoryUrl: "git://github.com/foo/bar.git#${var.remoteTag}",
+            },
+          ],
+        },
+      })
+
+      expectError(
+        () => garden.getProjectSources(),
+        (err) =>
+          expect(stripAnsi(err.message)).to.equal("Error validating remote source: key [0][name] must be a string")
+      )
+
+      delete process.env.TEST_ENV_VAR
+    })
+  })
+
   describe("scanForConfigs", () => {
     it("should find all garden configs in the project directory", async () => {
       const garden = await makeTestGardenA()