feat(pulumi): add new pulumi varfile schema that allows specifying ot…

…her top-level keys (#6729)

* feat(pulumi): add new pulumi varfile schema that allows specifying other top-level keys

* docs: generate reference docs

* chore: fix import

* chore: fix dependency in test

* chore: move new vars test to own dir

* chore: fix wrong comparison in test

* chore: add backend url to test comparison

docs/pulumi-plugin/about.md

@@ -81,6 +81,32 @@ For example:
 garden plugins pulumi preview -- my-pulumi-deploy-action my-other-pulumi-deploy-action
 ```
 
+## Pulumi varfile schema
+
+By default Garden uses a schema for the Pulumi varfiles that expects all content in a varfile to be values that are set under the `config` key in the pulumi config file. Garden will resolve any template strings in varfiles and then add them to the `config` section of the pulumi config file.
+
+Example of old varfile schema:
+
+```
+kubernetes:context: orbstack
+pulumi-k8s:namespace: ns-from-the-varfile
+```
+
+Since this does not allow for setting other top-level values like e.g. `secretsprovider` this schema has been updated and now requires config values to be referenced under the `config` key.
+
+Example of new varfile schema:
+
+```
+secretsprovider: gcpkms://projects/xyz/locations/global/keyRings/pulumi/cryptoKeys/pulumi-secrets
+encryptedkey: 123456
+config:
+  kubernetes:context: orbstack
+  pulumi-k8s:namespace: ns-from-the-varfile
+```
+
+To use the new schema in a single pulumi deploy action set `action.spec.useNewPulumiVarfileSchema` to `true`. To use
+the new schema for all of your pulumi deploy actions set `useNewPulumiVarfileSchema` in your pulumi provider to `true`. The flag will be removed in the next major release and the new schema will be used.
+
 ## Next steps
 
 Check out the [`pulumi` example](../../examples/pulumi) project.

docs/reference/action-types/Deploy/pulumi.md

@@ -319,6 +319,24 @@ Specify the path to the Pulumi project root, relative to the deploy action's roo
 | ----------- | ------- | -------- |
 | `posixPath` | `"."`   | No       |
 
+### `spec.useNewPulumiVarfileSchema`
+
+[spec](#spec) > useNewPulumiVarfileSchema
+
+If set to true, the deploy action will use the new Pulumi varfile schema, which does not nest all variables under
+the 'config' key automatically like the old schema. This allow setting variables at the root level of the varfile
+that don't belong to the 'config' key. Example:
+```
+config:
+  myVar: value
+secretsprovider: gcpkms://projects/xyz/locations/global/keyRings/pulumi/cryptoKeys/pulumi-secrets
+```
+For more information see [this guide on pulumi varfiles and variables](https://docs.garden.io/pulumi-plugin/about#pulumi-varfile-schema)
+
+| Type      | Default | Required |
+| --------- | ------- | -------- |
+| `boolean` | `false` | No       |
+
 ### `spec.pulumiVariables`
 
 [spec](#spec) > pulumiVariables
@@ -340,7 +358,7 @@ Instead, use pulumi stack references when using the `cacheStatus` config option.
 
 [spec](#spec) > pulumiVarfiles
 
-Specify one or more paths (relative to the deploy action's root) to YAML files containing pulumi config variables.
+Specify one or more paths (relative to the deploy action's root) to YAML files containing pulumi configuration.
 
 Templated paths that resolve to `null`, `undefined` or an empty string are ignored.
 
@@ -351,8 +369,8 @@ value type.
 
 If one or more varfiles is not found, no error is thrown (that varfile path is simply ignored).
 
-Note: There is no need to nest the variables under a `config` field as is done in a pulumi
-config. Simply specify all the config variables at the top level.
+Note: The old varfile schema nests all variables under the 'config' key automatically. If you need to set variables
+at the root level of the varfile that don't belong to the 'config' key, set `useNewPulumiVarfileSchema` to true.
 
 | Type               | Default | Required |
 | ------------------ | ------- | -------- |

docs/reference/module-types/pulumi.md

@@ -188,6 +188,16 @@ createStack: false
 # Specify the path to the Pulumi project root, relative to the deploy action's root.
 root: .
 
+# If set to true, the deploy action will use the new Pulumi varfile schema, which does not nest all variables under
+# the 'config' key automatically like the old schema. This allow setting variables at the root level of the varfile
+# that don't belong to the 'config' key. Example:
+# config:
+#   myVar: value
+# secretsprovider: gcpkms://projects/xyz/locations/global/keyRings/pulumi/cryptoKeys/pulumi-secrets
+# For more information see [this guide on pulumi varfiles and
+# variables](https://docs.garden.io/pulumi-plugin/about#pulumi-varfile-schema)
+useNewPulumiVarfileSchema: false
+
 # A map of config variables to use when applying the stack. These are merged with the contents of any `pulumiVarfiles`
 # provided
 # for this deploy action. The deploy action's stack config will be overwritten with the resulting merged config.
@@ -201,7 +211,7 @@ root: .
 # Instead, use pulumi stack references when using the `cacheStatus` config option.
 pulumiVariables: {}
 
-# Specify one or more paths (relative to the deploy action's root) to YAML files containing pulumi config variables.
+# Specify one or more paths (relative to the deploy action's root) to YAML files containing pulumi configuration.
 #
 # Templated paths that resolve to `null`, `undefined` or an empty string are ignored.
 #
@@ -212,8 +222,8 @@ pulumiVariables: {}
 #
 # If one or more varfiles is not found, no error is thrown (that varfile path is simply ignored).
 #
-# Note: There is no need to nest the variables under a `config` field as is done in a pulumi
-# config. Simply specify all the config variables at the top level.
+# Note: The old varfile schema nests all variables under the 'config' key automatically. If you need to set variables
+# at the root level of the varfile that don't belong to the 'config' key, set `useNewPulumiVarfileSchema` to true.
 pulumiVarfiles: []
 
 # The name of the pulumi organization to use. Overrides the `orgName` set on the pulumi provider (if any).
@@ -605,6 +615,22 @@ Specify the path to the Pulumi project root, relative to the deploy action's roo
 | ----------- | ------- | -------- |
 | `posixPath` | `"."`   | No       |
 
+### `useNewPulumiVarfileSchema`
+
+If set to true, the deploy action will use the new Pulumi varfile schema, which does not nest all variables under
+the 'config' key automatically like the old schema. This allow setting variables at the root level of the varfile
+that don't belong to the 'config' key. Example:
+```
+config:
+  myVar: value
+secretsprovider: gcpkms://projects/xyz/locations/global/keyRings/pulumi/cryptoKeys/pulumi-secrets
+```
+For more information see [this guide on pulumi varfiles and variables](https://docs.garden.io/pulumi-plugin/about#pulumi-varfile-schema)
+
+| Type      | Default | Required |
+| --------- | ------- | -------- |
+| `boolean` | `false` | No       |
+
 ### `pulumiVariables`
 
 A map of config variables to use when applying the stack. These are merged with the contents of any `pulumiVarfiles` provided
@@ -622,7 +648,7 @@ Instead, use pulumi stack references when using the `cacheStatus` config option.
 
 ### `pulumiVarfiles[]`
 
-Specify one or more paths (relative to the deploy action's root) to YAML files containing pulumi config variables.
+Specify one or more paths (relative to the deploy action's root) to YAML files containing pulumi configuration.
 
 Templated paths that resolve to `null`, `undefined` or an empty string are ignored.
 
@@ -633,8 +659,8 @@ value type.
 
 If one or more varfiles is not found, no error is thrown (that varfile path is simply ignored).
 
-Note: There is no need to nest the variables under a `config` field as is done in a pulumi
-config. Simply specify all the config variables at the top level.
+Note: The old varfile schema nests all variables under the 'config' key automatically. If you need to set variables
+at the root level of the varfile that don't belong to the 'config' key, set `useNewPulumiVarfileSchema` to true.
 
 | Type               | Default | Required |
 | ------------------ | ------- | -------- |

docs/reference/providers/pulumi.md

@@ -59,6 +59,18 @@ providers:
     #
     # Note: This limit is not applied when running built-in commands (e.g. `garden deploy`).
     pluginTaskConcurrencyLimit: 5
+
+    # If set to true, the deploy action will use the new Pulumi varfile schema, which does not nest all variables
+    # under
+    # the 'config' key automatically like the old schema. This allow setting variables at the root level of the
+    # varfile
+    # that don't belong to the 'config' key. Example:
+    # config:
+    #   myVar: value
+    # secretsprovider: gcpkms://projects/xyz/locations/global/keyRings/pulumi/cryptoKeys/pulumi-secrets
+    # For more information see [this guide on pulumi varfiles and
+    # variables](https://docs.garden.io/pulumi-plugin/about#pulumi-varfile-schema)
+    useNewPulumiVarfileSchema: false
 ```
 ## Configuration Keys
 
@@ -184,3 +196,21 @@ Note: This limit is not applied when running built-in commands (e.g. `garden dep
 | -------- | ------- | -------- |
 | `number` | `5`     | No       |
 
+### `providers[].useNewPulumiVarfileSchema`
+
+[providers](#providers) > useNewPulumiVarfileSchema
+
+If set to true, the deploy action will use the new Pulumi varfile schema, which does not nest all variables under
+the 'config' key automatically like the old schema. This allow setting variables at the root level of the varfile
+that don't belong to the 'config' key. Example:
+```
+config:
+  myVar: value
+secretsprovider: gcpkms://projects/xyz/locations/global/keyRings/pulumi/cryptoKeys/pulumi-secrets
+```
+For more information see [this guide on pulumi varfiles and variables](https://docs.garden.io/pulumi-plugin/about#pulumi-varfile-schema)
+
+| Type      | Default | Required |
+| --------- | ------- | -------- |
+| `boolean` | `false` | No       |
+

examples/pulumi/k8s-deployment/varfile1.yaml

@@ -1 +1,2 @@
-pulumi-k8s:appName: api-varfile1-override
+config:
+  pulumi-k8s:appName: api-varfile1-override

examples/pulumi/k8s-deployment/varfile2.yaml

@@ -1 +1,2 @@
-pulumi-k8s:appName: api-varfile2-override
+config:
+  pulumi-k8s:appName: api-varfile2-override

examples/pulumi/k8s-namespace/Pulumi.k8s-namespace.yaml

@@ -1,5 +1,6 @@
 config:
-  kubernetes:context: docker-desktop
-  pulumi-k8s:namespace: pulumi-k8s
+  kubernetes:context: orbstack
+  pulumi-k8s:namespace: fooooooooo
 backend:
   url: https://api.pulumi.com
+test: foo

examples/pulumi/k8s-namespace/garden.yml

@@ -6,6 +6,8 @@ spec:
   createStack: true
   cacheStatus: true
   stack: k8s-namespace
+  pulumiVarfiles:
+    - pulumi-vars.yaml
   pulumiVariables:
-    kubernetes:context: docker-desktop
+    kubernetes:context: orbstack
     pulumi-k8s:namespace: pulumi-k8s

examples/pulumi/k8s-namespace/pulumi-vars.yaml

@@ -0,0 +1,3 @@
+config:
+  kubernetes:context: orbstack
+  pulumi-k8s:namespace: ns-from-the-varfile

examples/pulumi/project.garden.yml

@@ -10,6 +10,7 @@ providers:
     # Ensure that the node SDK is installed for the k8s-namespace and k8s-deployment projects before we run pulumi.
     initScript: "for dir in k8s-namespace k8s-deployment; do [ ! -d $dir/node_modules ] && cd $dir && npm install && cd ..; done"
   - name: pulumi
+    useNewPulumiVarfileSchema: true
     dependencies: [exec]
     environments: [local]
     orgName: ${var.pulumiAppOrg}

plugins/pulumi/src/action.ts

@@ -47,6 +47,17 @@ export const pulumiDeploySchemaKeys = () => ({
   root: joi.posixPath().subPathOnly().default(".").description(dedent`
     Specify the path to the Pulumi project root, relative to the deploy action's root.
     `),
+  useNewPulumiVarfileSchema: joi.boolean().default(false).description(dedent`
+    If set to true, the deploy action will use the new Pulumi varfile schema, which does not nest all variables under
+    the 'config' key automatically like the old schema. This allow setting variables at the root level of the varfile
+    that don't belong to the 'config' key. Example:
+    \`\`\`
+    config:
+      myVar: value
+    secretsprovider: gcpkms://projects/xyz/locations/global/keyRings/pulumi/cryptoKeys/pulumi-secrets
+    \`\`\`
+    For more information see [this guide on pulumi varfiles and variables](https://docs.garden.io/pulumi-plugin/about#pulumi-varfile-schema)
+    `),
   pulumiVariables: joiVariables().default({}).description(dedent`
     A map of config variables to use when applying the stack. These are merged with the contents of any \`pulumiVarfiles\` provided
     for this deploy action. The deploy action's stack config will be overwritten with the resulting merged config.
@@ -59,7 +70,7 @@ export const pulumiDeploySchemaKeys = () => ({
     `),
   pulumiVarfiles: joiSparseArray(joi.posixPath().pattern(yamlFileRegex)).description(
     dedent`
-      Specify one or more paths (relative to the deploy action's root) to YAML files containing pulumi config variables.
+      Specify one or more paths (relative to the deploy action's root) to YAML files containing pulumi configuration.
 
       Templated paths that resolve to \`null\`, \`undefined\` or an empty string are ignored.
 
@@ -70,8 +81,8 @@ export const pulumiDeploySchemaKeys = () => ({
 
       If one or more varfiles is not found, no error is thrown (that varfile path is simply ignored).
 
-      Note: There is no need to nest the variables under a \`config\` field as is done in a pulumi
-      config. Simply specify all the config variables at the top level.
+      Note: The old varfile schema nests all variables under the 'config' key automatically. If you need to set variables
+      at the root level of the varfile that don't belong to the 'config' key, set \`useNewPulumiVarfileSchema\` to true.
         `
   ),
   orgName: joi.string().optional().empty(["", null]).description(dedent`

plugins/pulumi/src/helpers.ts

@@ -22,7 +22,7 @@ import {
   PluginError,
 } from "@garden-io/sdk/build/src/exceptions.js"
 import { dumpYaml } from "@garden-io/core/build/src/util/serialization.js"
-import type { DeepPrimitiveMap } from "@garden-io/core/build/src/config/common.js"
+import type { DeepPrimitiveMap, Primitive } from "@garden-io/core/build/src/config/common.js"
 import { loadAndValidateYaml } from "@garden-io/core/build/src/config/base.js"
 import { getPluginOutputsPath } from "@garden-io/sdk"
 import type { Log, PluginContext } from "@garden-io/sdk/build/src/types.js"
@@ -44,6 +44,7 @@ export interface PulumiParams {
 export interface PulumiConfig {
   config: DeepPrimitiveMap
   backend?: { url: string }
+  [key: string]: DeepPrimitiveMap | Primitive | undefined
 }
 
 interface PulumiManifest {
@@ -254,7 +255,7 @@ export function getActionStackRoot(action: Resolved<PulumiDeploy>): string {
  * For convenience, returns the path to the action's stack config file.
  */
 export async function applyConfig(params: PulumiParams & { previewDirPath?: string }): Promise<string> {
-  const { ctx, action, log } = params
+  const { ctx, action, log, provider } = params
   await ensureOutputDirs(ctx)
 
   const stackConfigPath = getStackConfigPath(action, ctx.environmentName)
@@ -298,20 +299,35 @@ export async function applyConfig(params: PulumiParams & { previewDirPath?: stri
   log.debug(`pulumiVariables from action: ${JSON.stringify(pulumiVars, null, 2)}`)
   log.debug(`varfileContents: ${JSON.stringify(varfileContents, null, 2)}`)
 
-  // Pulumi varfiles take precedence over action.spec.pulumiVariables, and are merged in declaration order.
   // Pulumi variables (from action.spec.pulumiVariables) take precedence over any variables declared in pulumi varfiles.
   let vars: DeepPrimitiveMap = {}
-  for (const varfileVars of varfileContents) {
-    vars = <DeepPrimitiveMap>merge(vars, varfileVars)
-  }
-  vars = <DeepPrimitiveMap>merge(vars, pulumiVars || {})
-  log.debug(`merged vars: ${JSON.stringify(vars, null, 2)}`)
-  stackConfig.config = vars
 
+  if (action.getSpec()["useNewPulumiVarfileSchema"] || provider.config.useNewPulumiVarfileSchema) {
+    for (const varfileVars of varfileContents) {
+      vars = <DeepPrimitiveMap>merge(vars, varfileVars)
+    }
+    stackConfig.config = <DeepPrimitiveMap>merge(vars.config, pulumiVars || {})
+    log.debug(`merged vars: ${JSON.stringify(stackConfig, null, 2)}`)
+    Object.entries(vars).forEach(([key, value]) => {
+      if (key !== "config") {
+        stackConfig[key] = value as Primitive | DeepPrimitiveMap
+      }
+    })
+  } else {
+    log.warn(dedent`
+      The old Pulumi varfile schema is deprecated and will be removed in a future version of Garden. 
+      For more information see: https://docs.garden.io/pulumi-plugin/about#pulumi-varfile-schema
+    `)
+    for (const varfileVars of varfileContents) {
+      vars = <DeepPrimitiveMap>merge(vars, varfileVars)
+    }
+    vars = <DeepPrimitiveMap>merge(vars, pulumiVars || {})
+    log.debug(`merged vars: ${JSON.stringify(vars, null, 2)}`)
+    stackConfig.config = vars
+  }
   stackConfig.backend = { url: params.provider.config.backendURL }
-
   if (stackConfigFileExists && isEmpty(vars)) {
-    log.debug(deline`
+    log.debug(dedent`
       stack config file exists but no variables are defined in pulumiVars or pulumiVarfiles - skip writing stack config
     `)
   } else {

plugins/pulumi/src/provider.ts

@@ -61,5 +61,16 @@ export const pulumiProviderConfigSchema = providerConfigBaseSchema()
 
       Note: This limit is not applied when running built-in commands (e.g. \`garden deploy\`).
     `),
+    useNewPulumiVarfileSchema: joi.boolean().default(false).description(dedent`
+      If set to true, the deploy action will use the new Pulumi varfile schema, which does not nest all variables under
+      the 'config' key automatically like the old schema. This allow setting variables at the root level of the varfile
+      that don't belong to the 'config' key. Example:
+      \`\`\`
+      config:
+        myVar: value
+      secretsprovider: gcpkms://projects/xyz/locations/global/keyRings/pulumi/cryptoKeys/pulumi-secrets
+      \`\`\`
+      For more information see [this guide on pulumi varfiles and variables](https://docs.garden.io/pulumi-plugin/about#pulumi-varfile-schema)
+      `),
   })
   .unknown(false)

plugins/pulumi/test/handlers.ts

@@ -28,8 +28,8 @@ const moduleDirName = dirname(fileURLToPath(import.meta.url))
 // TODO: Find a better way to do this.
 const projectRoot = resolve(moduleDirName, "../../test/", "test-project-k8s")
 
-const nsModuleRoot = join(projectRoot, "k8s-namespace")
-const deploymentModuleRoot = join(projectRoot, "k8s-deployment")
+const nsActionRoot = join(projectRoot, "k8s-namespace")
+const deploymentActionRoot = join(projectRoot, "k8s-deployment")
 
 // Note: By default, this test suite assumes that PULUMI_ACCESS_TOKEN is present in the environment (which is the case
 // in CI). To run this test suite with your own pulumi org, replace the `orgName` variable in
@@ -42,7 +42,7 @@ describe("pulumi plugin handlers", () => {
   let provider: PulumiProvider
 
   before(async () => {
-    await ensureNodeModules([nsModuleRoot, deploymentModuleRoot])
+    await ensureNodeModules([nsActionRoot, deploymentActionRoot])
     const plugin = pulumiPlugin()
     garden = await makeTestGarden(projectRoot, { plugins: [plugin] })
     log = garden.log