WIP: input tracking

core/src/commands/get/get-modules.ts

@@ -11,7 +11,7 @@ import { Command } from "../base.js"
 import { StringsParameter, BooleanParameter } from "../../cli/params.js"
 import type { GardenModule } from "../../types/module.js"
 import { moduleSchema } from "../../types/module.js"
-import { keyBy, omit, sortBy } from "lodash-es"
+import { isString, keyBy, omit, sortBy } from "lodash-es"
 import type { StringMap } from "../../config/common.js"
 import { joiIdentifierMap, createSchema } from "../../config/common.js"
 import { printEmoji, printHeader, renderDivider } from "../../logger/util.js"
@@ -21,8 +21,9 @@ import { relative, sep } from "path"
 import type { Garden } from "../../index.js"
 import type { Log } from "../../logger/log-entry.js"
 import { highlightYaml, safeDumpYaml } from "../../util/serialization.js"
-import { deepMap } from "../../util/objects.js"
+import { CollectionOrValue, deepMap } from "../../util/objects.js"
 import { styles } from "../../logger/styles.js"
+import { TemplatePrimitive } from "../../template-string/inputs.js"
 
 const getModulesArgs = {
   modules: new StringsParameter({
@@ -158,11 +159,11 @@ function getRelativeModulePath(projectRoot: string, modulePath: string): string
  *
  * Used for sanitizing output that may contain secret values.
  */
-function filterSecrets<T extends object>(object: T, secrets: StringMap): T {
+function filterSecrets<T extends CollectionOrValue<TemplatePrimitive>>(object: T, secrets: StringMap): T {
   const secretValues = new Set(Object.values(secrets))
   const secretNames = Object.keys(secrets)
   const sanitized = <T>deepMap(object, (value) => {
-    if (secretValues.has(value)) {
+    if (isString(value) && secretValues.has(value)) {
       const name = secretNames.find((n) => secrets[n] === value)!
       return `[filtered secret: ${name}]`
     } else {

core/src/config/template-contexts/base.ts

@@ -8,23 +8,27 @@
 
 import type Joi from "@hapi/joi"
 import { isString } from "lodash-es"
-import { ConfigurationError } from "../../exceptions.js"
-import {
-  resolveTemplateString,
-  TemplateStringMissingKeyException,
-  TemplateStringPassthroughException,
-} from "../../template-string/template-string.js"
+import { ConfigurationError, NotFoundError } from "../../exceptions.js"
+import { resolveTemplateString } from "../../template-string/template-string.js"
 import type { CustomObjectSchema } from "../common.js"
 import { isPrimitive, joi, joiIdentifier } from "../common.js"
 import { KeyedSet } from "../../util/keyed-set.js"
 import { naturalList } from "../../util/string.js"
 import { styles } from "../../logger/styles.js"
+import type { TemplateValue } from "../../template-string/inputs.js"
+import { TemplateLeaf, isTemplateLeafValue, isTemplateLeaf } from "../../template-string/inputs.js"
+import type { CollectionOrValue } from "../../util/objects.js"
+import { deepMap } from "../../util/objects.js"
+import { LazyValue } from "../../template-string/lazy.js"
 
 export type ContextKeySegment = string | number
 export type ContextKey = ContextKeySegment[]
 
+export type ObjectPath = (string | number)[]
+
 export interface ContextResolveOpts {
   // Allow templates to be partially resolved (used to defer runtime template resolution, for example)
+  // TODO: rename to optional
   allowPartial?: boolean
   // a list of previously resolved paths, used to detect circular references
   stack?: string[]
@@ -41,7 +45,10 @@ export interface ContextResolveParams {
 export interface ContextResolveOutput {
   message?: string
   partial?: boolean
-  resolved: any
+  result: CollectionOrValue<TemplateValue>
+  cached: boolean
+  // for input tracking
+  // ResolvedResult: ResolvedResult
 }
 
 export function schema(joiSchema: Joi.Schema) {
@@ -56,9 +63,10 @@ export interface ConfigContextType {
 }
 
 // Note: we're using classes here to be able to use decorators to describe each context node and key
+// TODO-steffen&thor: Make all instance variables of all config context classes read-only.
 export abstract class ConfigContext {
   private readonly _rootContext: ConfigContext
-  private readonly _resolvedValues: { [path: string]: any }
+  private readonly _resolvedValues: { [path: string]: CollectionOrValue<TemplateValue> }
 
   // This is used for special-casing e.g. runtime.* resolution
   protected _alwaysAllowPartial: boolean
@@ -87,10 +95,10 @@ export abstract class ConfigContext {
     const fullPath = renderKeyPath(nodePath.concat(key))
 
     // if the key has previously been resolved, return it directly
-    const resolved = this._resolvedValues[path]
+    const cachedResult = this._resolvedValues[path]
 
-    if (resolved) {
-      return { resolved }
+    if (cachedResult) {
+      return { cached: true, result: cachedResult }
     }
 
     opts.stack = [...(opts.stack || [])]
@@ -150,9 +158,14 @@ export abstract class ConfigContext {
         if (remainder.length > 0) {
           opts.stack.push(stackEntry)
           const res = value.resolve({ key: remainder, nodePath: nestedNodePath, opts })
-          value = res.resolved
+          value = res.result
           message = res.message
           partial = !!res.partial
+        } else {
+          // TODO: improve error message
+          throw new ConfigurationError({
+            message: `Resolving to a context is not allowed.`,
+          })
         }
         break
       }
@@ -163,6 +176,10 @@ export abstract class ConfigContext {
         value = resolveTemplateString({ string: value, context: this._rootContext, contextOpts: opts })
       }
 
+      if (isTemplateLeaf(value) || value instanceof LazyValue) {
+        break
+      }
+
       if (value === undefined) {
         break
       }
@@ -186,31 +203,55 @@ export abstract class ConfigContext {
         }
       }
 
-      // If we're allowing partial strings, we throw the error immediately to end the resolution flow. The error
-      // is caught in the surrounding template resolution code.
-      if (this._alwaysAllowPartial) {
-        // We use a separate exception type when contexts are specifically indicating that unresolvable keys should
-        // be passed through. This is caught in the template parser code.
-        throw new TemplateStringPassthroughException({
-          message,
-        })
-      } else if (opts.allowPartial) {
-        throw new TemplateStringMissingKeyException({
-          message,
-        })
+      if (!opts.allowPartial && !this._alwaysAllowPartial) {
+        throw new NotFoundError({ message })
       } else {
-        // Otherwise we return the undefined value, so that any logical expressions can be evaluated appropriately.
-        // The template resolver will throw the error later if appropriate.
-        return { resolved: undefined, message }
+        return {
+          message,
+          cached: false,
+          result: new TemplateLeaf({
+            expr: undefined,
+            value: undefined,
+            inputs: {},
+          }),
+        }
       }
     }
 
+    let result: CollectionOrValue<TemplateValue>
+
+    if (value instanceof LazyValue) {
+      result = value
+    } else if (isTemplateLeaf(value)) {
+      result = value
+    }
+    // Wrap normal data using deepMap
+    else if (isTemplateLeafValue(value)) {
+      result = new TemplateLeaf({
+        expr: undefined,
+        value,
+        inputs: {},
+      })
+    } else {
+      // value is a collection
+      result = deepMap(value, (v) => {
+        if (isTemplateLeaf(v) || v instanceof LazyValue) {
+          return v
+        }
+        return new TemplateLeaf({
+          expr: undefined,
+          value: v,
+          inputs: {},
+        })
+      })
+    }
+
     // Cache result, unless it is a partial resolution
     if (!partial) {
-      this._resolvedValues[path] = value
+      this._resolvedValues[path] = result
     }
 
-    return { resolved: value }
+    return { cached: false, result }
   }
 }
 
@@ -242,7 +283,11 @@ export class ScanContext extends ConfigContext {
   override resolve({ key, nodePath }: ContextResolveParams) {
     const fullKey = nodePath.concat(key)
     this.foundKeys.add(fullKey)
-    return { resolved: renderTemplateString(fullKey), partial: true }
+    return {
+      partial: true,
+      cached: false,
+      result: new TemplateLeaf({ value: renderTemplateString(fullKey), expr: undefined, inputs: {} }),
+    }
   }
 }
 
@@ -319,7 +364,7 @@ function renderTemplateString(key: ContextKeySegment[]) {
 /**
  * Given all the segments of a template string, return a string path for the key.
  */
-function renderKeyPath(key: ContextKeySegment[]): string {
+export function renderKeyPath(key: ContextKeySegment[]): string {
   // Note: We don't support bracket notation for the first part in a template string
   if (key.length === 0) {
     return ""

core/src/exceptions.ts

@@ -16,6 +16,9 @@ import indentString from "indent-string"
 import { constants } from "os"
 import dns from "node:dns"
 import { styles } from "./logger/styles.js"
+import { Location } from "./template-string/ast.js"
+import { ObjectPath } from "./config/template-contexts/base.js"
+import { ConfigSource } from "./config/validation.js"
 
 // Unfortunately, NodeJS does not provide a list of all error codes, so we have to maintain this list manually.
 // See https://nodejs.org/docs/latest-v18.x/api/dns.html#error-codes
@@ -300,14 +303,28 @@ export class CloudApiError extends GardenError {
   }
 }
 
+export class TemplateFunctionCallError extends GardenError {
+  type = "template-function-call"
+
+  constructor({ message, yamlPath, source }: { message: string; yamlPath?: ObjectPath; source?: ConfigSource }) {
+    super({ message })
+  }
+}
+
+type TemplateStringErrorParams = {
+  message: string
+  rawTemplateString: string
+  loc: Location
+}
 export class TemplateStringError extends GardenError {
   type = "template-string"
 
-  path?: (string | number)[]
-
-  constructor(params: GardenErrorParams & { path?: (string | number)[] }) {
-    super(params)
-    this.path = params.path
+  // TODO: improve error message, using position, yamlPath and source
+  constructor({ message, rawTemplateString, loc }: TemplateStringErrorParams) {
+    const prefix = `Invalid template string (${styles.accent(truncate(rawTemplateString, { length: 35 }).replace(/\n/g, "\\n"))}): `
+    super({
+      message: message.startsWith(prefix) ? message : `${prefix}${message}`,
+    })
   }
 }