perf: optimise template string resolving performance

core/src/config/render-template.ts

@@ -273,29 +273,29 @@ async function renderConfigs({
   context: RenderTemplateConfigContext
   renderConfig: RenderTemplateConfig
 }): Promise<TemplatableConfig[]> {
+  const source = { yamlDoc: template.internal.yamlDoc, path: ["configs"] }
+
   const templateDescription = `${configTemplateKind} '${template.name}'`
   const templateConfigs = template.configs || []
   const partiallyResolvedTemplateConfigs = resolveTemplateStrings({
     value: templateConfigs,
     context,
     contextOpts: { allowPartial: true },
-    source: { yamlDoc: template.internal.yamlDoc, path: ["inputs"] },
+    source,
   })
 
   return Promise.all(
-    partiallyResolvedTemplateConfigs.map(async (m) => {
+    partiallyResolvedTemplateConfigs.map(async (m, index) => {
       // Resolve just the name, which must be immediately resolvable
       let resolvedName = m.name
 
       try {
-        const result = resolveTemplateString({ string: m.name, context, contextOpts: { allowPartial: false } })
-        if (typeof result === "string") {
-          resolvedName = result
-        } else {
-          throw new ConfigurationError({
-            message: "must resolve to string",
-          })
-        }
+        resolvedName = resolveTemplateString({
+          string: m.name,
+          context,
+          contextOpts: { allowPartial: false },
+          source: { ...source, path: [...source.path, index, "name"] },
+        }) as string
       } catch (error) {
         throw new ConfigurationError({
           message: `Could not resolve the \`name\` field (${m.name}) for a config in ${templateDescription}: ${error}\n\nNote that template strings in config names in must be fully resolvable at the time of scanning. This means that e.g. references to other actions, modules or runtime outputs cannot be used.`,

core/src/graph/actions.ts

@@ -7,7 +7,7 @@
  */
 
 import cloneDeep from "fast-copy"
-import { isEqual, mapValues, memoize, omit, pick, uniq } from "lodash-es"
+import { isEqual, isString, mapValues, memoize, omit, pick, uniq } from "lodash-es"
 import type {
   Action,
   ActionConfig,
@@ -68,6 +68,7 @@ import { profileAsync } from "../util/profiling.js"
 import { uuidv4 } from "../util/random.js"
 import { getSourcePath } from "../vcs/vcs.js"
 import { styles } from "../logger/styles.js"
+import { isUnresolvableValue } from "../template-string/static-analysis.js"
 
 function* sliceToBatches<T>(dict: Record<string, T>, batchSize: number) {
   const entries = Object.entries(dict)
@@ -1034,35 +1035,49 @@ function dependenciesFromActionConfig({
   for (const ref of getActionTemplateReferences(config, templateContext)) {
     let needsExecuted = false
 
-    const outputKey = ref.fullRef[4] as string
+    const outputType = ref.keyPath[0]
+
+    if (isUnresolvableValue(outputType)) {
+      const err = outputType.getError()
+      throw new ConfigurationError({
+        message: `Found invalid action reference: ${err}`,
+      })
+    }
+
+    const outputKey = ref.keyPath[1]
 
-    // also avoid execution when referencing the static output keys of the ref action type.
-    // e.g. a helm deploy referencing container build static output deploymentImageName
-    // ${actions.build.my-container.outputs.deploymentImageName}
     const refActionKey = actionReferenceToString(ref)
     const refActionType = configsByKey[refActionKey]?.type
-    let refStaticOutputKeys: string[] = []
-    if (refActionType) {
-      const refActionSpec = actionTypes[ref.kind][refActionType]?.spec
-      refStaticOutputKeys = refActionSpec?.staticOutputsSchema
-        ? describeSchema(refActionSpec.staticOutputsSchema).keys
-        : []
-    }
 
-    if (
-      ref.fullRef[3] === "outputs" &&
-      !staticOutputKeys.includes(outputKey) &&
-      !refStaticOutputKeys.includes(outputKey)
-    ) {
-      needsExecuted = true
+    if (outputType === "outputs") {
+      let refStaticOutputKeys: string[] = []
+      if (refActionType) {
+        const refActionSpec = actionTypes[ref.kind][refActionType]?.spec
+        refStaticOutputKeys = refActionSpec?.staticOutputsSchema
+          ? describeSchema(refActionSpec.staticOutputsSchema).keys
+          : []
+      }
+
+      // Avoid execution when referencing the static output keys of the ref action type.
+      // e.g. a helm deploy referencing container build static output deploymentImageName
+      // ${actions.build.my-container.outputs.deploymentImageName}
+      if (!isString(outputKey)) {
+        needsExecuted = true
+      } else {
+        needsExecuted = !staticOutputKeys.includes(outputKey) && !refStaticOutputKeys.includes(outputKey)
+      }
     }
 
     const refWithType = {
       ...ref,
       type: refActionType,
     }
 
-    addDep(refWithType, { explicit: false, needsExecutedOutputs: needsExecuted, needsStaticOutputs: !needsExecuted })
+    addDep(omit(refWithType, ["keyPath"]), {
+      explicit: false,
+      needsExecutedOutputs: needsExecuted,
+      needsStaticOutputs: !needsExecuted,
+    })
   }
 
   return deps

core/src/outputs.ts

@@ -53,12 +53,9 @@ export async function resolveProjectOutputs(garden: Garden, log: Log): Promise<O
     })
   )
   for (const finding of generator) {
-    if (finding.type === "unresolvable") {
-      continue
-    }
     const keyPath = finding.keyPath
     const refName = keyPath[1]
-    if (!refName) {
+    if (!refName || !isString(refName)) {
       continue
     }
 

core/src/template-string/static-analysis.ts

@@ -82,6 +82,16 @@ export function* visitAll({
   }
 }
 
+export class UnresolvableValue {
+  constructor(public readonly getError: () => GardenError) {}
+}
+
+export function isUnresolvableValue(
+  val: CollectionOrValue<TemplatePrimitive | UnresolvableValue>
+): val is UnresolvableValue {
+  return val instanceof UnresolvableValue
+}
+
 export type ContextLookupReferenceFinding =
   | {
       type: "resolvable"
@@ -90,7 +100,7 @@ export type ContextLookupReferenceFinding =
     }
   | {
       type: "unresolvable"
-      keyPath: (string | number | { getError: () => GardenError })[]
+      keyPath: (string | number | UnresolvableValue)[]
       yamlSource: ConfigSource
     }
 
@@ -126,17 +136,17 @@ export function* getContextLookupReferences(
         })
         if (typeof key === "symbol") {
           isResolvable = false
-          return {
-            getError: captureError(() =>
+          return new UnresolvableValue(
+            captureError(() =>
               // this will throw an error, because the key could not be resolved
               keyPathExpression.evaluate({
                 context,
                 opts: { allowPartial: false },
                 optional: false,
                 yamlSource,
               })
-            ),
-          }
+            )
+          )
         }
         return key
       })

core/src/template-string/template-string.ts

@@ -16,7 +16,7 @@ import {
   NoOpContext,
 } from "../config/template-contexts/base.js"
 import cloneDeep from "fast-copy"
-import { difference, isNumber, isPlainObject, isString } from "lodash-es"
+import { difference, isPlainObject, isString } from "lodash-es"
 import type { ActionReference, Primitive, StringMap } from "../config/common.js"
 import {
   arrayConcatKey,
@@ -44,8 +44,8 @@ import type { ObjectPath } from "../config/base.js"
 import type { TemplatePrimitive } from "./types.js"
 import * as ast from "./ast.js"
 import { LRUCache } from "lru-cache"
-import type { ContextLookupReferenceFinding } from "./static-analysis.js"
-import { getContextLookupReferences, visitAll } from "./static-analysis.js"
+import type { ContextLookupReferenceFinding, UnresolvableValue } from "./static-analysis.js"
+import { getContextLookupReferences, isUnresolvableValue, visitAll } from "./static-analysis.js"
 import type { ModuleConfig } from "../config/module.js"
 
 const escapePrefix = "$${"
@@ -609,113 +609,120 @@ export function mayContainTemplateString(obj: any): boolean {
 }
 
 interface ActionTemplateReference extends ActionReference {
-  fullRef: ContextKeySegment[]
+  keyPath: (ContextKeySegment | UnresolvableValue)[]
 }
 
 export function extractActionReference(finding: ContextLookupReferenceFinding): ActionTemplateReference {
-  if (finding.type === "unresolvable") {
-    for (const k of finding.keyPath) {
-      if (typeof k === "object" && "getError" in k) {
-        const err = k.getError()
-        throw new ConfigurationError({
-          message: `Found invalid action reference: ${err.message}`,
-        })
-      }
-    }
-    throw new InternalError({
-      message: "No error found in unresolvable finding",
+  const kind = finding.keyPath[1]
+  if (!kind) {
+    throw new ConfigurationError({
+      message: `Found invalid action reference (missing kind).`,
     })
   }
 
-  if (!finding.keyPath[1]) {
+  if (isUnresolvableValue(kind)) {
+    const err = kind.getError()
     throw new ConfigurationError({
-      message: `Found invalid action reference (missing kind).`,
+      message: `Found invalid action reference: ${err.message}`,
     })
   }
 
-  if (!isString(finding.keyPath[1])) {
+  if (!isString(kind)) {
     throw new ConfigurationError({
       message: `Found invalid action reference (kind is not a string).`,
     })
   }
 
-  if (!actionKindsLower.includes(finding.keyPath[1])) {
+  if (!actionKindsLower.includes(kind)) {
     throw new ConfigurationError({
-      message: `Found invalid action reference (invalid kind '${finding.keyPath[1]}')`,
+      message: `Found invalid action reference (invalid kind '${kind}')`,
     })
   }
 
-  if (!finding.keyPath[2]) {
+  const name = finding.keyPath[2]
+  if (!name) {
     throw new ConfigurationError({
       message: "Found invalid action reference (missing name)",
     })
   }
 
-  if (!isString(finding.keyPath[2])) {
+  if (isUnresolvableValue(name)) {
+    const err = name.getError()
+    throw new ConfigurationError({
+      message: `Found invalid action reference: ${err.message}`,
+    })
+  }
+
+  if (!isString(name)) {
     throw new ConfigurationError({
       message: "Found invalid action reference (name is not a string)",
     })
   }
 
   return {
-    kind: <ActionKind>titleize(finding.keyPath[1]),
-    name: finding.keyPath[2],
-    fullRef: finding.keyPath,
+    kind: <ActionKind>titleize(kind),
+    name,
+    keyPath: finding.keyPath.slice(3),
   }
 }
 
 export function extractRuntimeReference(finding: ContextLookupReferenceFinding): ActionTemplateReference {
-  if (finding.type === "unresolvable") {
-    for (const k of finding.keyPath) {
-      if (typeof k === "object" && "getError" in k) {
-        const err = k.getError()
-        throw new ConfigurationError({
-          message: `Found invalid action reference: ${err.message}`,
-        })
-      }
-    }
-    throw new InternalError({
-      message: "No error found in unresolvable finding",
+  const runtimeKind = finding.keyPath[1]
+  if (!runtimeKind) {
+    throw new ConfigurationError({
+      message: "Found invalid runtime reference (missing kind)",
     })
   }
 
-  if (!finding.keyPath[1]) {
+  if (isUnresolvableValue(runtimeKind)) {
+    const err = runtimeKind.getError()
     throw new ConfigurationError({
-      message: "Found invalid runtime reference (missing kind)",
+      message: `Found invalid runtime reference: ${err.message}`,
     })
   }
-  if (!isString(finding.keyPath[1])) {
+
+  if (!isString(runtimeKind)) {
     throw new ConfigurationError({
       message: "Found invalid runtime reference (kind is not a string)",
     })
   }
 
   let kind: ActionKind
-  if (finding.keyPath[1] === "services") {
+  if (runtimeKind === "services") {
     kind = "Deploy"
-  } else if (finding.keyPath[1] === "tasks") {
+  } else if (runtimeKind === "tasks") {
     kind = "Run"
   } else {
     throw new ConfigurationError({
-      message: `Found invalid runtime reference (invalid kind '${finding.keyPath[1]}')`,
+      message: `Found invalid runtime reference (invalid kind '${runtimeKind}')`,
     })
   }
 
-  if (!finding.keyPath[2]) {
+  const name = finding.keyPath[2]
+
+  if (!name) {
     throw new ConfigurationError({
       message: `Found invalid runtime reference (missing name)`,
     })
   }
-  if (!isString(finding.keyPath[2])) {
+
+  if (isUnresolvableValue(name)) {
+    const err = name.getError()
+    throw new ConfigurationError({
+      message: `Found invalid action reference: ${err.message}`,
+    })
+  }
+
+  if (!isString(name)) {
     throw new ConfigurationError({
       message: "Found invalid runtime reference (name is not a string)",
     })
   }
 
   return {
     kind,
-    name: finding.keyPath[2],
-    fullRef: finding.keyPath,
+    name,
+    keyPath: finding.keyPath.slice(3),
   }
 }
 
@@ -773,7 +780,7 @@ export function getModuleTemplateReferences(config: ModuleConfig, context: Modul
     }
 
     const moduleName = keyPath[1]
-    if (!isString(moduleName) && !isNumber(moduleName)) {
+    if (isUnresolvableValue(moduleName)) {
       const err = moduleName.getError()
       throw new ConfigurationError({
         message: `Found invalid module reference: ${err.message}`,

core/src/util/testing.ts

@@ -530,7 +530,17 @@ export function expectError(fn: Function, assertion: ExpectErrorAssertion = {})
       expectFuzzyMatch(
         errorMessage,
         contains,
-        `Assertion failed: '${errorMessage}' does not contain '${contains}'.\n\nOriginal error: ${toGardenError(err).stack}`
+        dedent`
+          Expected
+
+            '${stripAnsi(errorMessage).toLowerCase()}'
+
+          to contain
+
+            '${typeof contains === "string" ? stripAnsi(contains).toLowerCase() : contains.map(stripAnsi).map((s) => s.toLowerCase())}'
+
+          Original error:
+          ${stripAnsi(toGardenError(err).stack || "<no stack>")}`
       )
     }