From b47e2323ce71b4a3213b68ee386bdab2d61a7323 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Mon, 2 Dec 2024 18:02:58 +0100 Subject: [PATCH 01/43] perf: optimise template string resolving performance We optimise the template string resolving performance by making sure that we only parse each template expression once. Once parsed, we use the AST to evaluate template expressions. --- core/src/commands/custom.ts | 8 +- core/src/commands/get/get-modules.ts | 2 +- core/src/commands/workflow.ts | 2 +- core/src/config/base.ts | 4 +- core/src/config/project.ts | 2 +- core/src/config/render-template.ts | 9 +- core/src/config/template-contexts/base.ts | 21 +- core/src/config/validation.ts | 15 +- core/src/exceptions.ts | 9 +- core/src/garden.ts | 4 +- core/src/graph/actions.ts | 7 +- core/src/resolve-module.ts | 18 +- core/src/template-string/ast.ts | 738 ++++++++++++++++++++ core/src/template-string/functions.ts | 112 +-- core/src/template-string/parser.pegjs | 412 ++++++----- core/src/template-string/static-analysis.ts | 86 +++ core/src/template-string/template-string.ts | 439 +++--------- core/src/template-string/types.ts | 18 + core/src/util/objects.ts | 51 +- 19 files changed, 1314 insertions(+), 643 deletions(-) create mode 100644 core/src/template-string/ast.ts create mode 100644 core/src/template-string/static-analysis.ts create mode 100644 core/src/template-string/types.ts diff --git a/core/src/commands/custom.ts b/core/src/commands/custom.ts index 28d52d439b..52976e8592 100644 --- a/core/src/commands/custom.ts +++ b/core/src/commands/custom.ts @@ -133,7 +133,7 @@ export class CustomCommandWrapper extends Command { if (this.spec.exec) { const startedAt = new Date() - const exec = validateWithPath({ + const exec = validateWithPath({ config: resolveTemplateStrings({ value: this.spec.exec, context: commandContext, @@ -144,7 +144,7 @@ export class CustomCommandWrapper extends Command { projectRoot: garden.projectRoot, configType: `exec field in custom Command '${this.name}'`, source: undefined, - }) + })! const command = exec.command log.debug(`Running exec command: ${command.join(" ")}`) @@ -185,7 +185,7 @@ export class CustomCommandWrapper extends Command { if (this.spec.gardenCommand) { const startedAt = new Date() - let gardenCommand = validateWithPath({ + let gardenCommand = validateWithPath({ config: resolveTemplateStrings({ value: this.spec.gardenCommand, context: commandContext, @@ -196,7 +196,7 @@ export class CustomCommandWrapper extends Command { projectRoot: garden.projectRoot, configType: `gardenCommand field in custom Command '${this.name}'`, source: undefined, - }) + })! log.debug(`Running Garden command: ${gardenCommand.join(" ")}`) diff --git a/core/src/commands/get/get-modules.ts b/core/src/commands/get/get-modules.ts index 4c5812d2a0..26743b712b 100644 --- a/core/src/commands/get/get-modules.ts +++ b/core/src/commands/get/get-modules.ts @@ -168,7 +168,7 @@ function filterSecrets(object: T, secrets: StringMap): T { const secretValues = new Set(Object.values(secrets)) const secretNames = Object.keys(secrets) const sanitized = deepMap(object, (value) => { - if (secretValues.has(value)) { + if (typeof value === "string" && secretValues.has(value)) { const name = secretNames.find((n) => secrets[n] === value)! return `[filtered secret: ${name}]` } else { diff --git a/core/src/commands/workflow.ts b/core/src/commands/workflow.ts index 4f46324396..5bdd2f7df1 100644 --- a/core/src/commands/workflow.ts +++ b/core/src/commands/workflow.ts @@ -180,7 +180,7 @@ export class WorkflowCommand extends Command { stepResult = await runStepCommand(stepParams) } else if (step.script) { - step.script = resolveTemplateString({ string: step.script, context: stepTemplateContext }) + step.script = resolveTemplateString({ string: step.script, context: stepTemplateContext }) as string stepResult = await runStepScript(stepParams) } else { stepResult = undefined diff --git a/core/src/config/base.ts b/core/src/config/base.ts index 6850bd9979..91724e6028 100644 --- a/core/src/config/base.ts +++ b/core/src/config/base.ts @@ -557,9 +557,9 @@ const _readFile = profileAsync(async function _readFile(path: string) { return await readFile(path) }) -const _loadYaml = profile(function _loadYaml(data: Buffer) { +function _loadYaml(data: Buffer) { return load(data.toString()) as PrimitiveMap -}) +} const loadVarfileCache = new LRUCache>({ max: 10000, diff --git a/core/src/config/project.ts b/core/src/config/project.ts index adaaef644e..9b0bc06ccd 100644 --- a/core/src/config/project.ts +++ b/core/src/config/project.ts @@ -642,7 +642,7 @@ export const pickEnvironment = profileAsync(async function _pickEnvironment({ source, }) - environmentConfig = validateWithPath({ + environmentConfig = validateWithPath({ config: environmentConfig, schema: environmentSchema(), configType: `environment ${environment}`, diff --git a/core/src/config/render-template.ts b/core/src/config/render-template.ts index 5b303af870..37d5f4e1d2 100644 --- a/core/src/config/render-template.ts +++ b/core/src/config/render-template.ts @@ -288,7 +288,14 @@ async function renderConfigs({ let resolvedName = m.name try { - resolvedName = resolveTemplateString({ string: m.name, context, contextOpts: { allowPartial: false } }) + const result = resolveTemplateString({ string: m.name, context, contextOpts: { allowPartial: false } }) + if (typeof result === "string") { + resolvedName = result + } else { + throw new ConfigurationError({ + message: "must resolve to string", + }) + } } catch (error) { throw new ConfigurationError({ message: `Could not resolve the \`name\` field (${m.name}) for a config in ${templateDescription}: ${error}\n\nNote that template strings in config names in must be fully resolvable at the time of scanning. This means that e.g. references to other actions, modules or runtime outputs cannot be used.`, diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index 26deb71a06..c1a0e8f642 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -11,8 +11,6 @@ import { isString } from "lodash-es" import { ConfigurationError } from "../../exceptions.js" import { resolveTemplateString, - TemplateStringMissingKeyException, - TemplateStringPassthroughException, } from "../../template-string/template-string.js" import type { CustomObjectSchema } from "../common.js" import { isPrimitive, joi, joiIdentifier } from "../common.js" @@ -57,6 +55,9 @@ export interface ConfigContextType { getSchema(): CustomObjectSchema } +export const CONTEXT_RESOLVE_KEY_NOT_FOUND: unique symbol = Symbol.for("ContextResolveKeyNotFound") +export const CONTEXT_RESOLVE_KEY_AVAILABLE_LATER: unique symbol = Symbol.for("ContextResolveKeyAvailableLater") + // Note: we're using classes here to be able to use decorators to describe each context node and key @Profile() export abstract class ConfigContext { @@ -193,16 +194,10 @@ export abstract class ConfigContext { // If we're allowing partial strings, we throw the error immediately to end the resolution flow. The error // is caught in the surrounding template resolution code. - if (this._alwaysAllowPartial) { - // We use a separate exception type when contexts are specifically indicating that unresolvable keys should - // be passed through. This is caught in the template parser code. - throw new TemplateStringPassthroughException({ - message, - }) - } else if (opts.allowPartial) { - throw new TemplateStringMissingKeyException({ - message, - }) + if (this._alwaysAllowPartial || opts.allowPartial) { + return { + resolved: CONTEXT_RESOLVE_KEY_NOT_FOUND, + } } else { // Otherwise we return the undefined value, so that any logical expressions can be evaluated appropriately. // The template resolver will throw the error later if appropriate. @@ -324,7 +319,7 @@ function renderTemplateString(key: ContextKeySegment[]) { /** * Given all the segments of a template string, return a string path for the key. */ -function renderKeyPath(key: ContextKeySegment[]): string { +export function renderKeyPath(key: ContextKeySegment[]): string { // Note: We don't support bracket notation for the first part in a template string if (key.length === 0) { return "" diff --git a/core/src/config/validation.ts b/core/src/config/validation.ts index ef46f88f5f..5783e52b43 100644 --- a/core/src/config/validation.ts +++ b/core/src/config/validation.ts @@ -10,7 +10,6 @@ import type Joi from "@hapi/joi" import { ConfigurationError } from "../exceptions.js" import { relative } from "path" import { uuidv4 } from "../util/random.js" -import { profile } from "../util/profiling.js" import type { BaseGardenResource, ObjectPath, YamlDocumentWithSource } from "./base.js" import type { ParsedNode, Range } from "yaml" import { padEnd } from "lodash-es" @@ -51,8 +50,8 @@ export interface ValidateOptions { source?: ConfigSource } -export interface ValidateWithPathParams { - config: T +export interface ValidateWithPathParams { + config: unknown schema: Joi.Schema path: string // Absolute path to the config file, including filename projectRoot: string @@ -68,7 +67,7 @@ export interface ValidateWithPathParams { * * This is to ensure consistent error messages that include the relative path to the failing file. */ -export const validateWithPath = profile(function $validateWithPath({ +export function validateWithPath({ config, schema, path, @@ -77,7 +76,7 @@ export const validateWithPath = profile(function $validateWithPath({ configType, ErrorClass, source, -}: ValidateWithPathParams) { +}: ValidateWithPathParams) { const context = `${configType} ${name ? `'${name}' ` : ""}` + `${path && projectRoot !== path ? "(" + relative(projectRoot, path) + ")" : ""}` @@ -92,7 +91,7 @@ export const validateWithPath = profile(function $validateWithPath({ } return validateSchema(config, schema, validateOpts) -}) +} export interface ValidateConfigParams { config: T @@ -119,7 +118,7 @@ export function validateConfig(params: ValidateCon }) } -export const validateSchema = profile(function $validateSchema( +export function validateSchema( value: T, schema: Joi.Schema, { source, context = "", ErrorClass = ConfigurationError }: ValidateOptions = {} @@ -168,7 +167,7 @@ export const validateSchema = profile(function $validateSchema( throw new ErrorClass({ message: `${msgPrefix}:\n\n${errorDescription}`, }) -}) +} export interface ArtifactSpec { source: string diff --git a/core/src/exceptions.ts b/core/src/exceptions.ts index e4e1ba6d89..fc5561ff1c 100644 --- a/core/src/exceptions.ts +++ b/core/src/exceptions.ts @@ -18,6 +18,7 @@ import dns from "node:dns" import { styles } from "./logger/styles.js" import type { ObjectPath } from "./config/base.js" import type { ExecaError } from "execa" +import { Location } from "./template-string/ast.js" // Unfortunately, NodeJS does not provide a list of all error codes, so we have to maintain this list manually. // See https://nodejs.org/docs/latest-v18.x/api/dns.html#error-codes @@ -309,11 +310,13 @@ export class CloudApiError extends GardenError { export class TemplateStringError extends GardenError { type = "template-string" - path?: ObjectPath + loc?: Location + rawTemplateString: string - constructor(params: GardenErrorParams & { path?: ObjectPath }) { + constructor(params: GardenErrorParams & { rawTemplateString: string, loc?: Location }) { super(params) - this.path = params.path + this.loc = params.loc + this.rawTemplateString = params.rawTemplateString } } diff --git a/core/src/garden.ts b/core/src/garden.ts index 13b455b1ea..40bf3b0284 100644 --- a/core/src/garden.ts +++ b/core/src/garden.ts @@ -1540,11 +1540,13 @@ export class Garden { }) } - return resolveTemplateString({ + const resolved = resolveTemplateString({ string: disabledFlag, context, contextOpts: { allowPartial: false }, }) + + return !!resolved } /** diff --git a/core/src/graph/actions.ts b/core/src/graph/actions.ts index 09ade4dbb3..8f23aa9cf2 100644 --- a/core/src/graph/actions.ts +++ b/core/src/graph/actions.ts @@ -945,7 +945,7 @@ export const preprocessActionConfig = profileAsync(async function preprocessActi } }) -const dependenciesFromActionConfig = profile(function dependenciesFromActionConfig({ +function dependenciesFromActionConfig({ log, config, configsByKey, @@ -1042,11 +1042,12 @@ const dependenciesFromActionConfig = profile(function dependenciesFromActionConf if (maybeTemplateString(ref.name)) { try { + // TODO: validate that we actually resolve to a string ref.name = resolveTemplateString({ string: ref.name, context: templateContext, contextOpts: { allowPartial: false }, - }) + }) as string } catch (err) { log.warn( `Unable to infer dependency from action reference in ${description}, because template string '${ref.name}' could not be resolved. Either fix the dependency or specify it explicitly.` @@ -1084,4 +1085,4 @@ const dependenciesFromActionConfig = profile(function dependenciesFromActionConf } return deps -}) +} diff --git a/core/src/resolve-module.ts b/core/src/resolve-module.ts index a35614f807..5733776b74 100644 --- a/core/src/resolve-module.ts +++ b/core/src/resolve-module.ts @@ -23,6 +23,7 @@ import { CircularDependenciesError, ConfigurationError, FilesystemError, + InternalError, PluginError, toGardenError, } from "./exceptions.js" @@ -62,6 +63,8 @@ import { styles } from "./logger/styles.js" import { actionReferenceToString } from "./actions/base.js" import type { DepGraph } from "dependency-graph" import { minimatch } from "minimatch" +import { CollectionOrValue } from "./util/objects.js" +import { TemplatePrimitive } from "./template-string/types.js" // This limit is fairly arbitrary, but we need to have some cap on concurrent processing. export const moduleResolutionConcurrencyLimit = 50 @@ -528,7 +531,9 @@ export class ModuleResolver { const configContext = new ModuleConfigContext(contextParams) - const templateRefs = getModuleTemplateReferences(rawConfig, configContext) + const templateRefs = getModuleTemplateReferences(rawConfig, + configContext + ) const templateDeps = templateRefs.filter((d) => d[1] !== rawConfig.name).map((d) => d[1]) // This is a bit of a hack, but we need to store the template dependencies on the raw config so we can check @@ -840,6 +845,12 @@ export class ModuleResolver { ? resolveTemplateString({ string: contents, context: configContext, contextOpts: { unescape: true } }) : contents + if (typeof resolvedContents !== "string") { + throw new InternalError({ + message: `Expected resolvedContents to be typeof string, but got typeof ${resolvedContents}`, + }) + } + const targetDir = resolve(resolvedConfig.path, ...posix.dirname(fileSpec.targetPath).split(posix.sep)) const targetPath = resolve(resolvedConfig.path, ...fileSpec.targetPath.split(posix.sep)) @@ -951,6 +962,11 @@ export class ModuleResolver { context: moduleConfigContext, contextOpts: resolveOpts, }) + if (typeof varfilePath !== "string") { + throw new ConfigurationError({ + message: `Expected varfile template expression in module configuration ${config.name} to resolve to string, actually got ${typeof varfilePath}`, + }) + } varfileVars = await loadVarfile({ configRoot: config.path, path: varfilePath, diff --git a/core/src/template-string/ast.ts b/core/src/template-string/ast.ts new file mode 100644 index 0000000000..4b220df78e --- /dev/null +++ b/core/src/template-string/ast.ts @@ -0,0 +1,738 @@ +/* + * Copyright (C) 2018-2023 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { isArray, isNumber, isString } from "lodash-es" +import { CONTEXT_RESOLVE_KEY_AVAILABLE_LATER, CONTEXT_RESOLVE_KEY_NOT_FOUND, renderKeyPath, type ConfigContext, type ContextResolveOpts } from "../config/template-contexts/base.js" +import { InternalError, TemplateStringError } from "../exceptions.js" +import { getHelperFunctions } from "./functions.js" +import { isTemplatePrimitive, type TemplatePrimitive } from "./types.js" +import { Collection, CollectionOrValue } from "../util/objects.js" +import { ConfigSource, validateSchema } from "../config/validation.js" +import { TemplateExpressionGenerator } from "./static-analysis.js" + +type EvaluateArgs = { + context: ConfigContext + opts: ContextResolveOpts + rawTemplateString: string + + /** + * Whether or not to throw an error if ContextLookupExpression fails to resolve variable. + * The FormatStringExpression will set this parameter based on wether the OptionalSuffix (?) is present or not. + */ + optional?: boolean +} + +/** + * Returned by the `location()` helper in PEG.js. + */ +export type Location = { + start: { + offset: number + line: number + column: number + } + end: { + offset: number + line: number + column: number + } + source?: ConfigSource +} + +export type TemplateEvaluationResult = TemplatePrimitive | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND + +function* astVisitAll(e: TemplateExpression): TemplateExpressionGenerator { + for (const key in e) { + if (key === "loc") { + continue + } + const propertyValue = e[key] + if (propertyValue instanceof TemplateExpression) { + yield propertyValue + yield* astVisitAll(propertyValue) + } else if (Array.isArray(propertyValue)) { + for (const item of propertyValue) { + if (item instanceof TemplateExpression) { + yield item + yield* astVisitAll(item) + } + } + } + } +} + +export abstract class TemplateExpression { + constructor(public readonly loc: Location) {} + + *visitAll(): TemplateExpressionGenerator { + yield* astVisitAll(this) + } + + abstract evaluate(args: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND +} + +export class IdentifierExpression extends TemplateExpression { + constructor( + loc: Location, + public readonly name: string + ) { + if (!isString(name)) { + throw new InternalError({ + message: `IdentifierExpression name must be a string. Got: ${typeof name}`, + }) + } + super(loc) + } + + override evaluate(): string { + return this.name + } +} + +export class LiteralExpression extends TemplateExpression { + constructor( + loc: Location, + public readonly literal: TemplatePrimitive + ) { + super(loc) + } + + override evaluate(args: EvaluateArgs): TemplatePrimitive { + return this.literal + } +} + +export class ArrayLiteralExpression extends TemplateExpression { + constructor( + loc: Location, + // an ArrayLiteralExpression consists of several template expressions, + // for example other literal expressions and context lookup expressions. + public readonly literal: TemplateExpression[] + ) { + super(loc) + } + + override evaluate(args: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND { + const result: CollectionOrValue = [] + for (const e of this.literal) { + const res = e.evaluate(args) + if (res === CONTEXT_RESOLVE_KEY_NOT_FOUND) { + return res + } + result.push(res) + } + + return result + } +} + +export abstract class UnaryExpression extends TemplateExpression { + constructor( + loc: Location, + public readonly innerExpression: TemplateExpression + ) { + super(loc) + } + + override evaluate(args: EvaluateArgs): TemplatePrimitive | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND { + const inner = this.innerExpression.evaluate(args) + + if (inner === CONTEXT_RESOLVE_KEY_NOT_FOUND) { + return inner + } + + return this.transform(inner) + } + + abstract transform(value: CollectionOrValue): TemplatePrimitive +} + +export class TypeofExpression extends UnaryExpression { + override transform(value: CollectionOrValue): string { + return typeof value + } +} + +export class NotExpression extends UnaryExpression { + override transform(value: CollectionOrValue): boolean { + return !value + } +} + +export abstract class LogicalExpression extends TemplateExpression { + constructor( + loc: Location, + public readonly operator: string, + public readonly left: TemplateExpression, + public readonly right: TemplateExpression + ) { + super(loc) + } +} + +// you need to call with unwrap: isTruthy(unwrap(value)) +export function isTruthy(v: CollectionOrValue): boolean { + if (isTemplatePrimitive(v)) { + return !!v + } else { + // collections are truthy, regardless wether they are empty or not. + v satisfies Collection + return true + } +} + +export class LogicalOrExpression extends LogicalExpression { + override evaluate(args: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND { + const left = this.left.evaluate({ + ...args, + optional: true, + }) + + if (left === CONTEXT_RESOLVE_KEY_NOT_FOUND && + // We're returning key not found here in partial mode + // bacazuse left might become resolvable later, so we should + // only resolve logical or expressions in the last possible moment + args.opts.allowPartial) { + return left + } + + if (left !== CONTEXT_RESOLVE_KEY_NOT_FOUND && isTruthy(left)) { + return left + } + + return this.right.evaluate(args) + } +} + +export class LogicalAndExpression extends LogicalExpression { + override evaluate(args: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND { + const left = this.left.evaluate({ + ...args, + // TODO: Why optional for &&? + optional: true, + }) + + // NOTE(steffen): I find this logic extremely weird. + // + // I would have expected the following: + // "value" && missing => error + // missing && "value" => error + // false && missing => false + // + // and similarly for ||: + // missing || "value" => "value" + // "value" || missing => "value" + // missing || missing => error + // false || missing => error + + if (left === CONTEXT_RESOLVE_KEY_NOT_FOUND && + // We're returning key not found here in partial mode + // bacazuse left might become resolvable later, so we should + // only resolve logical or expressions in the last possible moment + args.opts.allowPartial) { + return left + } + + if (left !== CONTEXT_RESOLVE_KEY_NOT_FOUND && !isTruthy(left)) { + // Javascript would return the value on the left; we return false in case the value is undefined. This is a quirk of Garden's template languate that we want to keep for backwards compatibility. + // TODO: Why? + if (left === undefined) { + return false + } else { + return left + } + } else { + const right = this.right.evaluate({ + ...args, + // TODO: is this right? + optional: true, + }) + if (right === undefined) { + return false + } else { + return right + } + } + } +} + +export abstract class BinaryExpression extends TemplateExpression { + constructor( + loc: Location, + public readonly operator: string, + public readonly left: TemplateExpression, + public readonly right: TemplateExpression + ) { + super(loc) + } + + override evaluate(args: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND { + const left = this.left.evaluate(args) + const right = this.right.evaluate(args) + + if (left === CONTEXT_RESOLVE_KEY_NOT_FOUND || right === CONTEXT_RESOLVE_KEY_NOT_FOUND) { + return CONTEXT_RESOLVE_KEY_NOT_FOUND + } + + return this.transform(left, right, args) + } + + abstract transform( + left: CollectionOrValue, + right: CollectionOrValue, + args: EvaluateArgs + ): CollectionOrValue +} + +export class EqualExpression extends BinaryExpression { + override transform( + left: CollectionOrValue, + right: CollectionOrValue + ): boolean { + return left === right + } +} + +export class NotEqualExpression extends BinaryExpression { + override transform( + left: CollectionOrValue, + right: CollectionOrValue + ): boolean { + return left !== right + } +} + +export class AddExpression extends BinaryExpression { + override transform( + left: CollectionOrValue, + right: CollectionOrValue, + args: EvaluateArgs + ): CollectionOrValue { + if (isNumber(left) && isNumber(right)) { + return left + right + } else if (isString(left) && isString(left)) { + return left + right + } else if (Array.isArray(left) && Array.isArray(right)) { + // In this special case, simply return the concatenated arrays. + // Input tracking has been taken care of already in this case, as leaf objects are preserved. + return left.concat(right) + } else { + throw new TemplateStringError({ + message: `Both terms need to be either arrays or strings or numbers for + operator (got ${typeof left} and ${typeof right}).`, + rawTemplateString: args.rawTemplateString, + loc: this.loc, + }) + } + } +} + +export class ContainsExpression extends BinaryExpression { + override transform( + collection: CollectionOrValue, + element: CollectionOrValue, + args: EvaluateArgs + ): boolean { + if (!isTemplatePrimitive(element)) { + throw new TemplateStringError({ + message: `The right-hand side of a 'contains' operator must be a string, number, boolean or null (got ${typeof element}).`, + rawTemplateString: args.rawTemplateString, + loc: this.loc, + }) + } + + if (typeof collection === "object" && collection !== null) { + if (isArray(collection)) { + return collection.some((v) => element === v) + } + + return collection.hasOwnProperty(String(element)) + } + + if (typeof collection === "string") { + return collection.includes(String(element)) + } + + throw new TemplateStringError({ + message: `The left-hand side of a 'contains' operator must be a string, array or object (got ${collection}).`, + rawTemplateString: args.rawTemplateString, + loc: this.loc, + }) + } +} + +export abstract class BinaryExpressionOnNumbers extends BinaryExpression { + override transform( + left: CollectionOrValue, + right: CollectionOrValue, + args: EvaluateArgs + ): CollectionOrValue { + // All other operators require numbers to make sense (we're not gonna allow random JS weirdness) + if (!isNumber(left) || !isNumber(right)) { + throw new TemplateStringError({ + message: `Both terms need to be numbers for ${ + this.operator + } operator (got ${typeof left} and ${typeof right}).`, + rawTemplateString: args.rawTemplateString, + loc: this.loc, + }) + } + + return this.calculate(left, right) + } + + abstract calculate(left: number, right: number): number | boolean +} + +export class MultiplyExpression extends BinaryExpressionOnNumbers { + override calculate(left: number, right: number): number { + return left * right + } +} + +export class DivideExpression extends BinaryExpressionOnNumbers { + override calculate(left: number, right: number): number { + return left / right + } +} + +export class ModuloExpression extends BinaryExpressionOnNumbers { + override calculate(left: number, right: number): number { + return left % right + } +} + +export class SubtractExpression extends BinaryExpressionOnNumbers { + override calculate(left: number, right: number): number { + return left - right + } +} + +export class LessThanEqualExpression extends BinaryExpressionOnNumbers { + override calculate(left: number, right: number): boolean { + return left <= right + } +} + +export class GreaterThanEqualExpression extends BinaryExpressionOnNumbers { + override calculate(left: number, right: number): boolean { + return left >= right + } +} + +export class LessThanExpression extends BinaryExpressionOnNumbers { + override calculate(left: number, right: number): boolean { + return left < right + } +} + +export class GreaterThanExpression extends BinaryExpressionOnNumbers { + override calculate(left: number, right: number): boolean { + return left > right + } +} + +export class FormatStringExpression extends TemplateExpression { + constructor( + loc: Location, + public readonly innerExpression: TemplateExpression, + public readonly isOptional: boolean + ) { + super(loc) + } + + override evaluate(args: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND { + const optional = args.optional !== undefined ? args.optional : this.isOptional + + return this.innerExpression.evaluate({ + ...args, + optional, + }) + } +} + +export class ElseBlockExpression extends TemplateExpression { + override evaluate(): never { + // See also `buildConditionalTree` in `parser.pegjs` + throw new InternalError({ + message: `{else} block expression should not end up in the final AST`, + }) + } +} + +export class EndIfBlockExpression extends TemplateExpression { + override evaluate(): never { + // See also `buildConditionalTree` in `parser.pegjs` + throw new InternalError({ + message: `{endif} block expression should not end up in the final AST`, + }) + } +} + +export class IfBlockExpression extends TemplateExpression { + constructor( + loc: Location, + public readonly condition: TemplateExpression, + public ifTrue: TemplateExpression | undefined, + public ifFalse: TemplateExpression | undefined + ) { + super(loc) + } + + override evaluate(args: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND { + const condition = this.condition.evaluate(args) + + if (condition === CONTEXT_RESOLVE_KEY_NOT_FOUND) { + return condition + } + + const evaluated = isTruthy(condition) + ? this.ifTrue?.evaluate(args) + : this.ifFalse?.evaluate(args) + + return evaluated + } +} + +export class StringConcatExpression extends TemplateExpression { + public readonly expressions: TemplateExpression[] + constructor(loc: Location, ...expressions: TemplateExpression[]) { + super(loc) + this.expressions = expressions + } + + override evaluate(args: EvaluateArgs): string | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND { + const evaluatedExpressions: TemplatePrimitive[] = [] + + for (const expr of this.expressions) { + const r = expr.evaluate(args) + + if (r === CONTEXT_RESOLVE_KEY_NOT_FOUND) { + return r + } + + if (!isTemplatePrimitive(r)) { + throw new TemplateStringError({ + message: `Cannot concatenate: expected primitive, but expression resolved to ${typeof r}`, + rawTemplateString: args.rawTemplateString, + loc: this.loc, + }) + } + + evaluatedExpressions.push(r) + } + + const result = evaluatedExpressions.reduce((acc, value) => { + return `${acc}${value === undefined ? "" : value}` + }, "") + + return result + } +} + +export class MemberExpression extends TemplateExpression { + constructor( + loc: Location, + public readonly innerExpression: TemplateExpression + ) { + super(loc) + } + + override evaluate(args: EvaluateArgs): string | number | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND { + const inner = this.innerExpression.evaluate(args) + + if (inner === CONTEXT_RESOLVE_KEY_NOT_FOUND) { + return inner + } + + if (typeof inner !== "string" && typeof inner !== "number") { + throw new TemplateStringError({ + message: `Expression in brackets must resolve to a string or number (got ${typeof inner}).`, + rawTemplateString: args.rawTemplateString, + loc: this.loc, + }) + } + + return inner + } +} + +export class ContextLookupExpression extends TemplateExpression { + constructor( + loc: Location, + public readonly keyPath: (IdentifierExpression | MemberExpression)[] + ) { + super(loc) + } + + override evaluate({ context, opts, optional, rawTemplateString }: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND { + const keyPath: (string | number)[] = [] + for (const k of this.keyPath) { + const evaluated = k.evaluate({ context, opts, optional, rawTemplateString }) + if (evaluated === CONTEXT_RESOLVE_KEY_NOT_FOUND) { + return evaluated + } + keyPath.push(evaluated) + } + + const { resolved } = context.resolve({ + key: keyPath, + nodePath: [], + opts, + }) + + // Partial resolution was allowed, so we should not throw here. + if (resolved === CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) { + // we return CONTEXT_RESOLVE_KEY_NOT_FOUND so we don't need to deal with two different symbols everywhere in AST + // TODO: revisit + return CONTEXT_RESOLVE_KEY_NOT_FOUND + } + + if (resolved === CONTEXT_RESOLVE_KEY_NOT_FOUND) { + if (optional) { + return CONTEXT_RESOLVE_KEY_NOT_FOUND + } + + throw new TemplateStringError({ + message: `Could not resolve key ${renderKeyPath(keyPath)}`, + rawTemplateString, + loc: this.loc, + }) + } + + return resolved + } +} + +export class FunctionCallExpression extends TemplateExpression { + constructor( + loc: Location, + public readonly functionName: IdentifierExpression, + public readonly args: TemplateExpression[] + ) { + super(loc) + } + + override evaluate(args: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND { + const functionArgs: CollectionOrValue[] = [] + for (const functionArg of this.args) { + const result = functionArg.evaluate(args) + if (result === CONTEXT_RESOLVE_KEY_NOT_FOUND) { + return result + } + functionArgs.push(result) + } + + const functionName = this.functionName.evaluate() + + const result: CollectionOrValue = this.callHelperFunction({ + functionName, + args: functionArgs, + text: args.rawTemplateString, + context: args.context, + opts: args.opts, + }) + + return result + } + + callHelperFunction({ + functionName, + args, + text, + }: { + functionName: string + args: CollectionOrValue[] + text: string + context: ConfigContext + opts: ContextResolveOpts + }): CollectionOrValue { + const helperFunctions = getHelperFunctions() + const spec = helperFunctions[functionName] + + if (!spec) { + const availableFns = Object.keys(helperFunctions).join(", ") + throw new TemplateStringError({ + message: `Could not find helper function '${functionName}'. Available helper functions: ${availableFns}`, + rawTemplateString: text, + loc: this.loc, + }) + } + + // Validate args + let i = 0 + for (const [argName, schema] of Object.entries(spec.arguments)) { + const value = args[i] + const schemaDescription = spec.argumentDescriptions[argName] + + if (value === undefined && schemaDescription.flags?.presence === "required") { + throw new TemplateStringError({ + message: `Missing argument '${argName}' (at index ${i}) for ${functionName} helper function.`, + rawTemplateString: text, + loc: this.loc, + }) + } + + const loc = this.loc + class FunctionCallValidationError extends TemplateStringError { + constructor({ message }: { message: string }) { + super({ + message: message, + rawTemplateString: text, + loc: loc, + }) + } + } + + args[i] = validateSchema(value, schema, { + context: `argument '${argName}' for ${functionName} helper function`, + ErrorClass: FunctionCallValidationError, + }) + i++ + } + + try { + return spec.fn(...args) + } catch (error) { + throw new TemplateStringError({ + message: `Error from helper function ${functionName}: ${error}`, + rawTemplateString: text, + loc: this.loc, + }) + } + } +} + +export class TernaryExpression extends TemplateExpression { + constructor( + loc: Location, + public readonly condition: TemplateExpression, + public readonly ifTrue: TemplateExpression, + public readonly ifFalse: TemplateExpression + ) { + super(loc) + } + + override evaluate(args: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND { + const conditionResult = this.condition.evaluate({ + ...args, + optional: true, + }) + + if (args.opts.allowPartial && conditionResult === CONTEXT_RESOLVE_KEY_NOT_FOUND) { + // The variable might become resolvable later + return CONTEXT_RESOLVE_KEY_NOT_FOUND + } + + // evaluate ternary expression + const evaluationResult = conditionResult !== CONTEXT_RESOLVE_KEY_NOT_FOUND && isTruthy(conditionResult) + ? this.ifTrue.evaluate(args) + : this.ifFalse.evaluate(args) + + return evaluationResult + } +} diff --git a/core/src/template-string/functions.ts b/core/src/template-string/functions.ts index 645481ae2e..c778588038 100644 --- a/core/src/template-string/functions.ts +++ b/core/src/template-string/functions.ts @@ -8,17 +8,21 @@ import { v4 as uuidv4 } from "uuid" import { createHash } from "node:crypto" -import { TemplateStringError } from "../exceptions.js" +import { GardenError } from "../exceptions.js" import { camelCase, escapeRegExp, isArrayLike, isEmpty, isString, kebabCase, keyBy, mapValues, trim } from "lodash-es" import type { JoiDescription, Primitive } from "../config/common.js" import { joi, joiPrimitive } from "../config/common.js" import type Joi from "@hapi/joi" -import { validateSchema } from "../config/validation.js" import { load, loadAll } from "js-yaml" import { safeDumpYaml } from "../util/serialization.js" import indentString from "indent-string" -import { mayContainTemplateString } from "./template-string.js" import { dateHelperFunctionSpecs } from "./date-functions.js" +import { CollectionOrValue } from "../util/objects.js" +import { TemplatePrimitive } from "./types.js" + +export class TemplateFunctionCallError extends GardenError { + type = "template-function-call" +} interface ExampleArgument { input: unknown[] @@ -32,7 +36,7 @@ export interface TemplateHelperFunction { arguments: { [name: string]: Joi.Schema } outputSchema: Joi.Schema exampleArguments: ExampleArgument[] - fn: (...args: any[]) => unknown + fn: (...args: any[]) => CollectionOrValue } const helperFunctionSpecs: TemplateHelperFunction[] = [ @@ -110,7 +114,7 @@ const helperFunctionSpecs: TemplateHelperFunction[] = [ } else if (Array.isArray(arg1) && Array.isArray(arg2)) { return [...arg1, ...arg2] } else { - throw new TemplateStringError({ + throw new TemplateFunctionCallError({ message: `Both terms need to be either arrays or strings (got ${typeof arg1} and ${typeof arg2}).`, }) } @@ -279,7 +283,7 @@ const helperFunctionSpecs: TemplateHelperFunction[] = [ const result = Number.parseInt(value, 10) if (Number.isNaN(result)) { - throw new TemplateStringError({ + throw new TemplateFunctionCallError({ message: `${name} index must be a number or a numeric string (got "${value}")`, }) } @@ -372,7 +376,7 @@ const helperFunctionSpecs: TemplateHelperFunction[] = [ ], }, ], - fn: (str: string, multi?: boolean) => (multi ? loadAll(str) : load(str)), + fn: (str: string, multi?: boolean) => (multi ? loadAll(str) : load(str)) as CollectionOrValue, }, { name: "yamlEncode", @@ -398,7 +402,7 @@ const helperFunctionSpecs: TemplateHelperFunction[] = [ fn: (value: any, multiDocument?: boolean) => { if (multiDocument) { if (!isArrayLike(value)) { - throw new TemplateStringError({ + throw new TemplateFunctionCallError({ message: `yamlEncode: Set multiDocument=true but value is not an array (got ${typeof value})`, }) } @@ -451,95 +455,3 @@ export function getHelperFunctions(): HelperFunctions { return _helperFunctions } - -export function callHelperFunction({ - functionName, - args, - text, - allowPartial, -}: { - functionName: string - args: any[] - text: string - allowPartial: boolean -}) { - const helperFunctions = getHelperFunctions() - const spec = helperFunctions[functionName] - - if (!spec) { - const availableFns = Object.keys(helperFunctions).join(", ") - const _error = new TemplateStringError({ - message: `Could not find helper function '${functionName}'. Available helper functions: ${availableFns}`, - }) - return { _error } - } - - const resolvedArgs: any[] = [] - - for (const arg of args) { - // arg can be null here because some helpers allow nulls as valid args - if (arg && arg._error) { - return arg - } - - // allow nulls as valid arg values - if (arg && arg.resolved !== undefined) { - resolvedArgs.push(arg.resolved) - } else { - resolvedArgs.push(arg) - } - } - - // Validate args - let i = 0 - - for (const [argName, schema] of Object.entries(spec.arguments)) { - const value = resolvedArgs[i] - const schemaDescription = spec.argumentDescriptions[argName] - - if (value === undefined && schemaDescription.flags?.presence === "required") { - return { - _error: new TemplateStringError({ - message: `Missing argument '${argName}' (at index ${i}) for ${functionName} helper function.`, - }), - } - } - - try { - resolvedArgs[i] = validateSchema(value, schema, { - context: `argument '${argName}' for ${functionName} helper function`, - ErrorClass: TemplateStringError, - }) - - // do not apply helper function for an unresolved template string - if (mayContainTemplateString(value)) { - if (allowPartial) { - return { resolved: "${" + text + "}" } - } else { - const _error = new TemplateStringError({ - message: `Function '${functionName}' cannot be applied on unresolved value`, - }) - return { _error } - } - } - } catch (_error) { - if (allowPartial) { - return { resolved: text } - } else { - return { _error } - } - } - - i++ - } - - try { - const resolved = spec.fn(...resolvedArgs) - return { resolved } - } catch (error) { - const _error = new TemplateStringError({ - message: `Error from helper function ${functionName}: ${error}`, - }) - return { _error } - } -} diff --git a/core/src/template-string/parser.pegjs b/core/src/template-string/parser.pegjs index 4df9f6602c..874fa8581e 100644 --- a/core/src/template-string/parser.pegjs +++ b/core/src/template-string/parser.pegjs @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018-2024 Garden Technologies, Inc. + * Copyright (C) 2018-2023 Garden Technologies, Inc. * * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this @@ -8,22 +8,18 @@ { const { - buildBinaryExpression, - buildLogicalExpression, - callHelperFunction, + ast, escapePrefix, - getKey, - getValue, - isArray, - isPlainObject, - isPrimitive, optionalSuffix, - missingKeyExceptionType, - passthroughExceptionType, - resolveNested, + parseNested, TemplateStringError, + rawTemplateString, } = options + function filledArray(count, value) { + return Array.from({ length: count }, () => value); + } + function extractOptional(optional, index) { return optional ? optional[index] : null; } @@ -36,97 +32,228 @@ return [head].concat(extractList(tail, index)); } - function optionalList(value) { - return value !== null ? value : []; + function buildBinaryExpression(head, tail) { + return tail.reduce(function(result, element) { + const operator = element[1] + const left = result + const right = element[3] + + if (operator === undefined && right === undefined) { + return left + } + + switch (operator) { + case "==": + return new ast.EqualExpression(location(), operator, left, right) + case "!=": + return new ast.NotEqualExpression(location(), operator, left, right) + case "<=": + return new ast.LessThanEqualExpression(location(), operator, left, right) + case ">=": + return new ast.GreaterThanEqualExpression(location(), operator, left, right) + case "<": + return new ast.LessThanExpression(location(), operator, left, right) + case ">": + return new ast.GreaterThanExpression(location(), operator, left, right) + case "+": + return new ast.AddExpression(location(), operator, left, right) + case "-": + return new ast.SubtractExpression(location(), operator, left, right) + case "*": + return new ast.MultiplyExpression(location(), operator, left, right) + case "/": + return new ast.DivideExpression(location(), operator, left, right) + case "%": + return new ast.ModuloExpression(location(), operator, left, right) + default: + throw new TemplateStringError({ message: `Unrecognized logical operator: ${operator}`, rawTemplateString, loc: location() }) + } + }, head); } - function resolveList(items) { - for (const part of items) { - if (part._error) { - return part + function buildLogicalExpression(head, tail) { + return tail.reduce(function(result, element) { + const operator = element[1] + const left = result + const right = element[3] + + if (operator === undefined && right === undefined) { + return left + } + + switch (operator) { + case "&&": + return new ast.LogicalAndExpression(location(), operator, left, right) + case "||": + return new ast.LogicalOrExpression(location(), operator, left, right) + default: + throw new TemplateStringError({ message: `Unrecognized logical operator: ${operator}`, rawTemplateString, loc: location() }) + } + }, head); + } + + /** + * Transforms a flat list of expressions with block operators in between to the proper ast structure, nesting expressions within blocks in the respective conditionals. + * + * @arg {(ast.TemplateExpression | Symbol)[]} elements - List of block operators and ast.TemplateExpression instances + **/ + function buildConditionalTree(...elements) { + // root level expressions + let rootExpressions = [] + + let currentCondition = undefined + let ifTrue = [] + let ifFalse = [] + let nestingLevel = 0 + let encounteredElse = false + const pushElement = (e) => { + if (!currentCondition) { + return rootExpressions.push(e) + } + if (encounteredElse) { + ifFalse.push(e) + } else { + ifTrue.push(e) } } - return items.map((part) => part.resolved || part) + for (const e of elements) { + if (e instanceof ast.IfBlockExpression) { + if (currentCondition) { + pushElement(e) + nestingLevel++ + } else { + currentCondition = e + } + } else if (e instanceof ast.ElseBlockExpression) { + if (currentCondition === undefined) { + throw new TemplateStringError({ message: "Found ${else} block without a preceding ${if...} block.", rawTemplateString, loc: location() }) + } + if (encounteredElse && nestingLevel === 0) { + throw new TemplateStringError({ message: "Encountered multiple ${else} blocks on the same ${if...} block nesting level.", rawTemplateString, loc: location() }) + } + + if (currentCondition && nestingLevel === 0) { + encounteredElse = true + } else { + pushElement(e) + } + } else if (e instanceof ast.EndIfBlockExpression) { + if (currentCondition === undefined) { + throw new TemplateStringError({ message: "Found ${endif} block without a preceding ${if...} block.", rawTemplateString, loc: location() }) + } + if (nestingLevel === 0) { + currentCondition.ifTrue = buildConditionalTree(...ifTrue) + currentCondition.ifFalse = buildConditionalTree(...ifFalse) + currentCondition.loc.end = e.loc.end + rootExpressions.push(currentCondition) + currentCondition = undefined + } else { + nestingLevel-- + pushElement(e) + } + } else { + pushElement(e) + } + } + + if (currentCondition) { + throw new TemplateStringError({ message: "Missing ${endif} after ${if ...} block.", rawTemplateString, loc: location() }) + } + + if (rootExpressions.length === 0) { + return undefined + } + if (rootExpressions.length === 1) { + return rootExpressions[0] + } + + return new ast.StringConcatExpression(location(), ...rootExpressions) + } + + function isBlockExpression(e) { + return e instanceof ast.IfBlockExpression || e instanceof ast.ElseBlockExpression || e instanceof ast.EndIfBlockExpression + } + + function optionalList(value) { + return value !== null ? value : []; } } -TemplateString - = a:(FormatString)+ b:TemplateString? { return [...a, ...(b || [])] } - / a:Prefix b:(FormatString)+ c:TemplateString? { return [a, ...b, ...(c || [])] } - / InvalidFormatString - / $(.*) { return text() === "" ? [] : [{ resolved: text() }] } +Start + = elements:TemplateStrings { + // If there is only one format string, return it's result directly without wrapping in StringConcatExpression + if (elements.length === 1 && !(isBlockExpression(elements[0]))) { + return elements[0] + } + + return buildConditionalTree(...elements) + } + +TemplateStrings + = head:(FormatString)+ tail:TemplateStrings? { + // This means we are concatenating strings, and there may be conditional block statements + return [...head, ...(tail ? tail : [])] + } + / a:Prefix b:(FormatString)+ c:TemplateStrings? { + // This means we are concatenating strings, and there may be conditional block statements + return [a, ...b, ...(c ? c : [])] + } + / UnclosedFormatString + / $(.+) { + return [new ast.LiteralExpression(location(), text())] + } FormatString - = EscapeStart (!FormatEndWithOptional SourceCharacter)* FormatEndWithOptional { + = EscapeStart SourceCharacter* FormatEndWithOptional { if (options.unescape) { - return text().slice(1) + return new ast.LiteralExpression(location(), text().slice(1)) } else { - return text() + return new ast.LiteralExpression(location(), text()) } } / FormatStart op:BlockOperator FormatEnd { - return { block: op } + // These expressions will not show up in the final AST, but will be used to build the conditional tree + // We instantiate expressions here to get the correct locations for constructng good error messages + switch (op) { + case "else": + return new ast.ElseBlockExpression(location()) + case "endif": + return new ast.EndIfBlockExpression(location()) + default: + throw new TemplateStringError({ message: `Unrecognized block operator: ${op}`, rawTemplateString, loc: location() }) + } } / pre:FormatStartWithEscape blockOperator:(ExpressionBlockOperator __)* e:Expression end:FormatEndWithOptional { if (pre[0] === escapePrefix) { if (options.unescape) { return text().slice(1) + return new ast.LiteralExpression(location(), text().slice(1)) } else { return text() + return new ast.LiteralExpression(location(), text()) } } - // Any unexpected error is returned immediately. Certain exceptions have special semantics that are caught below. - if (e && e._error && e._error.type !== missingKeyExceptionType && e._error.type !== passthroughExceptionType) { - return e - } - - // Need to provide the optional suffix as a variable because of a parsing bug in pegjs - const allowUndefined = end[1] === optionalSuffix + const isOptional = end[1] === optionalSuffix + const expression = new ast.FormatStringExpression(location(), e, isOptional) - if (!isPlainObject(e)) { - e = { resolved: e } - } - - if (e && blockOperator[0] && blockOperator[0][0]) { - e.block = blockOperator[0][0] - } + if (blockOperator && blockOperator.length > 0) { + if (isOptional) { + throw new TemplateStringError({ message: "Cannot specify optional suffix in if-block.", rawTemplateString, loc: location() }) + } - if (e && e.block && allowUndefined) { - const _error = new TemplateStringError({ message: "Cannot specify optional suffix in if-block.", detail: { - text: text(), - }}) - return { _error } + // ifTrue and ifFalse will be filled in by `buildConditionalTree` + const ifTrue = undefined + const ifFalse = undefined + return new ast.IfBlockExpression(location(), expression, ifTrue, ifFalse, isOptional) } - if (getValue(e) === undefined) { - if (e && e._error && e._error.type === passthroughExceptionType) { - // We allow certain configuration contexts (e.g. placeholders for runtime.*) to indicate that a template - // string should be returned partially resolved even if allowPartial=false. - return text() - } else if (options.allowPartial) { - return text() - } else if (allowUndefined) { - if (e && e._error) { - return { ...e, _error: undefined } - } else { - return e - } - } else if (e && e._error) { - return e - } else { - const _error = new TemplateStringError({ message: e.message || "Unable to resolve one or more keys.", detail: { - text: text(), - }}) - return { _error } - } - } - return e + return expression } -InvalidFormatString +UnclosedFormatString = Prefix? FormatStart .* { - throw new TemplateStringError({ message: "Unable to parse as valid template string.", detail: {}}) + throw new TemplateStringError({ message: "Unable to parse as valid template string.", rawTemplateString, loc: location() }) } EscapeStart @@ -157,10 +284,10 @@ ExpressionBlockOperator = "if" Prefix - = !FormatStartWithEscape (. ! FormatStartWithEscape)* . { return text() } + = !FormatStartWithEscape (. ! FormatStartWithEscape)* . { return new ast.LiteralExpression(location(), text()) } Suffix - = !FormatEnd (. ! FormatEnd)* . { return text() } + = !FormatEnd (. ! FormatEnd)* . { return new ast.LiteralExpression(location(), text()) } // ---- expressions ----- // Reduced and adapted from: https://github.com/pegjs/pegjs/blob/master/examples/javascript.pegjs @@ -168,14 +295,7 @@ MemberExpression = head:Identifier tail:( "[" __ e:Expression __ "]" { - if (e.resolved && !isPrimitive(e.resolved)) { - const _error = new TemplateStringError( - { message: `Expression in bracket must resolve to a primitive (got ${typeof e}).`, - detail: { text: e.resolved }} - ) - return { _error } - } - return e + return new ast.MemberExpression(location(), e) } / "." e:Identifier { return e @@ -186,33 +306,36 @@ MemberExpression } CallExpression - = callee:Identifier __ args:Arguments { - // Workaround for parser issue (calling text() before referencing other values) - const functionName = callee - const _args = args - - return callHelperFunction({ functionName, args: _args, text: text(), allowPartial: options.allowPartial }) + = functionName:Identifier __ args:Arguments { + return new ast.FunctionCallExpression(location(), functionName, args) } Arguments - = "(" __ args:(ArgumentList __)? ")" { - return optionalList(extractOptional(args, 0)); + = "(" __ args:ArgumentList? __ ")" { + return args || []; } - ArgumentList - = head:Expression tail:(__ "," __ Expression)* { - return buildList(head, tail, 3); + = head:Expression tail:ArgumentListTail* { + return [head, ...(tail || [])]; + } +ArgumentListTail + = tail:(__ "," __ Expression) { + return tail[3] } ArrayLiteral + = v:_ArrayLiteral { + return new ast.ArrayLiteralExpression(location(), v) + } +_ArrayLiteral = "[" __ elision:(Elision __)? "]" { - return resolveList(optionalList(extractOptional(elision, 0))); + return optionalList(extractOptional(elision, 0)); } / "[" __ elements:ElementList __ "]" { - return resolveList(elements); + return elements; } / "[" __ elements:ElementList __ "," __ elision:(Elision __)? "]" { - return resolveList(elements.concat(optionalList(extractOptional(elision, 0)))); + return elements.concat(optionalList(extractOptional(elision, 0))); } ElementList @@ -233,24 +356,26 @@ Elision PrimaryExpression = v:NonStringLiteral { - return v + return new ast.LiteralExpression(location(), v) } / v:StringLiteral { + // Do not parse empty strings. + if (v === "") { + return new ast.LiteralExpression(location(), "") + } // Allow nested template strings in literals - return resolveNested(v) + const parsed = parseNested(v) + if (typeof parsed === "string") { + // The nested string did not contain template expressions, so it is a literal expression + return new ast.LiteralExpression(location(), parsed) + } + // v contained a template expression + return parsed } / ArrayLiteral / CallExpression / key:MemberExpression { - key = resolveList(key) - if (key._error) { - return key - } - try { - return getKey(key, { allowPartial: options.allowPartial }) - } catch (err) { - return { _error: err } - } + return new ast.ContextLookupExpression(location(), key) } / "(" __ e:Expression __ ")" { return e @@ -259,16 +384,13 @@ PrimaryExpression UnaryExpression = PrimaryExpression / operator:UnaryOperator __ argument:UnaryExpression { - const v = argument - - if (v && v._error) { - return v - } - - if (operator === "typeof") { - return typeof getValue(v) - } else if (operator === "!") { - return !getValue(v) + switch (operator) { + case "typeof": + return new ast.TypeofExpression(location(), argument) + case "!": + return new ast.NotExpression(location(), argument) + default: + throw new TemplateStringError({ message: `Unrecognized unary operator: ${operator}`, rawTemplateString, loc: location() }) } } @@ -277,44 +399,8 @@ UnaryOperator / "!" ContainsExpression - = head:UnaryExpression __ ContainsOperator __ tail:UnaryExpression { - if (head && head._error) { - return head - } - if (tail && tail._error) { - return tail - } - - head = getValue(head) - tail = getValue(tail) - - if (!isPrimitive(tail)) { - return { - _error: new TemplateStringError({ - message:`The right-hand side of a 'contains' operator must be a string, number, boolean or null (got ${typeof tail}).`, - detail: {} - }) - } - } - - const headType = head === null ? "null" : typeof head - - if (headType === "object") { - if (isArray(head)) { - return head.includes(tail) - } else { - return head.hasOwnProperty(tail) - } - } else if (headType === "string") { - return head.includes(tail.toString()) - } else { - return { - _error: new TemplateStringError({ - message: `The left-hand side of a 'contains' operator must be a string, array or object (got ${headType}).`, - detail: {} - }) - } - } + = iterable:UnaryExpression __ ContainsOperator __ element:UnaryExpression { + return new ast.ContainsExpression(location(), "contains", iterable, element) } / UnaryExpression @@ -364,7 +450,7 @@ EqualityOperator LogicalANDExpression = head:EqualityExpression tail:(__ LogicalANDOperator __ EqualityExpression)* - { return buildLogicalExpression(head, tail, options); } + { return buildLogicalExpression(head, tail); } LogicalANDOperator = "&&" @@ -372,7 +458,7 @@ LogicalANDOperator LogicalORExpression = head:LogicalANDExpression tail:(__ LogicalOROperator __ LogicalANDExpression)* - { return buildLogicalExpression(head, tail, options); } + { return buildLogicalExpression(head, tail); } LogicalOROperator = "||" @@ -382,10 +468,7 @@ ConditionalExpression "?" __ consequent:Expression __ ":" __ alternate:Expression { - if (test && test._error) { - return test - } - return getValue(test) ? consequent : alternate + return new ast.TernaryExpression(location(), test, consequent, alternate) } / LogicalORExpression @@ -430,13 +513,15 @@ SingleLineComment = "//" (!LineTerminator SourceCharacter)* Identifier - = !ReservedWord name:IdentifierName { return name; } + = !ReservedWord name:IdentifierName { return new ast.IdentifierExpression(location(), name) } IdentifierName "identifier" = head:IdentifierStart tail:IdentifierPart* { return head + tail.join("") } - / Integer + / Integer { + return text(); + } IdentifierStart = UnicodeLetter @@ -678,4 +763,3 @@ __ _ = __ - diff --git a/core/src/template-string/static-analysis.ts b/core/src/template-string/static-analysis.ts new file mode 100644 index 0000000000..6e522c60d7 --- /dev/null +++ b/core/src/template-string/static-analysis.ts @@ -0,0 +1,86 @@ +/* + * Copyright (C) 2018-2023 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { isNumber, isString, startsWith } from "lodash-es" +import { CollectionOrValue, isArray, isPlainObject } from "../util/objects.js" +import { ContextLookupExpression, IdentifierExpression, LiteralExpression, MemberExpression, TemplateExpression } from "./ast.js" +import { TemplatePrimitive } from "./types.js" + +export type TemplateExpressionGenerator = Generator +export function* visitAll(value: CollectionOrValue): TemplateExpressionGenerator { + if (isArray(value)) { + for (const [k, v] of value.entries()) { + yield* visitAll(v) + } + } else if (isPlainObject(value)) { + for (const k of Object.keys(value)) { + yield* visitAll(value[k]) + } + } else { + yield value + + if (value instanceof TemplateExpression) { + yield* value.visitAll() + } + } +} + +export function containsTemplateExpression(value: CollectionOrValue): boolean { + for (const node of visitAll(value)) { + if (node instanceof TemplateExpression) { + return true + } + } + + return false +} + +export function containsContextLookupReferences(value: CollectionOrValue, path: (string | number)[]): boolean { + for (const keyPath of getContextLookupReferences(value)) { + // TODO: What if the key name contains dots? We should compare arrays instead of comparing joined strings. + if (startsWith(`${keyPath.join(".")}.`, `${path.join(".")}.`)) { + return true + } + } + + return false +} + +export function* getContextLookupReferences( + value: CollectionOrValue> +): Generator<(string | number)[], void, undefined> { + for (const expression of visitAll(value)) { + if (expression instanceof ContextLookupExpression) { + const keyPath: (string | number)[] = [] + + for (const v of expression.keyPath.values()) { + if (v instanceof IdentifierExpression) { + keyPath.push(v.name) + } else if (v instanceof MemberExpression) { + if (v.innerExpression instanceof LiteralExpression) { + if (isString(v.innerExpression.literal) || isNumber(v.innerExpression.literal)) { + keyPath.push(v.innerExpression.literal) + } else { + // only strings and numbers are valid here + break + } + } else { + // it's a dynamic key, so we can't statically analyse the value + break + } + } else { + v satisfies never + } + } + + if (keyPath.length > 0) { + yield keyPath + } + } + } +} diff --git a/core/src/template-string/template-string.ts b/core/src/template-string/template-string.ts index c23053cba1..df04beeacd 100644 --- a/core/src/template-string/template-string.ts +++ b/core/src/template-string/template-string.ts @@ -7,16 +7,11 @@ */ import type { GardenErrorParams } from "../exceptions.js" -import { ConfigurationError, GardenError, TemplateStringError } from "../exceptions.js" -import type { - ConfigContext, - ContextKeySegment, - ContextResolveOpts, - ContextResolveOutput, -} from "../config/template-contexts/base.js" -import { GenericContext, ScanContext } from "../config/template-contexts/base.js" +import { ConfigurationError, GardenError, InternalError, TemplateStringError } from "../exceptions.js" +import type { ConfigContext, ContextKeySegment, ContextResolveOpts } from "../config/template-contexts/base.js" +import { CONTEXT_RESOLVE_KEY_NOT_FOUND, CONTEXT_RESOLVE_KEY_AVAILABLE_LATER, GenericContext, ScanContext } from "../config/template-contexts/base.js" import cloneDeep from "fast-copy" -import { difference, isNumber, isPlainObject, isString, uniq } from "lodash-es" +import { difference, isPlainObject, isString, uniq } from "lodash-es" import type { ActionReference, Primitive, StringMap } from "../config/common.js" import { arrayConcatKey, @@ -34,53 +29,18 @@ import { dedent, deline, naturalList, titleize, truncate } from "../util/string. import type { ObjectWithName } from "../util/util.js" import type { Log } from "../logger/log-entry.js" import type { ModuleConfigContext } from "../config/template-contexts/module.js" -import { callHelperFunction } from "./functions.js" import type { ActionKind } from "../actions/types.js" import { actionKindsLower } from "../actions/types.js" -import { deepMap } from "../util/objects.js" +import { CollectionOrValue, deepMap } from "../util/objects.js" import type { ConfigSource } from "../config/validation.js" import * as parser from "./parser.js" -import { styles } from "../logger/styles.js" import type { ObjectPath } from "../config/base.js" -import { profile } from "../util/profiling.js" +import { TemplatePrimitive } from "./types.js" +import * as ast from "./ast.js" +import { LRUCache } from "lru-cache" -const missingKeyExceptionType = "template-string-missing-key" -const passthroughExceptionType = "template-string-passthrough" const escapePrefix = "$${" -export class TemplateStringMissingKeyException extends GardenError { - type = missingKeyExceptionType -} - -export class TemplateStringPassthroughException extends GardenError { - type = passthroughExceptionType -} - -interface ResolvedClause extends ContextResolveOutput { - block?: "if" | "else" | "else if" | "endif" - _error?: Error -} - -interface ConditionalTree { - type: "root" | "if" | "else" | "value" - value?: any - children: ConditionalTree[] - parent?: ConditionalTree -} - -function getValue(v: Primitive | undefined | ResolvedClause) { - return isPlainObject(v) ? (v as ResolvedClause).resolved : v -} - -function isPartiallyResolved(v: Primitive | undefined | ResolvedClause): boolean { - if (!isPlainObject(v)) { - return false - } - - const clause = v as ResolvedClause - return !!clause.partial -} - export class TemplateError extends GardenError { type = "template" @@ -96,6 +56,11 @@ export class TemplateError extends GardenError { } } +type ParseParams = Parameters +function parseWithPegJs(params: ParseParams) { + return parser.parse(...params) +} + const shouldUnescape = (ctxOpts: ContextResolveOpts) => { // Explicit non-escaping takes the highest priority. if (ctxOpts.unescape === false) { @@ -105,11 +70,61 @@ const shouldUnescape = (ctxOpts: ContextResolveOpts) => { return !!ctxOpts.unescape || !ctxOpts.allowPartial } -type ParseParams = Parameters -const parseWithPegJs = profile(function parseWithPegJs(params: ParseParams) { - return parser.parse(...params) +const parseTemplateStringCache = new LRUCache({ + max: 100000, }) +export function parseTemplateString({ + rawTemplateString, + source, + unescape, +}: { + rawTemplateString: string + source?: ConfigSource + unescape: boolean +}): string | ast.TemplateExpression { + if (!unescape) { + const cached = parseTemplateStringCache.get(rawTemplateString) + + if (cached) { + return cached + } + } + + // Just return immediately if this is definitely not a template string + if (!maybeTemplateString(rawTemplateString)) { + if (!unescape) { + parseTemplateStringCache.set(rawTemplateString, rawTemplateString) + } + return rawTemplateString + } + + if (source === undefined) { + source = { + basePath: [], + yamlDoc: undefined, + } + } + + const parsed = parseWithPegJs([ + rawTemplateString, + { + ast, + escapePrefix, + optionalSuffix: "}?", + parseNested: (nested: string) => parseTemplateString({ rawTemplateString: nested, source, unescape }), + TemplateStringError, + unescape, + }, + ]) + + if (!unescape) { + parseTemplateStringCache.set(rawTemplateString, parsed) + } + + return parsed +} + /** * Parse and resolve a templated string, with the given context. The template format is similar to native JS templated * strings but only supports simple lookups from the given context, e.g. "prefix-${nested.key}-suffix", and not @@ -118,155 +133,59 @@ const parseWithPegJs = profile(function parseWithPegJs(params: ParseParams) { * The context should be a ConfigContext instance. The optional `stack` parameter is used to detect circular * dependencies when resolving context variables. */ -export const resolveTemplateString = profile(function resolveTemplateString({ +export function resolveTemplateString({ string, context, contextOpts = {}, + // TODO: Path and source? what to do with path here? path, + source, }: { string: string context: ConfigContext contextOpts?: ContextResolveOpts + source?: ConfigSource path?: ObjectPath -}): any { - // Just return immediately if this is definitely not a template string - if (!maybeTemplateString(string)) { - return string - } +}): CollectionOrValue { + const parsed = parseTemplateString({ + rawTemplateString: string, + source, + // TODO: remove unescape hacks. + unescape: shouldUnescape(contextOpts), + }) - try { - const parsed = parseWithPegJs([ - string, - { - getKey: (key: string[], resolveOpts?: ContextResolveOpts) => { - return context.resolve({ key, nodePath: [], opts: { ...contextOpts, ...(resolveOpts || {}) } }) - }, - getValue, - resolveNested: (nested: string) => resolveTemplateString({ string: nested, context, contextOpts }), - buildBinaryExpression, - buildLogicalExpression, - isArray: Array.isArray, - ConfigurationError, - TemplateStringError, - missingKeyExceptionType, - passthroughExceptionType, - allowPartial: !!contextOpts.allowPartial, - unescape: shouldUnescape(contextOpts), - escapePrefix, - optionalSuffix: "}?", - isPlainObject, - isPrimitive, - callHelperFunction, - }, - ]) - - const outputs: ResolvedClause[] = parsed.map((p: any) => { - return isPlainObject(p) ? p : { resolved: getValue(p) } + if (parsed instanceof ast.TemplateExpression) { + const result = parsed.evaluate({ + rawTemplateString: string, + context, + opts: contextOpts, }) - // We need to manually propagate errors in the parser, so we catch them here - for (const r of outputs) { - if (r && r["_error"]) { - throw r["_error"] - } - } - - // Use value directly if there is only one (or no) value in the output. - let resolved: any = outputs[0]?.resolved - - if (outputs.length > 1) { - // Assemble the parts into a conditional tree - const tree: ConditionalTree = { - type: "root", - children: [], - } - let currentNode = tree - - for (const part of outputs) { - if (part.block === "if") { - const node: ConditionalTree = { - type: "if", - value: !!part.resolved, - children: [], - parent: currentNode, - } - currentNode.children.push(node) - currentNode = node - } else if (part.block === "else") { - if (currentNode.type !== "if") { - throw new TemplateStringError({ - message: "Found ${else} block without a preceding ${if...} block.", - }) - } - const node: ConditionalTree = { - type: "else", - value: !currentNode.value, - children: [], - parent: currentNode.parent, - } - currentNode.parent!.children.push(node) - currentNode = node - } else if (part.block === "endif") { - if (currentNode.type === "if" || currentNode.type === "else") { - currentNode = currentNode.parent! - } else { - throw new TemplateStringError({ - message: "Found ${endif} block without a preceding ${if...} block.", - }) - } - } else { - const v = getValue(part) - - currentNode.children.push({ - type: "value", - value: v === null ? "null" : v, - children: [], - }) - } - } - - if (currentNode.type === "if" || currentNode.type === "else") { - throw new TemplateStringError({ message: "Missing ${endif} after ${if ...} block." }) - } - - // Walk down tree and resolve the output string - resolved = "" - - function resolveTree(node: ConditionalTree) { - if (node.type === "value" && node.value !== undefined) { - resolved += node.value - } else if (node.type === "root" || ((node.type === "if" || node.type === "else") && !!node.value)) { - for (const child of node.children) { - resolveTree(child) - } - } - } - - resolveTree(tree) - } - - return resolved - } catch (err) { - if (!(err instanceof GardenError)) { - throw err + if (!contextOpts.allowPartial && result === CONTEXT_RESOLVE_KEY_NOT_FOUND) { + throw new InternalError({ + message: "allowPartial is false, but template expression evaluated to symbol.", + }) + } else if (result === CONTEXT_RESOLVE_KEY_NOT_FOUND) { + // The template expression cannot be evaluated yet, we may be able to do it later. + // TODO: return ast.TemplateExpression here, instead of string; Otherwise we'll inevitably have a bug + // where garden will resolve template expressions that might be contained in expression evaluation results + // e.g. if an environment variable contains template string, we don't want to evaluate the template string in there. + // See also https://github.com/garden-io/garden/issues/5825 + return string } - const pathDescription = path ? ` at path ${styles.accent(path.join("."))}` : "" - const prefix = `Invalid template string (${styles.accent( - truncate(string, 200).replace(/\n/g, "\\n") - )})${pathDescription}: ` - const message = err.message.startsWith(prefix) ? err.message : prefix + err.message - throw new TemplateStringError({ message, path }) + return result } -}) + + // string does not contain a template expression + return parsed +} /** * Recursively parses and resolves all templated strings in the given object. */ - -// `extends any` here isn't pretty but this function is hard to type correctly // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-constraint -export const resolveTemplateStrings = profile(function resolveTemplateStrings({ +export function resolveTemplateStrings({ value, context, contextOpts = {}, @@ -291,7 +210,7 @@ export const resolveTemplateStrings = profile(function resolveTemplateStringsresolveTemplateString({ string: value, context, path, contextOpts }) + return resolveTemplateString({ string: value, context, path, source, contextOpts }) } else if (Array.isArray(value)) { const output: unknown[] = [] @@ -379,7 +298,7 @@ export const resolveTemplateStrings = profile(function resolveTemplateStringsvalue } -}) +} const expectedForEachKeys = [arrayForEachKey, arrayForEachReturnKey, arrayForEachFilterKey] @@ -645,13 +564,16 @@ export function mayContainTemplateString(obj: any): boolean { /** * Scans for all template strings in the given object and lists the referenced keys. */ -export function collectTemplateReferences(obj: T): ContextKeySegment[][] { +export function collectTemplateReferences( + obj: object +): ContextKeySegment[][] { + // TODO: Statically analyse AST instead of using ScanContext const context = new ScanContext() resolveTemplateStrings({ value: obj, context, contextOpts: { allowPartial: true }, source: undefined }) return uniq(context.foundKeys.entries()).sort() } -export function getRuntimeTemplateReferences(obj: T) { +export function getRuntimeTemplateReferences(obj: object) { const refs = collectTemplateReferences(obj) return refs.filter((ref) => ref[0] === "runtime") } @@ -666,7 +588,9 @@ interface ActionTemplateReference extends ActionReference { * * TODO-0.13.1: Allow such nested references in certain cases, e.g. if resolvable with a ProjectConfigContext. */ -export function getActionTemplateReferences(config: T): ActionTemplateReference[] { +export function getActionTemplateReferences( + config: object +): ActionTemplateReference[] { const rawRefs = collectTemplateReferences(config) // ${action.*} @@ -757,7 +681,7 @@ export function getActionTemplateReferences(config: T): Action return refs } -export function getModuleTemplateReferences(obj: T, context: ModuleConfigContext) { +export function getModuleTemplateReferences(obj: object, context: ModuleConfigContext) { const refs = collectTemplateReferences(obj) const moduleNames = refs.filter((ref) => ref[0] === "modules" && ref.length > 1) // Resolve template strings in name refs. This would ideally be done ahead of this function, but is currently @@ -823,7 +747,10 @@ export function throwOnMissingSecretKeys(configs: ObjectWithName[], secrets: Str * Collects template references to secrets in obj, and returns an array of any secret keys referenced in it that * aren't present (or have blank values) in the provided secrets map. */ -export function detectMissingSecretKeys(obj: T, secrets: StringMap): ContextKeySegment[] { +export function detectMissingSecretKeys( + obj: object, + secrets: StringMap +): ContextKeySegment[] { const referencedKeys = collectTemplateReferences(obj) .filter((ref) => ref[0] === "secrets") .map((ref) => ref[1]) @@ -837,141 +764,3 @@ export function detectMissingSecretKeys(obj: T, secrets: Strin const missingKeys = difference(referencedKeys, keysWithValues) return missingKeys.sort() } - -function buildBinaryExpression(head: any, tail: any) { - return tail.reduce((result: any, element: any) => { - const operator = element[1] - const leftRes = result - const rightRes = element[3] - - // We need to manually handle and propagate errors because the parser doesn't support promises - if (leftRes && leftRes._error) { - return leftRes - } - if (rightRes && rightRes._error) { - return rightRes - } - const left = getValue(leftRes) - const right = getValue(rightRes) - - // if any operand is partially resolved, preserve the original expression - const leftResPartial = isPartiallyResolved(leftRes) - const rightResPartial = isPartiallyResolved(rightRes) - if (leftResPartial || rightResPartial) { - return `${left} ${operator} ${right}` - } - - // Disallow undefined values for comparisons - if (left === undefined || right === undefined) { - const message = [leftRes, rightRes] - .map((res) => res?.message) - .filter(Boolean) - .join(" ") - const err = new TemplateStringError({ - message: message || "Could not resolve one or more keys.", - }) - return { _error: err } - } - - if (operator === "==") { - return left === right - } - if (operator === "!=") { - return left !== right - } - - if (operator === "+") { - if (isNumber(left) && isNumber(right)) { - return left + right - } else if (isString(left) && isString(right)) { - return left + right - } else if (Array.isArray(left) && Array.isArray(right)) { - return left.concat(right) - } else { - const err = new TemplateStringError({ - message: `Both terms need to be either arrays or strings or numbers for + operator (got ${typeof left} and ${typeof right}).`, - }) - return { _error: err } - } - } - - // All other operators require numbers to make sense (we're not gonna allow random JS weirdness) - if (!isNumber(left) || !isNumber(right)) { - const err = new TemplateStringError({ - message: `Both terms need to be numbers for ${operator} operator (got ${typeof left} and ${typeof right}).`, - }) - return { _error: err } - } - - switch (operator) { - case "*": - return left * right - case "/": - return left / right - case "%": - return left % right - case "-": - return left - right - case "<=": - return left <= right - case ">=": - return left >= right - case "<": - return left < right - case ">": - return left > right - default: - const err = new TemplateStringError({ message: "Unrecognized operator: " + operator }) - return { _error: err } - } - }, head) -} - -function buildLogicalExpression(head: any, tail: any, opts: ContextResolveOpts) { - return tail.reduce((result: any, element: any) => { - const operator = element[1] - const leftRes = result - const rightRes = element[3] - - switch (operator) { - case "&&": - if (leftRes && leftRes._error) { - if (!opts.allowPartial && leftRes._error.type === missingKeyExceptionType) { - return false - } - return leftRes - } - - const leftValue = getValue(leftRes) - - if (leftValue === undefined) { - return { resolved: false } - } else if (!leftValue) { - return { resolved: leftValue } - } else { - if (rightRes && rightRes._error) { - if (!opts.allowPartial && rightRes._error.type === missingKeyExceptionType) { - return false - } - return rightRes - } - - const rightValue = getValue(rightRes) - - if (rightValue === undefined) { - return { resolved: false } - } else { - return rightRes - } - } - case "||": - if (leftRes && leftRes._error) { - return leftRes - } - return getValue(leftRes) ? leftRes : rightRes - default: - const err = new TemplateStringError({ message: "Unrecognized operator: " + operator }) - return { _error: err } - } - }, head) -} diff --git a/core/src/template-string/types.ts b/core/src/template-string/types.ts new file mode 100644 index 0000000000..4e5b8cb8c3 --- /dev/null +++ b/core/src/template-string/types.ts @@ -0,0 +1,18 @@ +/* + * Copyright (C) 2018-2023 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { isPrimitive, Primitive } from "utility-types" + +export function isTemplatePrimitive(value: unknown): value is TemplatePrimitive { + return isPrimitive(value) && typeof value !== "symbol" +} + +export type EmptyArray = never[] +export type EmptyObject = { [key: string]: never } + +export type TemplatePrimitive = Exclude diff --git a/core/src/util/objects.ts b/core/src/util/objects.ts index 3b0c73cc93..379a4ed072 100644 --- a/core/src/util/objects.ts +++ b/core/src/util/objects.ts @@ -6,23 +6,43 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import { isArray, isPlainObject, mapValues, pickBy } from "lodash-es" +import { isPlainObject as lodashIsPlainObject, mapValues, pickBy } from "lodash-es" + +export type Collection

= + | CollectionOrValue

[] + | { [key: string]: CollectionOrValue

} + +export type CollectionOrValue

= P | Collection

+ +// adds appropriate type guard to Array.isArray +export function isArray

( + value: CollectionOrValue

+): value is CollectionOrValue

[] { + return Array.isArray(value) +} + +// adds appropriate type guard to lodash isPlainObject +export function isPlainObject

( + value: CollectionOrValue

+): value is { [key: string]: CollectionOrValue

} { + return lodashIsPlainObject(value) +} /** * Recursively process all values in the given input, * walking through all object keys _and array items_. */ -export function deepMap( - value: T | Iterable, - fn: (value: any, key: string | number) => any, - key?: number | string -): U | Iterable { +export function deepMap( + value: CollectionOrValue, + fn: (value: Exclude>, key: string | number, keyPath: (number | string)[]) => R, + keyPath: (number | string)[] = [] +): CollectionOrValue { if (isArray(value)) { - return value.map((v, k) => deepMap(v, fn, k)) + return value.map((v, k) => deepMap(v, fn, [...keyPath, k])) } else if (isPlainObject(value)) { - return mapValues(value, (v, k) => deepMap((v), fn, k)) + return mapValues(value, (v, k) => deepMap(v, fn, [...keyPath, k])) } else { - return fn(value, key || 0) + return fn(value as Exclude>, keyPath[keyPath.length-1] || 0, keyPath) } } @@ -30,19 +50,20 @@ export function deepMap( * Recursively filter all keys and values in the given input, * walking through all object keys _and array items_. */ -export function deepFilter( - value: T | Iterable, +export function deepFilter( + value: CollectionOrValue, fn: (value: any, key: string | number) => boolean -): U | Iterable { +): CollectionOrValue { if (isArray(value)) { - return >value.filter(fn).map((v) => deepFilter(v, fn)) + return value.filter(fn).map((v) => deepFilter(v, fn)) } else if (isPlainObject(value)) { - return mapValues(pickBy(value, fn), (v) => deepFilter(v, fn)) + return mapValues(pickBy(value, fn), (v) => deepFilter(v, fn)) } else { - return value + return value } } + export function omitUndefined(o: object) { return pickBy(o, (v: any) => v !== undefined) } From 82cc0dd4c084c4d6233d7d28530192954b706d6c Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Tue, 3 Dec 2024 10:49:17 +0100 Subject: [PATCH 02/43] chore: fix lint issues --- core/src/config/base.ts | 2 +- core/src/config/template-contexts/base.ts | 4 +- core/src/exceptions.ts | 5 +- core/src/graph/actions.ts | 2 +- core/src/resolve-module.ts | 6 +- core/src/template-string/ast.ts | 63 ++++++++++++--------- core/src/template-string/functions.ts | 4 +- core/src/template-string/static-analysis.ts | 26 ++++++--- core/src/template-string/template-string.ts | 22 +++---- core/src/template-string/types.ts | 5 +- core/src/util/objects.ts | 15 ++--- 11 files changed, 78 insertions(+), 76 deletions(-) diff --git a/core/src/config/base.ts b/core/src/config/base.ts index 91724e6028..d539931ab4 100644 --- a/core/src/config/base.ts +++ b/core/src/config/base.ts @@ -32,7 +32,7 @@ import type { Document, DocumentOptions } from "yaml" import { parseAllDocuments } from "yaml" import { dedent, deline } from "../util/string.js" import { makeDocsLinkStyled } from "../docs/common.js" -import { profile, profileAsync } from "../util/profiling.js" +import { profileAsync } from "../util/profiling.js" import { readFile } from "fs/promises" import { LRUCache } from "lru-cache" diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index c1a0e8f642..83a7966545 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -9,9 +9,7 @@ import type Joi from "@hapi/joi" import { isString } from "lodash-es" import { ConfigurationError } from "../../exceptions.js" -import { - resolveTemplateString, -} from "../../template-string/template-string.js" +import { resolveTemplateString } from "../../template-string/template-string.js" import type { CustomObjectSchema } from "../common.js" import { isPrimitive, joi, joiIdentifier } from "../common.js" import { KeyedSet } from "../../util/keyed-set.js" diff --git a/core/src/exceptions.ts b/core/src/exceptions.ts index fc5561ff1c..7295c95ff9 100644 --- a/core/src/exceptions.ts +++ b/core/src/exceptions.ts @@ -16,9 +16,8 @@ import indentString from "indent-string" import { constants } from "os" import dns from "node:dns" import { styles } from "./logger/styles.js" -import type { ObjectPath } from "./config/base.js" import type { ExecaError } from "execa" -import { Location } from "./template-string/ast.js" +import type { Location } from "./template-string/ast.js" // Unfortunately, NodeJS does not provide a list of all error codes, so we have to maintain this list manually. // See https://nodejs.org/docs/latest-v18.x/api/dns.html#error-codes @@ -313,7 +312,7 @@ export class TemplateStringError extends GardenError { loc?: Location rawTemplateString: string - constructor(params: GardenErrorParams & { rawTemplateString: string, loc?: Location }) { + constructor(params: GardenErrorParams & { rawTemplateString: string; loc?: Location }) { super(params) this.loc = params.loc this.rawTemplateString = params.rawTemplateString diff --git a/core/src/graph/actions.ts b/core/src/graph/actions.ts index 8f23aa9cf2..0d4a7a9ba3 100644 --- a/core/src/graph/actions.ts +++ b/core/src/graph/actions.ts @@ -65,7 +65,7 @@ import { minimatch } from "minimatch" import type { ConfigContext } from "../config/template-contexts/base.js" import type { LinkedSource, LinkedSourceMap } from "../config-store/local.js" import { relative } from "path" -import { profile, profileAsync } from "../util/profiling.js" +import { profileAsync } from "../util/profiling.js" import { uuidv4 } from "../util/random.js" import { getSourcePath } from "../vcs/vcs.js" import { styles } from "../logger/styles.js" diff --git a/core/src/resolve-module.ts b/core/src/resolve-module.ts index 5733776b74..d322cf60b6 100644 --- a/core/src/resolve-module.ts +++ b/core/src/resolve-module.ts @@ -63,8 +63,6 @@ import { styles } from "./logger/styles.js" import { actionReferenceToString } from "./actions/base.js" import type { DepGraph } from "dependency-graph" import { minimatch } from "minimatch" -import { CollectionOrValue } from "./util/objects.js" -import { TemplatePrimitive } from "./template-string/types.js" // This limit is fairly arbitrary, but we need to have some cap on concurrent processing. export const moduleResolutionConcurrencyLimit = 50 @@ -531,9 +529,7 @@ export class ModuleResolver { const configContext = new ModuleConfigContext(contextParams) - const templateRefs = getModuleTemplateReferences(rawConfig, - configContext - ) + const templateRefs = getModuleTemplateReferences(rawConfig, configContext) const templateDeps = templateRefs.filter((d) => d[1] !== rawConfig.name).map((d) => d[1]) // This is a bit of a hack, but we need to store the template dependencies on the raw config so we can check diff --git a/core/src/template-string/ast.ts b/core/src/template-string/ast.ts index 4b220df78e..d8c2d3bee3 100644 --- a/core/src/template-string/ast.ts +++ b/core/src/template-string/ast.ts @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018-2023 Garden Technologies, Inc. + * Copyright (C) 2018-2024 Garden Technologies, Inc. * * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this @@ -7,13 +7,20 @@ */ import { isArray, isNumber, isString } from "lodash-es" -import { CONTEXT_RESOLVE_KEY_AVAILABLE_LATER, CONTEXT_RESOLVE_KEY_NOT_FOUND, renderKeyPath, type ConfigContext, type ContextResolveOpts } from "../config/template-contexts/base.js" +import { + CONTEXT_RESOLVE_KEY_AVAILABLE_LATER, + CONTEXT_RESOLVE_KEY_NOT_FOUND, + renderKeyPath, + type ConfigContext, + type ContextResolveOpts, +} from "../config/template-contexts/base.js" import { InternalError, TemplateStringError } from "../exceptions.js" import { getHelperFunctions } from "./functions.js" import { isTemplatePrimitive, type TemplatePrimitive } from "./types.js" -import { Collection, CollectionOrValue } from "../util/objects.js" -import { ConfigSource, validateSchema } from "../config/validation.js" -import { TemplateExpressionGenerator } from "./static-analysis.js" +import type { Collection, CollectionOrValue } from "../util/objects.js" +import type { ConfigSource } from "../config/validation.js" +import { validateSchema } from "../config/validation.js" +import type { TemplateExpressionGenerator } from "./static-analysis.js" type EvaluateArgs = { context: ConfigContext @@ -102,7 +109,7 @@ export class LiteralExpression extends TemplateExpression { super(loc) } - override evaluate(args: EvaluateArgs): TemplatePrimitive { + override evaluate(_args: EvaluateArgs): TemplatePrimitive { return this.literal } } @@ -193,11 +200,13 @@ export class LogicalOrExpression extends LogicalExpression { optional: true, }) - if (left === CONTEXT_RESOLVE_KEY_NOT_FOUND && + if ( + left === CONTEXT_RESOLVE_KEY_NOT_FOUND && // We're returning key not found here in partial mode // bacazuse left might become resolvable later, so we should // only resolve logical or expressions in the last possible moment - args.opts.allowPartial) { + args.opts.allowPartial + ) { return left } @@ -230,11 +239,13 @@ export class LogicalAndExpression extends LogicalExpression { // missing || missing => error // false || missing => error - if (left === CONTEXT_RESOLVE_KEY_NOT_FOUND && + if ( + left === CONTEXT_RESOLVE_KEY_NOT_FOUND && // We're returning key not found here in partial mode // bacazuse left might become resolvable later, so we should // only resolve logical or expressions in the last possible moment - args.opts.allowPartial) { + args.opts.allowPartial + ) { return left } @@ -290,19 +301,13 @@ export abstract class BinaryExpression extends TemplateExpression { } export class EqualExpression extends BinaryExpression { - override transform( - left: CollectionOrValue, - right: CollectionOrValue - ): boolean { + override transform(left: CollectionOrValue, right: CollectionOrValue): boolean { return left === right } } export class NotEqualExpression extends BinaryExpression { - override transform( - left: CollectionOrValue, - right: CollectionOrValue - ): boolean { + override transform(left: CollectionOrValue, right: CollectionOrValue): boolean { return left !== right } } @@ -490,9 +495,7 @@ export class IfBlockExpression extends TemplateExpression { return condition } - const evaluated = isTruthy(condition) - ? this.ifTrue?.evaluate(args) - : this.ifFalse?.evaluate(args) + const evaluated = isTruthy(condition) ? this.ifTrue?.evaluate(args) : this.ifFalse?.evaluate(args) return evaluated } @@ -569,7 +572,12 @@ export class ContextLookupExpression extends TemplateExpression { super(loc) } - override evaluate({ context, opts, optional, rawTemplateString }: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND { + override evaluate({ + context, + opts, + optional, + rawTemplateString, + }: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND { const keyPath: (string | number)[] = [] for (const k of this.keyPath) { const evaluated = k.evaluate({ context, opts, optional, rawTemplateString }) @@ -681,9 +689,9 @@ export class FunctionCallExpression extends TemplateExpression { class FunctionCallValidationError extends TemplateStringError { constructor({ message }: { message: string }) { super({ - message: message, + message, rawTemplateString: text, - loc: loc, + loc, }) } } @@ -729,9 +737,10 @@ export class TernaryExpression extends TemplateExpression { } // evaluate ternary expression - const evaluationResult = conditionResult !== CONTEXT_RESOLVE_KEY_NOT_FOUND && isTruthy(conditionResult) - ? this.ifTrue.evaluate(args) - : this.ifFalse.evaluate(args) + const evaluationResult = + conditionResult !== CONTEXT_RESOLVE_KEY_NOT_FOUND && isTruthy(conditionResult) + ? this.ifTrue.evaluate(args) + : this.ifFalse.evaluate(args) return evaluationResult } diff --git a/core/src/template-string/functions.ts b/core/src/template-string/functions.ts index c778588038..10253b6229 100644 --- a/core/src/template-string/functions.ts +++ b/core/src/template-string/functions.ts @@ -17,8 +17,8 @@ import { load, loadAll } from "js-yaml" import { safeDumpYaml } from "../util/serialization.js" import indentString from "indent-string" import { dateHelperFunctionSpecs } from "./date-functions.js" -import { CollectionOrValue } from "../util/objects.js" -import { TemplatePrimitive } from "./types.js" +import type { CollectionOrValue } from "../util/objects.js" +import type { TemplatePrimitive } from "./types.js" export class TemplateFunctionCallError extends GardenError { type = "template-function-call" diff --git a/core/src/template-string/static-analysis.ts b/core/src/template-string/static-analysis.ts index 6e522c60d7..5c3fd894d5 100644 --- a/core/src/template-string/static-analysis.ts +++ b/core/src/template-string/static-analysis.ts @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018-2023 Garden Technologies, Inc. + * Copyright (C) 2018-2024 Garden Technologies, Inc. * * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this @@ -7,14 +7,23 @@ */ import { isNumber, isString, startsWith } from "lodash-es" -import { CollectionOrValue, isArray, isPlainObject } from "../util/objects.js" -import { ContextLookupExpression, IdentifierExpression, LiteralExpression, MemberExpression, TemplateExpression } from "./ast.js" -import { TemplatePrimitive } from "./types.js" +import type { CollectionOrValue } from "../util/objects.js" +import { isArray, isPlainObject } from "../util/objects.js" +import { + ContextLookupExpression, + IdentifierExpression, + LiteralExpression, + MemberExpression, + TemplateExpression, +} from "./ast.js" +import type { TemplatePrimitive } from "./types.js" export type TemplateExpressionGenerator = Generator -export function* visitAll(value: CollectionOrValue): TemplateExpressionGenerator { +export function* visitAll( + value: CollectionOrValue +): TemplateExpressionGenerator { if (isArray(value)) { - for (const [k, v] of value.entries()) { + for (const [_k, v] of value.entries()) { yield* visitAll(v) } } else if (isPlainObject(value)) { @@ -40,7 +49,10 @@ export function containsTemplateExpression(value: CollectionOrValue, path: (string | number)[]): boolean { +export function containsContextLookupReferences( + value: CollectionOrValue, + path: (string | number)[] +): boolean { for (const keyPath of getContextLookupReferences(value)) { // TODO: What if the key name contains dots? We should compare arrays instead of comparing joined strings. if (startsWith(`${keyPath.join(".")}.`, `${path.join(".")}.`)) { diff --git a/core/src/template-string/template-string.ts b/core/src/template-string/template-string.ts index df04beeacd..65b576ad68 100644 --- a/core/src/template-string/template-string.ts +++ b/core/src/template-string/template-string.ts @@ -9,7 +9,7 @@ import type { GardenErrorParams } from "../exceptions.js" import { ConfigurationError, GardenError, InternalError, TemplateStringError } from "../exceptions.js" import type { ConfigContext, ContextKeySegment, ContextResolveOpts } from "../config/template-contexts/base.js" -import { CONTEXT_RESOLVE_KEY_NOT_FOUND, CONTEXT_RESOLVE_KEY_AVAILABLE_LATER, GenericContext, ScanContext } from "../config/template-contexts/base.js" +import { CONTEXT_RESOLVE_KEY_NOT_FOUND, GenericContext, ScanContext } from "../config/template-contexts/base.js" import cloneDeep from "fast-copy" import { difference, isPlainObject, isString, uniq } from "lodash-es" import type { ActionReference, Primitive, StringMap } from "../config/common.js" @@ -25,17 +25,18 @@ import { isSpecialKey, objectSpreadKey, } from "../config/common.js" -import { dedent, deline, naturalList, titleize, truncate } from "../util/string.js" +import { dedent, deline, naturalList, titleize } from "../util/string.js" import type { ObjectWithName } from "../util/util.js" import type { Log } from "../logger/log-entry.js" import type { ModuleConfigContext } from "../config/template-contexts/module.js" import type { ActionKind } from "../actions/types.js" import { actionKindsLower } from "../actions/types.js" -import { CollectionOrValue, deepMap } from "../util/objects.js" +import type { CollectionOrValue } from "../util/objects.js" +import { deepMap } from "../util/objects.js" import type { ConfigSource } from "../config/validation.js" import * as parser from "./parser.js" import type { ObjectPath } from "../config/base.js" -import { TemplatePrimitive } from "./types.js" +import type { TemplatePrimitive } from "./types.js" import * as ast from "./ast.js" import { LRUCache } from "lru-cache" @@ -564,9 +565,7 @@ export function mayContainTemplateString(obj: any): boolean { /** * Scans for all template strings in the given object and lists the referenced keys. */ -export function collectTemplateReferences( - obj: object -): ContextKeySegment[][] { +export function collectTemplateReferences(obj: object): ContextKeySegment[][] { // TODO: Statically analyse AST instead of using ScanContext const context = new ScanContext() resolveTemplateStrings({ value: obj, context, contextOpts: { allowPartial: true }, source: undefined }) @@ -588,9 +587,7 @@ interface ActionTemplateReference extends ActionReference { * * TODO-0.13.1: Allow such nested references in certain cases, e.g. if resolvable with a ProjectConfigContext. */ -export function getActionTemplateReferences( - config: object -): ActionTemplateReference[] { +export function getActionTemplateReferences(config: object): ActionTemplateReference[] { const rawRefs = collectTemplateReferences(config) // ${action.*} @@ -747,10 +744,7 @@ export function throwOnMissingSecretKeys(configs: ObjectWithName[], secrets: Str * Collects template references to secrets in obj, and returns an array of any secret keys referenced in it that * aren't present (or have blank values) in the provided secrets map. */ -export function detectMissingSecretKeys( - obj: object, - secrets: StringMap -): ContextKeySegment[] { +export function detectMissingSecretKeys(obj: object, secrets: StringMap): ContextKeySegment[] { const referencedKeys = collectTemplateReferences(obj) .filter((ref) => ref[0] === "secrets") .map((ref) => ref[1]) diff --git a/core/src/template-string/types.ts b/core/src/template-string/types.ts index 4e5b8cb8c3..480fa9af39 100644 --- a/core/src/template-string/types.ts +++ b/core/src/template-string/types.ts @@ -1,12 +1,13 @@ /* - * Copyright (C) 2018-2023 Garden Technologies, Inc. + * Copyright (C) 2018-2024 Garden Technologies, Inc. * * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import { isPrimitive, Primitive } from "utility-types" +import type { Primitive } from "utility-types" +import { isPrimitive } from "utility-types" export function isTemplatePrimitive(value: unknown): value is TemplatePrimitive { return isPrimitive(value) && typeof value !== "symbol" diff --git a/core/src/util/objects.ts b/core/src/util/objects.ts index 379a4ed072..4b8971bdb3 100644 --- a/core/src/util/objects.ts +++ b/core/src/util/objects.ts @@ -8,23 +8,17 @@ import { isPlainObject as lodashIsPlainObject, mapValues, pickBy } from "lodash-es" -export type Collection

= - | CollectionOrValue

[] - | { [key: string]: CollectionOrValue

} +export type Collection

= CollectionOrValue

[] | { [key: string]: CollectionOrValue

} export type CollectionOrValue

= P | Collection

// adds appropriate type guard to Array.isArray -export function isArray

( - value: CollectionOrValue

-): value is CollectionOrValue

[] { +export function isArray

(value: CollectionOrValue

): value is CollectionOrValue

[] { return Array.isArray(value) } // adds appropriate type guard to lodash isPlainObject -export function isPlainObject

( - value: CollectionOrValue

-): value is { [key: string]: CollectionOrValue

} { +export function isPlainObject

(value: CollectionOrValue

): value is { [key: string]: CollectionOrValue

} { return lodashIsPlainObject(value) } @@ -42,7 +36,7 @@ export function deepMap( } else if (isPlainObject(value)) { return mapValues(value, (v, k) => deepMap(v, fn, [...keyPath, k])) } else { - return fn(value as Exclude>, keyPath[keyPath.length-1] || 0, keyPath) + return fn(value as Exclude>, keyPath[keyPath.length - 1] || 0, keyPath) } } @@ -63,7 +57,6 @@ export function deepFilter( } } - export function omitUndefined(o: object) { return pickBy(o, (v: any) => v !== undefined) } From f4a88a908cc2a9f38770919815ee550aac2c00f0 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Tue, 3 Dec 2024 12:35:25 +0100 Subject: [PATCH 03/43] test: fix tests --- core/src/config/template-contexts/base.ts | 6 +- core/src/exceptions.ts | 13 +- core/src/template-string/ast.ts | 149 ++++++++---------- core/src/template-string/parser.pegjs | 4 +- core/src/template-string/template-string.ts | 3 +- .../unit/src/config/template-contexts/base.ts | 4 +- core/test/unit/src/template-string.ts | 8 +- 7 files changed, 91 insertions(+), 96 deletions(-) diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index 83a7966545..08eaab0f5e 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -194,12 +194,10 @@ export abstract class ConfigContext { // is caught in the surrounding template resolution code. if (this._alwaysAllowPartial || opts.allowPartial) { return { - resolved: CONTEXT_RESOLVE_KEY_NOT_FOUND, + resolved: CONTEXT_RESOLVE_KEY_AVAILABLE_LATER, } } else { - // Otherwise we return the undefined value, so that any logical expressions can be evaluated appropriately. - // The template resolver will throw the error later if appropriate. - return { resolved: undefined, message } + return { resolved: CONTEXT_RESOLVE_KEY_NOT_FOUND, message } } } diff --git a/core/src/exceptions.ts b/core/src/exceptions.ts index 7295c95ff9..352ad8a15d 100644 --- a/core/src/exceptions.ts +++ b/core/src/exceptions.ts @@ -312,8 +312,17 @@ export class TemplateStringError extends GardenError { loc?: Location rawTemplateString: string - constructor(params: GardenErrorParams & { rawTemplateString: string; loc?: Location }) { - super(params) + constructor(params: GardenErrorParams & { rawTemplateString: string, loc?: Location }) { + // TODO: figure out how to get correct path + const path = params.loc?.source?.basePath + + const pathDescription = path ? ` at path ${styles.accent(path.join("."))}` : "" + const prefix = `Invalid template string (${styles.accent( + truncate(params.rawTemplateString, { length: 200 }).replace(/\n/g, "\\n") + )})${pathDescription}: ` + const message = params.message.startsWith(prefix) ? params.message : prefix + params.message + + super({ ...params, message }) this.loc = params.loc this.rawTemplateString = params.rawTemplateString } diff --git a/core/src/template-string/ast.ts b/core/src/template-string/ast.ts index d8c2d3bee3..6cf663731c 100644 --- a/core/src/template-string/ast.ts +++ b/core/src/template-string/ast.ts @@ -51,7 +51,7 @@ export type Location = { source?: ConfigSource } -export type TemplateEvaluationResult = TemplatePrimitive | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND +export type TemplateEvaluationResult = TemplatePrimitive | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER function* astVisitAll(e: TemplateExpression): TemplateExpressionGenerator { for (const key in e) { @@ -80,7 +80,7 @@ export abstract class TemplateExpression { yield* astVisitAll(this) } - abstract evaluate(args: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND + abstract evaluate(args: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER } export class IdentifierExpression extends TemplateExpression { @@ -124,11 +124,11 @@ export class ArrayLiteralExpression extends TemplateExpression { super(loc) } - override evaluate(args: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND { + override evaluate(args: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { const result: CollectionOrValue = [] for (const e of this.literal) { const res = e.evaluate(args) - if (res === CONTEXT_RESOLVE_KEY_NOT_FOUND) { + if (typeof res === "symbol") { return res } result.push(res) @@ -146,10 +146,10 @@ export abstract class UnaryExpression extends TemplateExpression { super(loc) } - override evaluate(args: EvaluateArgs): TemplatePrimitive | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND { + override evaluate(args: EvaluateArgs): TemplatePrimitive | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { const inner = this.innerExpression.evaluate(args) - if (inner === CONTEXT_RESOLVE_KEY_NOT_FOUND) { + if (typeof inner === "symbol") { return inner } @@ -194,19 +194,14 @@ export function isTruthy(v: CollectionOrValue): boolean { } export class LogicalOrExpression extends LogicalExpression { - override evaluate(args: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND { + override evaluate(args: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { const left = this.left.evaluate({ ...args, optional: true, }) - if ( - left === CONTEXT_RESOLVE_KEY_NOT_FOUND && - // We're returning key not found here in partial mode - // bacazuse left might become resolvable later, so we should - // only resolve logical or expressions in the last possible moment - args.opts.allowPartial - ) { + if (left === CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) { + // If key might be available later, we can't decide which branch to take in the logical expression yet. return left } @@ -219,56 +214,41 @@ export class LogicalOrExpression extends LogicalExpression { } export class LogicalAndExpression extends LogicalExpression { - override evaluate(args: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND { + override evaluate(args: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { const left = this.left.evaluate({ ...args, - // TODO: Why optional for &&? optional: true, }) - // NOTE(steffen): I find this logic extremely weird. - // - // I would have expected the following: - // "value" && missing => error - // missing && "value" => error - // false && missing => false - // - // and similarly for ||: - // missing || "value" => "value" - // "value" || missing => "value" - // missing || missing => error - // false || missing => error - - if ( - left === CONTEXT_RESOLVE_KEY_NOT_FOUND && - // We're returning key not found here in partial mode - // bacazuse left might become resolvable later, so we should - // only resolve logical or expressions in the last possible moment - args.opts.allowPartial - ) { + if (left === CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) { + // If key might be available later, we can't decide which branch to take in the logical expression yet. return left } - if (left !== CONTEXT_RESOLVE_KEY_NOT_FOUND && !isTruthy(left)) { - // Javascript would return the value on the left; we return false in case the value is undefined. This is a quirk of Garden's template languate that we want to keep for backwards compatibility. - // TODO: Why? - if (left === undefined) { - return false - } else { - return left - } - } else { - const right = this.right.evaluate({ - ...args, - // TODO: is this right? - optional: true, - }) - if (right === undefined) { - return false - } else { - return right - } + // We return false in case the variable could not be resolved. This is a quirk of Garden's template language that we want to keep for backwards compatibility. + if (left === CONTEXT_RESOLVE_KEY_NOT_FOUND) { + return false + } + + if (!isTruthy(left)) { + return left } + + const right = this.right.evaluate({ + ...args, + optional: true, + }) + + if (right === CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) { + // If key might be available later, we can't decide on a final value yet and the logical expression needs to be reevaluated later. + return right + } + + if (right === CONTEXT_RESOLVE_KEY_NOT_FOUND) { + return false + } + + return right } } @@ -282,10 +262,19 @@ export abstract class BinaryExpression extends TemplateExpression { super(loc) } - override evaluate(args: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND { + override evaluate(args: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { const left = this.left.evaluate(args) + + if (left === CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) { + return left + } + const right = this.right.evaluate(args) + if (right === CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) { + return right + } + if (left === CONTEXT_RESOLVE_KEY_NOT_FOUND || right === CONTEXT_RESOLVE_KEY_NOT_FOUND) { return CONTEXT_RESOLVE_KEY_NOT_FOUND } @@ -450,13 +439,19 @@ export class FormatStringExpression extends TemplateExpression { super(loc) } - override evaluate(args: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND { + override evaluate(args: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { const optional = args.optional !== undefined ? args.optional : this.isOptional - return this.innerExpression.evaluate({ + const result = this.innerExpression.evaluate({ ...args, optional, }) + + if (optional && result === CONTEXT_RESOLVE_KEY_NOT_FOUND) { + return undefined + } + + return result } } @@ -488,10 +483,10 @@ export class IfBlockExpression extends TemplateExpression { super(loc) } - override evaluate(args: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND { + override evaluate(args: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { const condition = this.condition.evaluate(args) - if (condition === CONTEXT_RESOLVE_KEY_NOT_FOUND) { + if (typeof condition === "symbol") { return condition } @@ -508,7 +503,7 @@ export class StringConcatExpression extends TemplateExpression { this.expressions = expressions } - override evaluate(args: EvaluateArgs): string | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND { + override evaluate(args: EvaluateArgs): string | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { const evaluatedExpressions: TemplatePrimitive[] = [] for (const expr of this.expressions) { @@ -545,7 +540,7 @@ export class MemberExpression extends TemplateExpression { super(loc) } - override evaluate(args: EvaluateArgs): string | number | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND { + override evaluate(args: EvaluateArgs): string | number | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { const inner = this.innerExpression.evaluate(args) if (inner === CONTEXT_RESOLVE_KEY_NOT_FOUND) { @@ -572,16 +567,11 @@ export class ContextLookupExpression extends TemplateExpression { super(loc) } - override evaluate({ - context, - opts, - optional, - rawTemplateString, - }: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND { + override evaluate({ context, opts, optional, rawTemplateString }: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { const keyPath: (string | number)[] = [] for (const k of this.keyPath) { const evaluated = k.evaluate({ context, opts, optional, rawTemplateString }) - if (evaluated === CONTEXT_RESOLVE_KEY_NOT_FOUND) { + if (typeof evaluated === "symbol") { return evaluated } keyPath.push(evaluated) @@ -593,20 +583,20 @@ export class ContextLookupExpression extends TemplateExpression { opts, }) - // Partial resolution was allowed, so we should not throw here. + // if context returns key available later, then we do not need to throw, because partial mode is enabled. if (resolved === CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) { - // we return CONTEXT_RESOLVE_KEY_NOT_FOUND so we don't need to deal with two different symbols everywhere in AST - // TODO: revisit - return CONTEXT_RESOLVE_KEY_NOT_FOUND + return resolved } + // if we encounter a key not found symbol, it's an error unless the optional flag is true, which is used by + // logical operators and expressions, as well as the optional suffix in FormatStringExpression. if (resolved === CONTEXT_RESOLVE_KEY_NOT_FOUND) { if (optional) { return CONTEXT_RESOLVE_KEY_NOT_FOUND } throw new TemplateStringError({ - message: `Could not resolve key ${renderKeyPath(keyPath)}`, + message: `Could not find key ${renderKeyPath(keyPath)}`, rawTemplateString, loc: this.loc, }) @@ -625,11 +615,11 @@ export class FunctionCallExpression extends TemplateExpression { super(loc) } - override evaluate(args: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND { + override evaluate(args: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { const functionArgs: CollectionOrValue[] = [] for (const functionArg of this.args) { const result = functionArg.evaluate(args) - if (result === CONTEXT_RESOLVE_KEY_NOT_FOUND) { + if (typeof result === "symbol") { return result } functionArgs.push(result) @@ -725,15 +715,14 @@ export class TernaryExpression extends TemplateExpression { super(loc) } - override evaluate(args: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND { + override evaluate(args: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { const conditionResult = this.condition.evaluate({ ...args, optional: true, }) - if (args.opts.allowPartial && conditionResult === CONTEXT_RESOLVE_KEY_NOT_FOUND) { - // The variable might become resolvable later - return CONTEXT_RESOLVE_KEY_NOT_FOUND + if (conditionResult === CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) { + return conditionResult } // evaluate ternary expression diff --git a/core/src/template-string/parser.pegjs b/core/src/template-string/parser.pegjs index 874fa8581e..4ba216dcc4 100644 --- a/core/src/template-string/parser.pegjs +++ b/core/src/template-string/parser.pegjs @@ -213,7 +213,7 @@ FormatString } / FormatStart op:BlockOperator FormatEnd { // These expressions will not show up in the final AST, but will be used to build the conditional tree - // We instantiate expressions here to get the correct locations for constructng good error messages + // We instantiate expressions here to get the correct locations for constructing good error messages switch (op) { case "else": return new ast.ElseBlockExpression(location()) @@ -226,10 +226,8 @@ FormatString / pre:FormatStartWithEscape blockOperator:(ExpressionBlockOperator __)* e:Expression end:FormatEndWithOptional { if (pre[0] === escapePrefix) { if (options.unescape) { - return text().slice(1) return new ast.LiteralExpression(location(), text().slice(1)) } else { - return text() return new ast.LiteralExpression(location(), text()) } } diff --git a/core/src/template-string/template-string.ts b/core/src/template-string/template-string.ts index 65b576ad68..8859e6ac30 100644 --- a/core/src/template-string/template-string.ts +++ b/core/src/template-string/template-string.ts @@ -166,7 +166,8 @@ export function resolveTemplateString({ throw new InternalError({ message: "allowPartial is false, but template expression evaluated to symbol.", }) - } else if (result === CONTEXT_RESOLVE_KEY_NOT_FOUND) { + // TODO: think about if it's really ok to partially resolve if allowPartial is false. This can happen if a context with _alwaysPartial is used together with allowPartial false. + } else if (result === CONTEXT_RESOLVE_KEY_NOT_FOUND || result === CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) { // The template expression cannot be evaluated yet, we may be able to do it later. // TODO: return ast.TemplateExpression here, instead of string; Otherwise we'll inevitably have a bug // where garden will resolve template expressions that might be contained in expression evaluation results diff --git a/core/test/unit/src/config/template-contexts/base.ts b/core/test/unit/src/config/template-contexts/base.ts index e0421c42c2..36911e0744 100644 --- a/core/test/unit/src/config/template-contexts/base.ts +++ b/core/test/unit/src/config/template-contexts/base.ts @@ -48,14 +48,14 @@ describe("ConfigContext", () => { const c = new TestContext({}) const { resolved, message } = resolveKey(c, ["basic"]) expect(resolved).to.be.undefined - expect(stripAnsi(message!)).to.include("Could not find key basic.") + expect(stripAnsi(message!)).to.include("Could not find key basic") }) context("allowPartial=true", () => { it("should throw on missing key when allowPartial=true", async () => { const c = new TestContext({}) await expectError(() => resolveKey(c, ["basic"], { allowPartial: true }), { - contains: "Could not find key basic.", + contains: "Could not find key basic", }) }) diff --git a/core/test/unit/src/template-string.ts b/core/test/unit/src/template-string.ts index 7ba8b5600c..bd4730593a 100644 --- a/core/test/unit/src/template-string.ts +++ b/core/test/unit/src/template-string.ts @@ -214,7 +214,7 @@ describe("resolveTemplateString", () => { it("should throw when a key is not found", () => { void expectError(() => resolveTemplateString({ string: "${some}", context: new TestContext({}) }), { - contains: "Invalid template string (${some}): Could not find key some.", + contains: "Invalid template string (${some}): Could not find key some", }) }) @@ -234,14 +234,14 @@ describe("resolveTemplateString", () => { void expectError( () => resolveTemplateString({ string: "${some}\nmulti\nline\nstring", context: new TestContext({}) }), { - contains: "Invalid template string (${some}\\nmulti\\nline\\nstring): Could not find key some.", + contains: "Invalid template string (${some}\\nmulti\\nline\\nstring): Could not find key some", } ) }) it("should throw when a nested key is not found", () => { void expectError(() => resolveTemplateString({ string: "${some.other}", context: new TestContext({ some: {} }) }), { - contains: "Invalid template string (${some.other}): Could not find key other under some.", + contains: "Invalid template string (${some.other}): Could not find key other under some", }) }) @@ -386,7 +386,7 @@ describe("resolveTemplateString", () => { it("should throw if neither key in logical OR is valid", () => { void expectError(() => resolveTemplateString({ string: "${a || b}", context: new TestContext({}) }), { - contains: "Invalid template string (${a || b}): Could not find key b.", + contains: "Invalid template string (${a || b}): Could not find key b", }) }) From c7e03447b3a5118ad3078a44621dd5c14989226d Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Tue, 3 Dec 2024 13:05:28 +0100 Subject: [PATCH 04/43] chore: fix compilation and lint errors --- core/src/exceptions.ts | 2 +- core/src/template-string/ast.ts | 90 +++++++++++++++++---- core/src/template-string/template-string.ts | 4 +- 3 files changed, 80 insertions(+), 16 deletions(-) diff --git a/core/src/exceptions.ts b/core/src/exceptions.ts index 352ad8a15d..18ab111c79 100644 --- a/core/src/exceptions.ts +++ b/core/src/exceptions.ts @@ -312,7 +312,7 @@ export class TemplateStringError extends GardenError { loc?: Location rawTemplateString: string - constructor(params: GardenErrorParams & { rawTemplateString: string, loc?: Location }) { + constructor(params: GardenErrorParams & { rawTemplateString: string; loc?: Location }) { // TODO: figure out how to get correct path const path = params.loc?.source?.basePath diff --git a/core/src/template-string/ast.ts b/core/src/template-string/ast.ts index 6cf663731c..fa992a69e1 100644 --- a/core/src/template-string/ast.ts +++ b/core/src/template-string/ast.ts @@ -51,7 +51,10 @@ export type Location = { source?: ConfigSource } -export type TemplateEvaluationResult = TemplatePrimitive | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER +export type TemplateEvaluationResult = + | TemplatePrimitive + | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND + | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER function* astVisitAll(e: TemplateExpression): TemplateExpressionGenerator { for (const key in e) { @@ -80,7 +83,12 @@ export abstract class TemplateExpression { yield* astVisitAll(this) } - abstract evaluate(args: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER + abstract evaluate( + args: EvaluateArgs + ): + | CollectionOrValue + | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND + | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER } export class IdentifierExpression extends TemplateExpression { @@ -124,7 +132,12 @@ export class ArrayLiteralExpression extends TemplateExpression { super(loc) } - override evaluate(args: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { + override evaluate( + args: EvaluateArgs + ): + | CollectionOrValue + | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND + | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { const result: CollectionOrValue = [] for (const e of this.literal) { const res = e.evaluate(args) @@ -146,7 +159,9 @@ export abstract class UnaryExpression extends TemplateExpression { super(loc) } - override evaluate(args: EvaluateArgs): TemplatePrimitive | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { + override evaluate( + args: EvaluateArgs + ): TemplatePrimitive | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { const inner = this.innerExpression.evaluate(args) if (typeof inner === "symbol") { @@ -194,7 +209,12 @@ export function isTruthy(v: CollectionOrValue): boolean { } export class LogicalOrExpression extends LogicalExpression { - override evaluate(args: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { + override evaluate( + args: EvaluateArgs + ): + | CollectionOrValue + | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND + | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { const left = this.left.evaluate({ ...args, optional: true, @@ -214,7 +234,12 @@ export class LogicalOrExpression extends LogicalExpression { } export class LogicalAndExpression extends LogicalExpression { - override evaluate(args: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { + override evaluate( + args: EvaluateArgs + ): + | CollectionOrValue + | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND + | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { const left = this.left.evaluate({ ...args, optional: true, @@ -262,7 +287,12 @@ export abstract class BinaryExpression extends TemplateExpression { super(loc) } - override evaluate(args: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { + override evaluate( + args: EvaluateArgs + ): + | CollectionOrValue + | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND + | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { const left = this.left.evaluate(args) if (left === CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) { @@ -439,7 +469,12 @@ export class FormatStringExpression extends TemplateExpression { super(loc) } - override evaluate(args: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { + override evaluate( + args: EvaluateArgs + ): + | CollectionOrValue + | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND + | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { const optional = args.optional !== undefined ? args.optional : this.isOptional const result = this.innerExpression.evaluate({ @@ -483,7 +518,12 @@ export class IfBlockExpression extends TemplateExpression { super(loc) } - override evaluate(args: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { + override evaluate( + args: EvaluateArgs + ): + | CollectionOrValue + | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND + | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { const condition = this.condition.evaluate(args) if (typeof condition === "symbol") { @@ -503,7 +543,9 @@ export class StringConcatExpression extends TemplateExpression { this.expressions = expressions } - override evaluate(args: EvaluateArgs): string | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { + override evaluate( + args: EvaluateArgs + ): string | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { const evaluatedExpressions: TemplatePrimitive[] = [] for (const expr of this.expressions) { @@ -540,7 +582,9 @@ export class MemberExpression extends TemplateExpression { super(loc) } - override evaluate(args: EvaluateArgs): string | number | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { + override evaluate( + args: EvaluateArgs + ): string | number | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { const inner = this.innerExpression.evaluate(args) if (inner === CONTEXT_RESOLVE_KEY_NOT_FOUND) { @@ -567,7 +611,15 @@ export class ContextLookupExpression extends TemplateExpression { super(loc) } - override evaluate({ context, opts, optional, rawTemplateString }: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { + override evaluate({ + context, + opts, + optional, + rawTemplateString, + }: EvaluateArgs): + | CollectionOrValue + | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND + | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { const keyPath: (string | number)[] = [] for (const k of this.keyPath) { const evaluated = k.evaluate({ context, opts, optional, rawTemplateString }) @@ -615,7 +667,12 @@ export class FunctionCallExpression extends TemplateExpression { super(loc) } - override evaluate(args: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { + override evaluate( + args: EvaluateArgs + ): + | CollectionOrValue + | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND + | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { const functionArgs: CollectionOrValue[] = [] for (const functionArg of this.args) { const result = functionArg.evaluate(args) @@ -715,7 +772,12 @@ export class TernaryExpression extends TemplateExpression { super(loc) } - override evaluate(args: EvaluateArgs): CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { + override evaluate( + args: EvaluateArgs + ): + | CollectionOrValue + | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND + | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { const conditionResult = this.condition.evaluate({ ...args, optional: true, diff --git a/core/src/template-string/template-string.ts b/core/src/template-string/template-string.ts index 8859e6ac30..b5c70ef8f2 100644 --- a/core/src/template-string/template-string.ts +++ b/core/src/template-string/template-string.ts @@ -9,6 +9,7 @@ import type { GardenErrorParams } from "../exceptions.js" import { ConfigurationError, GardenError, InternalError, TemplateStringError } from "../exceptions.js" import type { ConfigContext, ContextKeySegment, ContextResolveOpts } from "../config/template-contexts/base.js" +import { CONTEXT_RESOLVE_KEY_AVAILABLE_LATER } from "../config/template-contexts/base.js" import { CONTEXT_RESOLVE_KEY_NOT_FOUND, GenericContext, ScanContext } from "../config/template-contexts/base.js" import cloneDeep from "fast-copy" import { difference, isPlainObject, isString, uniq } from "lodash-es" @@ -58,6 +59,7 @@ export class TemplateError extends GardenError { } type ParseParams = Parameters + function parseWithPegJs(params: ParseParams) { return parser.parse(...params) } @@ -166,7 +168,7 @@ export function resolveTemplateString({ throw new InternalError({ message: "allowPartial is false, but template expression evaluated to symbol.", }) - // TODO: think about if it's really ok to partially resolve if allowPartial is false. This can happen if a context with _alwaysPartial is used together with allowPartial false. + // TODO: think about if it's really ok to partially resolve if allowPartial is false. This can happen if a context with _alwaysPartial is used together with allowPartial false. } else if (result === CONTEXT_RESOLVE_KEY_NOT_FOUND || result === CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) { // The template expression cannot be evaluated yet, we may be able to do it later. // TODO: return ast.TemplateExpression here, instead of string; Otherwise we'll inevitably have a bug From 4346ef9de21e24cdd2d4a1a25cdac28672e8694c Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Tue, 3 Dec 2024 14:14:27 +0100 Subject: [PATCH 05/43] chore: print details on assertion failure in `expectError` function --- core/.mocharc.yml | 2 +- core/src/util/testing.ts | 24 ++++++++++++++++++------ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/core/.mocharc.yml b/core/.mocharc.yml index ea351a511e..33b552fa72 100644 --- a/core/.mocharc.yml +++ b/core/.mocharc.yml @@ -1,5 +1,5 @@ require: - - build/test/setup.js + - ../../../../build/test/setup.js watch-files: - build/**/* ignore: diff --git a/core/src/util/testing.ts b/core/src/util/testing.ts index c1724339ce..ede7c589bc 100644 --- a/core/src/util/testing.ts +++ b/core/src/util/testing.ts @@ -465,12 +465,24 @@ export function expectFuzzyMatch(str: string, sample: string | string[]) { const errorMessageNonAnsi = stripAnsi(str) const samples = typeof sample === "string" ? [sample] : sample const samplesNonAnsi = samples.map(stripAnsi) - try { - samplesNonAnsi.forEach((s) => expect(errorMessageNonAnsi.toLowerCase()).to.contain(s.toLowerCase())) - } catch (err) { - // eslint-disable-next-line no-console - console.log("Error message:\n", styles.error(errorMessageNonAnsi), "\n") - throw err + for (const s of samplesNonAnsi) { + const actualErrorMsgLowercase = errorMessageNonAnsi.toLowerCase() + const expectedErrorSample = s.toLowerCase() + try { + expect(actualErrorMsgLowercase).to.contain(expectedErrorSample) + } catch (err) { + // eslint-disable-next-line no-console + console.log( + "Expected string", + "\n", + `"${actualErrorMsgLowercase}"`, + "\n", + "to contain string", + "\n", + `"${expectedErrorSample}"` + ) + throw err + } } } From acde6d3f97bcb1fa0b562bc0b99e8e040e3d08b3 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Tue, 3 Dec 2024 14:30:05 +0100 Subject: [PATCH 06/43] test: fix some assertions --- core/test/unit/src/template-string.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/core/test/unit/src/template-string.ts b/core/test/unit/src/template-string.ts index bd4730593a..0f8846ec98 100644 --- a/core/test/unit/src/template-string.ts +++ b/core/test/unit/src/template-string.ts @@ -241,7 +241,7 @@ describe("resolveTemplateString", () => { it("should throw when a nested key is not found", () => { void expectError(() => resolveTemplateString({ string: "${some.other}", context: new TestContext({ some: {} }) }), { - contains: "Invalid template string (${some.other}): Could not find key other under some", + contains: "Invalid template string (${some.other}): Could not find key some.other", }) }) @@ -760,7 +760,7 @@ describe("resolveTemplateString", () => { () => resolveTemplateString({ string: "${foo[bar]}", context: new TestContext({ foo: {}, bar: {} }) }), { contains: - "Invalid template string (${foo[bar]}): Expression in bracket must resolve to a primitive (got object).", + "Invalid template string (${foo[bar]}): Expression in brackets must resolve to a string or number (got object).", } ) }) @@ -853,7 +853,7 @@ describe("resolveTemplateString", () => { }), { contains: - "Invalid template string (${nested.missing}): Could not find key missing under nested. Available keys: bar, baz and foo.", + "Invalid template string (${nested.missing}): Could not find key nested.missing. Available keys: bar, baz and foo.", } ) }) @@ -867,7 +867,7 @@ describe("resolveTemplateString", () => { }), { contains: - "Invalid template string (${nested.missing}): Could not find key missing under nested. Available keys: bar and foo.", + "Invalid template string (${nested.missing}): Could not find key nested.missing. Available keys: bar and foo.", } ) }) @@ -876,7 +876,7 @@ describe("resolveTemplateString", () => { const c = new TestContext({ nested: new TestContext({ deeper: {} }) }) void expectError(() => resolveTemplateString({ string: "${nested.deeper.missing}", context: c }), { - contains: "Invalid template string (${nested.deeper.missing}): Could not find key missing under nested.deeper.", + contains: "Invalid template string (${nested.deeper.missing}): Could not find key nested.deeper.missing.", }) }) @@ -884,7 +884,7 @@ describe("resolveTemplateString", () => { const c = new TestContext({ nested: new TestContext({ deeper: new TestContext({}) }) }) void expectError(() => resolveTemplateString({ string: "${nested.deeper.missing}", context: c }), { - contains: "Invalid template string (${nested.deeper.missing}): Could not find key missing under nested.deeper.", + contains: "Invalid template string (${nested.deeper.missing}): Could not find key nested.deeper.missing.", }) }) From 8a3e3cbec2db39f5bbb61b404a0fea29ab509072 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Tue, 3 Dec 2024 14:42:43 +0100 Subject: [PATCH 07/43] fix: read raw template string from input --- core/src/template-string/parser.pegjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/template-string/parser.pegjs b/core/src/template-string/parser.pegjs index 4ba216dcc4..c5f9d61cf7 100644 --- a/core/src/template-string/parser.pegjs +++ b/core/src/template-string/parser.pegjs @@ -7,13 +7,14 @@ */ { + const rawTemplateString = input + const { ast, escapePrefix, optionalSuffix, parseNested, TemplateStringError, - rawTemplateString, } = options function filledArray(count, value) { From 08dbd3927033ef38aa333fd20e911cc036d945f9 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Tue, 3 Dec 2024 15:29:18 +0100 Subject: [PATCH 08/43] test: fix tests for `resolveTemplateString` --- core/src/config/template-contexts/base.ts | 16 ++++++---- core/src/template-string/ast.ts | 6 ++-- core/test/unit/src/template-string.ts | 36 +++++++++++++++++------ 3 files changed, 41 insertions(+), 17 deletions(-) diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index 08eaab0f5e..fe9baab44e 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -124,9 +124,10 @@ export abstract class ConfigContext { if (typeof nextKey === "string" && nextKey.startsWith("_")) { value = undefined } else if (isPrimitive(value)) { - throw new ConfigurationError({ + return { + resolved: CONTEXT_RESOLVE_KEY_NOT_FOUND, message: `Attempted to look up key ${JSON.stringify(nextKey)} on a ${typeof value}.`, - }) + } } else if (value instanceof Map) { available = [...value.keys()] value = value.get(nextKey) @@ -138,9 +139,10 @@ export abstract class ConfigContext { if (typeof value === "function") { // call the function to resolve the value, then continue if (opts.stack.includes(stackEntry)) { - throw new ConfigurationError({ + return { + resolved: CONTEXT_RESOLVE_KEY_NOT_FOUND, message: `Circular reference detected when resolving key ${stackEntry} (from ${opts.stack.join(" -> ")})`, - }) + } } opts.stack.push(stackEntry) @@ -170,7 +172,7 @@ export abstract class ConfigContext { } } - if (value === undefined) { + if (value === undefined || typeof value === "symbol") { if (message === undefined) { message = styles.error(`Could not find key ${styles.highlight(String(nextKey))}`) if (nestedNodePath.length > 1) { @@ -190,6 +192,10 @@ export abstract class ConfigContext { } } + if (typeof resolved === "symbol") { + return { resolved, message } + } + // If we're allowing partial strings, we throw the error immediately to end the resolution flow. The error // is caught in the surrounding template resolution code. if (this._alwaysAllowPartial || opts.allowPartial) { diff --git a/core/src/template-string/ast.ts b/core/src/template-string/ast.ts index fa992a69e1..3b7e722a03 100644 --- a/core/src/template-string/ast.ts +++ b/core/src/template-string/ast.ts @@ -551,7 +551,7 @@ export class StringConcatExpression extends TemplateExpression { for (const expr of this.expressions) { const r = expr.evaluate(args) - if (r === CONTEXT_RESOLVE_KEY_NOT_FOUND) { + if (typeof r === "symbol") { return r } @@ -629,7 +629,7 @@ export class ContextLookupExpression extends TemplateExpression { keyPath.push(evaluated) } - const { resolved } = context.resolve({ + const { resolved, message } = context.resolve({ key: keyPath, nodePath: [], opts, @@ -648,7 +648,7 @@ export class ContextLookupExpression extends TemplateExpression { } throw new TemplateStringError({ - message: `Could not find key ${renderKeyPath(keyPath)}`, + message: message || `Could not find key ${renderKeyPath(keyPath)}`, rawTemplateString, loc: this.loc, }) diff --git a/core/test/unit/src/template-string.ts b/core/test/unit/src/template-string.ts index 0f8846ec98..0c5ab31650 100644 --- a/core/test/unit/src/template-string.ts +++ b/core/test/unit/src/template-string.ts @@ -241,7 +241,7 @@ describe("resolveTemplateString", () => { it("should throw when a nested key is not found", () => { void expectError(() => resolveTemplateString({ string: "${some.other}", context: new TestContext({ some: {} }) }), { - contains: "Invalid template string (${some.other}): Could not find key some.other", + contains: "Invalid template string (${some.other}): Could not find key other under some", }) }) @@ -853,7 +853,7 @@ describe("resolveTemplateString", () => { }), { contains: - "Invalid template string (${nested.missing}): Could not find key nested.missing. Available keys: bar, baz and foo.", + "Invalid template string (${nested.missing}): Could not find key missing under nested. Available keys: bar, baz and foo.", } ) }) @@ -867,7 +867,7 @@ describe("resolveTemplateString", () => { }), { contains: - "Invalid template string (${nested.missing}): Could not find key nested.missing. Available keys: bar and foo.", + "Invalid template string (${nested.missing}): Could not find key missing under nested. Available keys: bar and foo.", } ) }) @@ -876,7 +876,7 @@ describe("resolveTemplateString", () => { const c = new TestContext({ nested: new TestContext({ deeper: {} }) }) void expectError(() => resolveTemplateString({ string: "${nested.deeper.missing}", context: c }), { - contains: "Invalid template string (${nested.deeper.missing}): Could not find key nested.deeper.missing.", + contains: "Invalid template string (${nested.deeper.missing}): Could not find key missing under nested.deeper.", }) }) @@ -884,7 +884,7 @@ describe("resolveTemplateString", () => { const c = new TestContext({ nested: new TestContext({ deeper: new TestContext({}) }) }) void expectError(() => resolveTemplateString({ string: "${nested.deeper.missing}", context: c }), { - contains: "Invalid template string (${nested.deeper.missing}): Could not find key nested.deeper.missing.", + contains: "Invalid template string (${nested.deeper.missing}): Could not find key missing under nested.deeper.", }) }) @@ -971,16 +971,25 @@ describe("resolveTemplateString", () => { }) context("allowPartial=true", () => { - it("passes through template strings with missing key", () => { + it("does not resolve template expressions when 'b' is missing in the context", () => { + const res = resolveTemplateString({ + string: "${a}-${b}", + context: new TestContext({ a: "foo" }), + contextOpts: { allowPartial: true }, + }) + expect(res).to.equal("${a}-${b}") + }) + + it("does not resolve template expressions when 'a' is missing in the context", () => { const res = resolveTemplateString({ string: "${a}-${b}", context: new TestContext({ b: "foo" }), contextOpts: { allowPartial: true }, }) - expect(res).to.equal("${a}-foo") + expect(res).to.equal("${a}-${b}") }) - it("passes through a template string with a missing key in an optional clause", () => { + it("does not resolve template expressions when 'a' is missing in the context when evaluating a conditional expression", () => { const res = resolveTemplateString({ string: "${a || b}-${c}", context: new TestContext({ b: 123, c: "foo" }), @@ -988,7 +997,16 @@ describe("resolveTemplateString", () => { allowPartial: true, }, }) - expect(res).to.equal("${a || b}-foo") + expect(res).to.equal("${a || b}-${c}") + }) + + it("resolves template expressions when the context is fully available", () => { + const res = resolveTemplateString({ + string: "${a}-${b}", + context: new TestContext({ a: "foo", b: "bar" }), + contextOpts: { allowPartial: true }, + }) + expect(res).to.equal("foo-bar") }) }) }) From 65a7ae59e5f11d7ec0c34c0a2f0c5cf135885912 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Tue, 3 Dec 2024 15:49:26 +0100 Subject: [PATCH 09/43] chore: return value instead of throwing --- core/src/config/template-contexts/base.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index fe9baab44e..59f3b92778 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -98,9 +98,10 @@ export abstract class ConfigContext { opts.stack = [...(opts.stack || [])] if (opts.stack.includes(fullPath)) { - throw new ConfigurationError({ + return { + resolved: CONTEXT_RESOLVE_KEY_NOT_FOUND, message: `Circular reference detected when resolving key ${path} (${opts.stack.join(" -> ")})`, - }) + } } // keep track of which resolvers have been called, in order to detect circular references From 47b62a520081ec6284a0620a35eeb3d198153bc3 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Tue, 3 Dec 2024 16:08:36 +0100 Subject: [PATCH 10/43] test: fix remaining resolveTemplateString tests Co-authored-by: Vladimir Vagaytsev --- core/src/config/template-contexts/base.ts | 1 + core/src/template-string/ast.ts | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index 59f3b92778..6e2616f97a 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -95,6 +95,7 @@ export abstract class ConfigContext { return { resolved } } + // TODO: freeze opts object instead of using shallow copy opts.stack = [...(opts.stack || [])] if (opts.stack.includes(fullPath)) { diff --git a/core/src/template-string/ast.ts b/core/src/template-string/ast.ts index 3b7e722a03..48ba6ace12 100644 --- a/core/src/template-string/ast.ts +++ b/core/src/template-string/ast.ts @@ -632,7 +632,10 @@ export class ContextLookupExpression extends TemplateExpression { const { resolved, message } = context.resolve({ key: keyPath, nodePath: [], - opts, + // TODO: freeze opts object instead of using shallow copy + opts: { + ...opts, + }, }) // if context returns key available later, then we do not need to throw, because partial mode is enabled. From 3eb87243aea4c8ad24dd056be1c1f6455610f274 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Tue, 3 Dec 2024 16:27:20 +0100 Subject: [PATCH 11/43] fix: bug fix + unit test --- core/src/template-string/ast.ts | 4 +++- core/test/unit/src/template-string.ts | 13 +++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/core/src/template-string/ast.ts b/core/src/template-string/ast.ts index 48ba6ace12..3e6807ff77 100644 --- a/core/src/template-string/ast.ts +++ b/core/src/template-string/ast.ts @@ -538,6 +538,7 @@ export class IfBlockExpression extends TemplateExpression { export class StringConcatExpression extends TemplateExpression { public readonly expressions: TemplateExpression[] + constructor(loc: Location, ...expressions: TemplateExpression[]) { super(loc) this.expressions = expressions @@ -587,7 +588,7 @@ export class MemberExpression extends TemplateExpression { ): string | number | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { const inner = this.innerExpression.evaluate(args) - if (inner === CONTEXT_RESOLVE_KEY_NOT_FOUND) { + if (typeof inner === "symbol") { return inner } @@ -736,6 +737,7 @@ export class FunctionCallExpression extends TemplateExpression { } const loc = this.loc + class FunctionCallValidationError extends TemplateStringError { constructor({ message }: { message: string }) { super({ diff --git a/core/test/unit/src/template-string.ts b/core/test/unit/src/template-string.ts index 0c5ab31650..1c80825d09 100644 --- a/core/test/unit/src/template-string.ts +++ b/core/test/unit/src/template-string.ts @@ -67,6 +67,19 @@ describe("resolveTemplateString", () => { expect(res).to.equal("${foo}?") }) + it("should not crash when variable in a member expression cannot be resolved", () => { + const res = resolveTemplateString({ + string: '${actions.run["${inputs.deployableTarget}-dummy"].var}', + context: new TestContext({ + actions: { + run: {}, + }, + }), + contextOpts: { allowPartial: true }, + }) + expect(res).to.equal('${actions.run["${inputs.deployableTarget}-dummy"].var}') + }) + it("should support a string literal in a template string as a means to escape it", () => { const res = resolveTemplateString({ string: "${'$'}{bar}", context: new TestContext({}) }) expect(res).to.equal("${bar}") From 56dd85025f4c072031f22e19b2933f42107eaf68 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Tue, 3 Dec 2024 17:08:33 +0100 Subject: [PATCH 12/43] fix: test failures --- core/src/config/template-contexts/base.ts | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index 6e2616f97a..eb35f8014a 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -246,7 +246,7 @@ export class ScanContext extends ConfigContext { override resolve({ key, nodePath }: ContextResolveParams) { const fullKey = nodePath.concat(key) this.foundKeys.add(fullKey) - return { resolved: renderTemplateString(fullKey), partial: true } + return { resolved: CONTEXT_RESOLVE_KEY_AVAILABLE_LATER, partial: true } } } @@ -313,13 +313,6 @@ export class TemplateContext extends ConfigContext { } } -/** - * Given all the segments of a template string, return a new template string that can be resolved later. - */ -function renderTemplateString(key: ContextKeySegment[]) { - return "${" + renderKeyPath(key) + "}" -} - /** * Given all the segments of a template string, return a string path for the key. */ From 8aca5ea5ed5850696925d0d9a9cc64db9c4095d4 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Tue, 3 Dec 2024 17:26:22 +0100 Subject: [PATCH 13/43] fix: always use error message from context if it's available --- core/src/config/template-contexts/base.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index eb35f8014a..5863703018 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -203,6 +203,7 @@ export abstract class ConfigContext { if (this._alwaysAllowPartial || opts.allowPartial) { return { resolved: CONTEXT_RESOLVE_KEY_AVAILABLE_LATER, + message, } } else { return { resolved: CONTEXT_RESOLVE_KEY_NOT_FOUND, message } From dd1602bd5107a0c6ecfc993f2e0642973e83dc50 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Tue, 3 Dec 2024 18:19:03 +0100 Subject: [PATCH 14/43] perf: optimize `ConfigContext.resolve` --- core/src/config/template-contexts/base.ts | 91 ++++++++++--------- core/src/template-string/ast.ts | 4 +- .../unit/src/config/template-contexts/base.ts | 28 +++--- .../src/config/template-contexts/project.ts | 16 ++-- 4 files changed, 73 insertions(+), 66 deletions(-) diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index 5863703018..c6b5c4161c 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -24,7 +24,7 @@ export interface ContextResolveOpts { // Allow templates to be partially resolved (used to defer runtime template resolution, for example) allowPartial?: boolean // a list of previously resolved paths, used to detect circular references - stack?: string[] + stack?: Set // Unescape escaped template strings unescape?: boolean } @@ -36,7 +36,7 @@ export interface ContextResolveParams { } export interface ContextResolveOutput { - message?: string + getUnavailableReason?: () => string partial?: boolean resolved: any } @@ -96,68 +96,71 @@ export abstract class ConfigContext { } // TODO: freeze opts object instead of using shallow copy - opts.stack = [...(opts.stack || [])] + opts.stack = new Set(opts.stack || []) - if (opts.stack.includes(fullPath)) { + if (opts.stack.has(fullPath)) { return { resolved: CONTEXT_RESOLVE_KEY_NOT_FOUND, - message: `Circular reference detected when resolving key ${path} (${opts.stack.join(" -> ")})`, + getUnavailableReason: () => `Circular reference detected when resolving key ${path} (${(new Array(opts.stack || [])).join(" -> ")})`, } } // keep track of which resolvers have been called, in order to detect circular references - let available: any[] | null = null + let getAvailableKeys: (() => string[]) | undefined = undefined // eslint-disable-next-line @typescript-eslint/no-this-alias let value: any = this let partial = false let nextKey = key[0] - let lookupPath: ContextKeySegment[] = [] let nestedNodePath = nodePath - let message: string | undefined = undefined + let getUnavailableReason: (() => string) | undefined = undefined for (let p = 0; p < key.length; p++) { nextKey = key[p] - lookupPath = key.slice(0, p + 1) - const remainder = key.slice(p + 1) - nestedNodePath = nodePath.concat(lookupPath) - const stackEntry = renderKeyPath(nestedNodePath) - available = null + + const getRemainder = () => key.slice(p + 1) + const getNestedNodePath = () => nodePath.concat(key.slice(0, p + 1)) + const getStackEntry = () => renderKeyPath(getNestedNodePath()) + getAvailableKeys = undefined if (typeof nextKey === "string" && nextKey.startsWith("_")) { value = undefined } else if (isPrimitive(value)) { return { resolved: CONTEXT_RESOLVE_KEY_NOT_FOUND, - message: `Attempted to look up key ${JSON.stringify(nextKey)} on a ${typeof value}.`, + getUnavailableReason: () => `Attempted to look up key ${JSON.stringify(nextKey)} on a ${typeof value}.`, } } else if (value instanceof Map) { - available = [...value.keys()] + getAvailableKeys = () => value.keys() value = value.get(nextKey) } else { - available = Object.keys(value).filter((k) => !k.startsWith("_")) + getAvailableKeys = () => Object.keys(value).filter((k) => !k.startsWith("_")) value = value[nextKey] } if (typeof value === "function") { // call the function to resolve the value, then continue - if (opts.stack.includes(stackEntry)) { + const stackEntry = getStackEntry() + if (opts.stack?.has(stackEntry)) { return { resolved: CONTEXT_RESOLVE_KEY_NOT_FOUND, - message: `Circular reference detected when resolving key ${stackEntry} (from ${opts.stack.join(" -> ")})`, + getUnavailableReason: () => + `Circular reference detected when resolving key ${stackEntry} (from ${(new Array(opts.stack || [])).join(" -> ")})`, } } - opts.stack.push(stackEntry) - value = value({ key: remainder, nodePath: nestedNodePath, opts }) + opts.stack.add(stackEntry) + value = value({ key: getRemainder(), nodePath: nestedNodePath, opts }) } // handle nested contexts if (value instanceof ConfigContext) { + const remainder = getRemainder() if (remainder.length > 0) { - opts.stack.push(stackEntry) + const stackEntry = getStackEntry() + opts.stack.add(stackEntry) const res = value.resolve({ key: remainder, nodePath: nestedNodePath, opts }) value = res.resolved - message = res.message + getUnavailableReason = res.getUnavailableReason partial = !!res.partial } break @@ -165,7 +168,7 @@ export abstract class ConfigContext { // handle templated strings in context variables if (isString(value)) { - opts.stack.push(stackEntry) + opts.stack.add(getStackEntry()) value = resolveTemplateString({ string: value, context: this._rootContext, contextOpts: opts }) } @@ -175,27 +178,31 @@ export abstract class ConfigContext { } if (value === undefined || typeof value === "symbol") { - if (message === undefined) { - message = styles.error(`Could not find key ${styles.highlight(String(nextKey))}`) - if (nestedNodePath.length > 1) { - message += styles.error(" under ") + styles.highlight(renderKeyPath(nestedNodePath.slice(0, -1))) - } - message += styles.error(".") - - if (available) { - const availableStr = available.length - ? naturalList(available.sort().map((k) => styles.highlight(k))) - : "(none)" - message += styles.error(" Available keys: " + availableStr + ".") - } - const messageFooter = this.getMissingKeyErrorFooter(nextKey, nestedNodePath.slice(0, -1)) - if (messageFooter) { - message += `\n\n${messageFooter}` + if (getUnavailableReason === undefined) { + getUnavailableReason = () => { + let message = styles.error(`Could not find key ${styles.highlight(String(nextKey))}`) + if (nestedNodePath.length > 1) { + message += styles.error(" under ") + styles.highlight(renderKeyPath(nestedNodePath.slice(0, -1))) + } + message += styles.error(".") + + if (getAvailableKeys) { + const availableKeys = getAvailableKeys() + const availableStr = availableKeys.length + ? naturalList(availableKeys.sort().map((k) => styles.highlight(k))) + : "(none)" + message += styles.error(" Available keys: " + availableStr + ".") + } + const messageFooter = this.getMissingKeyErrorFooter(nextKey, nestedNodePath.slice(0, -1)) + if (messageFooter) { + message += `\n\n${messageFooter}` + } + return message } } if (typeof resolved === "symbol") { - return { resolved, message } + return { resolved, getUnavailableReason } } // If we're allowing partial strings, we throw the error immediately to end the resolution flow. The error @@ -203,10 +210,10 @@ export abstract class ConfigContext { if (this._alwaysAllowPartial || opts.allowPartial) { return { resolved: CONTEXT_RESOLVE_KEY_AVAILABLE_LATER, - message, + getUnavailableReason, } } else { - return { resolved: CONTEXT_RESOLVE_KEY_NOT_FOUND, message } + return { resolved: CONTEXT_RESOLVE_KEY_NOT_FOUND, getUnavailableReason } } } diff --git a/core/src/template-string/ast.ts b/core/src/template-string/ast.ts index 3e6807ff77..48b7210d64 100644 --- a/core/src/template-string/ast.ts +++ b/core/src/template-string/ast.ts @@ -630,7 +630,7 @@ export class ContextLookupExpression extends TemplateExpression { keyPath.push(evaluated) } - const { resolved, message } = context.resolve({ + const { resolved, getUnavailableReason } = context.resolve({ key: keyPath, nodePath: [], // TODO: freeze opts object instead of using shallow copy @@ -652,7 +652,7 @@ export class ContextLookupExpression extends TemplateExpression { } throw new TemplateStringError({ - message: message || `Could not find key ${renderKeyPath(keyPath)}`, + message: getUnavailableReason?.() || `Could not find key ${renderKeyPath(keyPath)}`, rawTemplateString, loc: this.loc, }) diff --git a/core/test/unit/src/config/template-contexts/base.ts b/core/test/unit/src/config/template-contexts/base.ts index 36911e0744..5121296db8 100644 --- a/core/test/unit/src/config/template-contexts/base.ts +++ b/core/test/unit/src/config/template-contexts/base.ts @@ -46,9 +46,9 @@ describe("ConfigContext", () => { it("should return undefined for missing key", async () => { const c = new TestContext({}) - const { resolved, message } = resolveKey(c, ["basic"]) + const { resolved, getUnavailableReason: message } = resolveKey(c, ["basic"]) expect(resolved).to.be.undefined - expect(stripAnsi(message!)).to.include("Could not find key basic") + expect(stripAnsi(message!())).to.include("Could not find key basic") }) context("allowPartial=true", () => { @@ -90,9 +90,9 @@ describe("ConfigContext", () => { const c = new TestContext({ nested: new TestContext({ key: "value" }), }) - const { resolved, message } = resolveKey(c, ["basic", "bla"]) + const { resolved, getUnavailableReason: message } = resolveKey(c, ["basic", "bla"]) expect(resolved).to.be.undefined - expect(stripAnsi(message!)).to.equal("Could not find key basic. Available keys: nested.") + expect(stripAnsi(message!())).to.equal("Could not find key basic. Available keys: nested.") }) it("should resolve keys with value behind callable", async () => { @@ -125,7 +125,7 @@ describe("ConfigContext", () => { nested: new TestContext({ key: "value" }), }) const key = ["nested", "key"] - const stack = [key.join(".")] + const stack = new Set([key.join(".")]) await expectError(() => c.resolve({ key, nodePath: [], opts: { stack } }), "configuration") }) @@ -133,7 +133,7 @@ describe("ConfigContext", () => { class NestedContext extends ConfigContext { override resolve({ key, nodePath, opts }: ContextResolveParams) { const circularKey = nodePath.concat(key) - opts.stack!.push(circularKey.join(".")) + opts.stack!.add(circularKey.join(".")) return c.resolve({ key: circularKey, nodePath: [], opts }) } } @@ -155,8 +155,8 @@ describe("ConfigContext", () => { } const c = new Context() - const { message } = resolveKey(c, ["nested", "bla"]) - expect(stripAnsi(message!)).to.include("Could not find key bla under nested.") + const { getUnavailableReason: message } = resolveKey(c, ["nested", "bla"]) + expect(stripAnsi(message!())).to.include("Could not find key bla under nested.") }) it("should show helpful error when unable to resolve nested key in object", async () => { @@ -171,8 +171,8 @@ describe("ConfigContext", () => { } const c = new Context() - const { message } = resolveKey(c, ["nested", "bla"]) - expect(stripAnsi(message!)).to.include("Could not find key bla under nested.") + const { getUnavailableReason: message } = resolveKey(c, ["nested", "bla"]) + expect(stripAnsi(message!())).to.include("Could not find key bla under nested.") }) it("should show helpful error when unable to resolve two-level nested key in object", async () => { @@ -187,8 +187,8 @@ describe("ConfigContext", () => { } const c = new Context() - const { message } = resolveKey(c, ["nested", "deeper", "bla"]) - expect(stripAnsi(message!)).to.include("Could not find key bla under nested.deeper.") + const { getUnavailableReason: message } = resolveKey(c, ["nested", "deeper", "bla"]) + expect(stripAnsi(message!())).to.include("Could not find key bla under nested.deeper.") }) it("should show helpful error when unable to resolve in nested context", async () => { @@ -204,8 +204,8 @@ describe("ConfigContext", () => { } const c = new Context() - const { message } = resolveKey(c, ["nested", "bla"]) - expect(stripAnsi(message!)).to.include("Could not find key bla under nested.") + const { getUnavailableReason: message } = resolveKey(c, ["nested", "bla"]) + expect(stripAnsi(message!())).to.include("Could not find key bla under nested.") }) it("should resolve template strings", async () => { diff --git a/core/test/unit/src/config/template-contexts/project.ts b/core/test/unit/src/config/template-contexts/project.ts index c8e94a3349..64fc541b96 100644 --- a/core/test/unit/src/config/template-contexts/project.ts +++ b/core/test/unit/src/config/template-contexts/project.ts @@ -149,9 +149,9 @@ describe("ProjectConfigContext", () => { commandInfo: { name: "test", args: {}, opts: {} }, }) - const { message } = c.resolve({ key: ["secrets", "bar"], nodePath: [], opts: {} }) + const { getUnavailableReason: message } = c.resolve({ key: ["secrets", "bar"], nodePath: [], opts: {} }) - expect(stripAnsi(message!)).to.match(/Please log in via the garden login command to use Garden with secrets/) + expect(stripAnsi(message!())).to.match(/Please log in via the garden login command to use Garden with secrets/) }) context("when logged in", () => { @@ -168,13 +168,13 @@ describe("ProjectConfigContext", () => { commandInfo: { name: "test", args: {}, opts: {} }, }) - const { message } = c.resolve({ key: ["secrets", "bar"], nodePath: [], opts: {} }) + const { getUnavailableReason: message } = c.resolve({ key: ["secrets", "bar"], nodePath: [], opts: {} }) const errMsg = deline` Looks like no secrets have been created for this project and/or environment in Garden Cloud. To create secrets, please visit ${enterpriseDomain} and navigate to the secrets section for this project. ` - expect(stripAnsi(message!)).to.match(new RegExp(errMsg)) + expect(stripAnsi(message!())).to.match(new RegExp(errMsg)) }) it("if a non-empty set of secrets was returned by the backend, provide a helpful suggestion", () => { @@ -190,13 +190,13 @@ describe("ProjectConfigContext", () => { commandInfo: { name: "test", args: {}, opts: {} }, }) - const { message } = c.resolve({ key: ["secrets", "bar"], nodePath: [], opts: {} }) + const { getUnavailableReason: message } = c.resolve({ key: ["secrets", "bar"], nodePath: [], opts: {} }) const errMsg = deline` Please make sure that all required secrets for this project exist in Garden Cloud, and are accessible in this environment. ` - expect(stripAnsi(message!)).to.match(new RegExp(errMsg)) + expect(stripAnsi(message!())).to.match(new RegExp(errMsg)) }) }) }) @@ -215,9 +215,9 @@ describe("ProjectConfigContext", () => { }) const key = "fiaogsyecgbsjyawecygaewbxrbxajyrgew" - const { message } = c.resolve({ key: ["local", "env", key], nodePath: [], opts: {} }) + const { getUnavailableReason: message } = c.resolve({ key: ["local", "env", key], nodePath: [], opts: {} }) - expect(stripAnsi(message!)).to.match( + expect(stripAnsi(message!())).to.match( /Could not find key fiaogsyecgbsjyawecygaewbxrbxajyrgew under local.env. Available keys: / ) }) From 02be20ec51fe870fef44fd7d0fa9e819f9313b0b Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Tue, 3 Dec 2024 19:04:12 +0100 Subject: [PATCH 15/43] perf: don't cache non-template strings and also cache unescape --- core/src/template-string/template-string.ts | 68 ++++++++++----------- 1 file changed, 31 insertions(+), 37 deletions(-) diff --git a/core/src/template-string/template-string.ts b/core/src/template-string/template-string.ts index b5c70ef8f2..988729afb6 100644 --- a/core/src/template-string/template-string.ts +++ b/core/src/template-string/template-string.ts @@ -85,23 +85,19 @@ export function parseTemplateString({ rawTemplateString: string source?: ConfigSource unescape: boolean -}): string | ast.TemplateExpression { - if (!unescape) { - const cached = parseTemplateStringCache.get(rawTemplateString) - - if (cached) { - return cached - } - } - +}): ast.TemplateExpression | string { // Just return immediately if this is definitely not a template string if (!maybeTemplateString(rawTemplateString)) { - if (!unescape) { - parseTemplateStringCache.set(rawTemplateString, rawTemplateString) - } return rawTemplateString } + const key = `u-${unescape ? "1" : "0"}-${rawTemplateString}` + const cached = parseTemplateStringCache.get(key) + + if (cached) { + return cached + } + if (source === undefined) { source = { basePath: [], @@ -121,9 +117,7 @@ export function parseTemplateString({ }, ]) - if (!unescape) { - parseTemplateStringCache.set(rawTemplateString, parsed) - } + parseTemplateStringCache.set(key, parsed) return parsed } @@ -157,32 +151,32 @@ export function resolveTemplateString({ unescape: shouldUnescape(contextOpts), }) - if (parsed instanceof ast.TemplateExpression) { - const result = parsed.evaluate({ - rawTemplateString: string, - context, - opts: contextOpts, - }) + // string does not contain + if (typeof parsed === "string") { + return parsed + } - if (!contextOpts.allowPartial && result === CONTEXT_RESOLVE_KEY_NOT_FOUND) { - throw new InternalError({ - message: "allowPartial is false, but template expression evaluated to symbol.", - }) - // TODO: think about if it's really ok to partially resolve if allowPartial is false. This can happen if a context with _alwaysPartial is used together with allowPartial false. - } else if (result === CONTEXT_RESOLVE_KEY_NOT_FOUND || result === CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) { - // The template expression cannot be evaluated yet, we may be able to do it later. - // TODO: return ast.TemplateExpression here, instead of string; Otherwise we'll inevitably have a bug - // where garden will resolve template expressions that might be contained in expression evaluation results - // e.g. if an environment variable contains template string, we don't want to evaluate the template string in there. - // See also https://github.com/garden-io/garden/issues/5825 - return string - } + const result = parsed.evaluate({ + rawTemplateString: string, + context, + opts: contextOpts, + }) - return result + if (!contextOpts.allowPartial && result === CONTEXT_RESOLVE_KEY_NOT_FOUND) { + throw new InternalError({ + message: "allowPartial is false, but template expression evaluated to symbol.", + }) + // TODO: think about if it's really ok to partially resolve if allowPartial is false. This can happen if a context with _alwaysPartial is used together with allowPartial false. + } else if (result === CONTEXT_RESOLVE_KEY_NOT_FOUND || result === CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) { + // The template expression cannot be evaluated yet, we may be able to do it later. + // TODO: return ast.TemplateExpression here, instead of string; Otherwise we'll inevitably have a bug + // where garden will resolve template expressions that might be contained in expression evaluation results + // e.g. if an environment variable contains template string, we don't want to evaluate the template string in there. + // See also https://github.com/garden-io/garden/issues/5825 + return string } - // string does not contain a template expression - return parsed + return result } /** From 9abba8a86474a3052c44536d5d94886d2d0986bb Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Wed, 4 Dec 2024 10:16:36 +0100 Subject: [PATCH 16/43] test: fix "exposes arguments and options correctly in command templates" --- core/src/template-string/ast.ts | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/core/src/template-string/ast.ts b/core/src/template-string/ast.ts index 48b7210d64..b45cb971bf 100644 --- a/core/src/template-string/ast.ts +++ b/core/src/template-string/ast.ts @@ -547,7 +547,7 @@ export class StringConcatExpression extends TemplateExpression { override evaluate( args: EvaluateArgs ): string | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { - const evaluatedExpressions: TemplatePrimitive[] = [] + let result: string = "" for (const expr of this.expressions) { const r = expr.evaluate(args) @@ -556,21 +556,14 @@ export class StringConcatExpression extends TemplateExpression { return r } - if (!isTemplatePrimitive(r)) { - throw new TemplateStringError({ - message: `Cannot concatenate: expected primitive, but expression resolved to ${typeof r}`, - rawTemplateString: args.rawTemplateString, - loc: this.loc, - }) + if (r === undefined) { + continue } - evaluatedExpressions.push(r) + // Calls toString when encountering non-primitives like objects or arrays. + result += `${r}` } - const result = evaluatedExpressions.reduce((acc, value) => { - return `${acc}${value === undefined ? "" : value}` - }, "") - return result } } From 95262d462fdaa7781b34ff7daca7c89f3fcfd8a8 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Wed, 4 Dec 2024 10:50:32 +0100 Subject: [PATCH 17/43] fix: throw on critical errors while context resolution --- core/src/config/template-contexts/base.ts | 34 +++++++++++-------- core/src/template-string/ast.ts | 33 +++++++++++++----- .../unit/src/config/template-contexts/base.ts | 6 ++-- 3 files changed, 48 insertions(+), 25 deletions(-) diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index c6b5c4161c..78bfd9df8e 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -8,7 +8,7 @@ import type Joi from "@hapi/joi" import { isString } from "lodash-es" -import { ConfigurationError } from "../../exceptions.js" +import { ConfigurationError, GardenError } from "../../exceptions.js" import { resolveTemplateString } from "../../template-string/template-string.js" import type { CustomObjectSchema } from "../common.js" import { isPrimitive, joi, joiIdentifier } from "../common.js" @@ -53,6 +53,14 @@ export interface ConfigContextType { getSchema(): CustomObjectSchema } +/** + * This error is thrown for a "final" errors, i.e. ones that cannot be ignored. + * For key not found errors that could be resolvable we still can return a special symbol. + */ +export class ContextResolveError extends GardenError { + type = "context-resolve" +} + export const CONTEXT_RESOLVE_KEY_NOT_FOUND: unique symbol = Symbol.for("ContextResolveKeyNotFound") export const CONTEXT_RESOLVE_KEY_AVAILABLE_LATER: unique symbol = Symbol.for("ContextResolveKeyAvailableLater") @@ -99,10 +107,10 @@ export abstract class ConfigContext { opts.stack = new Set(opts.stack || []) if (opts.stack.has(fullPath)) { - return { - resolved: CONTEXT_RESOLVE_KEY_NOT_FOUND, - getUnavailableReason: () => `Circular reference detected when resolving key ${path} (${(new Array(opts.stack || [])).join(" -> ")})`, - } + // Circular dependency error is critical, throwing here. + throw new ContextResolveError({ + message: `Circular reference detected when resolving key ${path} (${new Array(opts.stack || []).join(" -> ")})`, + }) } // keep track of which resolvers have been called, in order to detect circular references @@ -125,10 +133,9 @@ export abstract class ConfigContext { if (typeof nextKey === "string" && nextKey.startsWith("_")) { value = undefined } else if (isPrimitive(value)) { - return { - resolved: CONTEXT_RESOLVE_KEY_NOT_FOUND, - getUnavailableReason: () => `Attempted to look up key ${JSON.stringify(nextKey)} on a ${typeof value}.`, - } + throw new ContextResolveError({ + message: `Attempted to look up key ${JSON.stringify(nextKey)} on a ${typeof value}.`, + }) } else if (value instanceof Map) { getAvailableKeys = () => value.keys() value = value.get(nextKey) @@ -141,11 +148,10 @@ export abstract class ConfigContext { // call the function to resolve the value, then continue const stackEntry = getStackEntry() if (opts.stack?.has(stackEntry)) { - return { - resolved: CONTEXT_RESOLVE_KEY_NOT_FOUND, - getUnavailableReason: () => - `Circular reference detected when resolving key ${stackEntry} (from ${(new Array(opts.stack || [])).join(" -> ")})`, - } + // Circular dependency error is critical, throwing here. + throw new ContextResolveError({ + message: `Circular reference detected when resolving key ${stackEntry} (from ${new Array(opts.stack || []).join(" -> ")})`, + }) } opts.stack.add(stackEntry) diff --git a/core/src/template-string/ast.ts b/core/src/template-string/ast.ts index b45cb971bf..ff095c3fa9 100644 --- a/core/src/template-string/ast.ts +++ b/core/src/template-string/ast.ts @@ -13,6 +13,7 @@ import { renderKeyPath, type ConfigContext, type ContextResolveOpts, + ContextResolveError, } from "../config/template-contexts/base.js" import { InternalError, TemplateStringError } from "../exceptions.js" import { getHelperFunctions } from "./functions.js" @@ -623,14 +624,7 @@ export class ContextLookupExpression extends TemplateExpression { keyPath.push(evaluated) } - const { resolved, getUnavailableReason } = context.resolve({ - key: keyPath, - nodePath: [], - // TODO: freeze opts object instead of using shallow copy - opts: { - ...opts, - }, - }) + const { resolved, getUnavailableReason } = this.resolveContext(context, keyPath, opts, rawTemplateString) // if context returns key available later, then we do not need to throw, because partial mode is enabled. if (resolved === CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) { @@ -653,6 +647,29 @@ export class ContextLookupExpression extends TemplateExpression { return resolved } + + private resolveContext( + context: ConfigContext, + keyPath: (string | number)[], + opts: ContextResolveOpts, + rawTemplateString: string + ) { + try { + return context.resolve({ + key: keyPath, + nodePath: [], + // TODO: freeze opts object instead of using shallow copy + opts: { + ...opts, + }, + }) + } catch (e) { + if (e instanceof ContextResolveError) { + throw new TemplateStringError({ message: e.message, rawTemplateString, loc: this.loc }) + } + throw e + } + } } export class FunctionCallExpression extends TemplateExpression { diff --git a/core/test/unit/src/config/template-contexts/base.ts b/core/test/unit/src/config/template-contexts/base.ts index 5121296db8..d7aaa2b9c9 100644 --- a/core/test/unit/src/config/template-contexts/base.ts +++ b/core/test/unit/src/config/template-contexts/base.ts @@ -71,7 +71,7 @@ describe("ConfigContext", () => { it("should throw when looking for nested value on primitive", async () => { const c = new TestContext({ basic: "value" }) - await expectError(() => resolveKey(c, ["basic", "nested"]), "configuration") + await expectError(() => resolveKey(c, ["basic", "nested"]), "context-resolve") }) it("should resolve nested keys", async () => { @@ -126,7 +126,7 @@ describe("ConfigContext", () => { }) const key = ["nested", "key"] const stack = new Set([key.join(".")]) - await expectError(() => c.resolve({ key, nodePath: [], opts: { stack } }), "configuration") + await expectError(() => c.resolve({ key, nodePath: [], opts: { stack } }), "context-resolve") }) it("should detect a circular reference from a nested context", async () => { @@ -141,7 +141,7 @@ describe("ConfigContext", () => { const c = new TestContext({ nested: new NestedContext(), }) - await expectError(() => resolveKey(c, ["nested", "bla"]), "configuration") + await expectError(() => resolveKey(c, ["nested", "bla"]), "context-resolve") }) it("should return helpful message when unable to resolve nested key in map", async () => { From 8f9daaec886c35dfd84ff8e324fce1dc46578c83 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Wed, 4 Dec 2024 10:55:23 +0100 Subject: [PATCH 18/43] test: fix assertions --- .../unit/src/config/template-contexts/base.ts | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/core/test/unit/src/config/template-contexts/base.ts b/core/test/unit/src/config/template-contexts/base.ts index d7aaa2b9c9..54ff46fda5 100644 --- a/core/test/unit/src/config/template-contexts/base.ts +++ b/core/test/unit/src/config/template-contexts/base.ts @@ -8,7 +8,11 @@ import { expect } from "chai" import stripAnsi from "strip-ansi" -import type { ContextKey, ContextResolveParams } from "../../../../../src/config/template-contexts/base.js" +import { + CONTEXT_RESOLVE_KEY_AVAILABLE_LATER, + ContextKey, + ContextResolveParams, +} from "../../../../../src/config/template-contexts/base.js" import { ConfigContext, schema, ScanContext } from "../../../../../src/config/template-contexts/base.js" import { expectError } from "../../../../helpers.js" import { joi } from "../../../../../src/config/common.js" @@ -52,20 +56,18 @@ describe("ConfigContext", () => { }) context("allowPartial=true", () => { - it("should throw on missing key when allowPartial=true", async () => { + it("should return CONTEXT_RESOLVE_KEY_AVAILABLE_LATER symbol on missing key", async () => { const c = new TestContext({}) - await expectError(() => resolveKey(c, ["basic"], { allowPartial: true }), { - contains: "Could not find key basic", - }) + const result = resolveKey(c, ["basic"], { allowPartial: true }) + expect(result.resolved).to.eql(CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) }) - it("should throw on missing key on nested context", async () => { + it("should return CONTEXT_RESOLVE_KEY_AVAILABLE_LATER symbol on missing key on nested context", async () => { const c = new TestContext({ nested: new TestContext({ key: "value" }), }) - await expectError(() => resolveKey(c, ["nested", "bla"], { allowPartial: true }), { - contains: "Could not find key bla under nested. Available keys: key.", - }) + const result = resolveKey(c, ["nested", "bla"], { allowPartial: true }) + expect(result.resolved).to.eql(CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) }) }) From b65c093f49c2167432220cdea409204bfea051af Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Wed, 4 Dec 2024 11:10:37 +0100 Subject: [PATCH 19/43] test: fix undefined errors in ConfigContext tests Co-authored-by: Vladimir Vagaytsev --- core/src/config/template-contexts/base.ts | 29 +++++++++++-------- .../unit/src/config/template-contexts/base.ts | 9 +++--- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index 78bfd9df8e..624a573be7 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -11,11 +11,13 @@ import { isString } from "lodash-es" import { ConfigurationError, GardenError } from "../../exceptions.js" import { resolveTemplateString } from "../../template-string/template-string.js" import type { CustomObjectSchema } from "../common.js" -import { isPrimitive, joi, joiIdentifier } from "../common.js" +import { joi, joiIdentifier } from "../common.js" import { KeyedSet } from "../../util/keyed-set.js" import { naturalList } from "../../util/string.js" import { styles } from "../../logger/styles.js" import { Profile } from "../../util/profiling.js" +import { CollectionOrValue } from "../../util/objects.js" +import { isTemplatePrimitive, TemplatePrimitive } from "../../template-string/types.js" export type ContextKeySegment = string | number export type ContextKey = ContextKeySegment[] @@ -115,8 +117,8 @@ export abstract class ConfigContext { // keep track of which resolvers have been called, in order to detect circular references let getAvailableKeys: (() => string[]) | undefined = undefined - // eslint-disable-next-line @typescript-eslint/no-this-alias - let value: any = this + + let value: CollectionOrValue | ConfigContext | Function = this let partial = false let nextKey = key[0] let nestedNodePath = nodePath @@ -130,18 +132,21 @@ export abstract class ConfigContext { const getStackEntry = () => renderKeyPath(getNestedNodePath()) getAvailableKeys = undefined - if (typeof nextKey === "string" && nextKey.startsWith("_")) { - value = undefined - } else if (isPrimitive(value)) { + const parent: CollectionOrValue | ConfigContext | Function = value + if (isTemplatePrimitive(parent)) { throw new ContextResolveError({ - message: `Attempted to look up key ${JSON.stringify(nextKey)} on a ${typeof value}.`, + message: `Attempted to look up key ${JSON.stringify(nextKey)} on a ${typeof parent}.`, }) - } else if (value instanceof Map) { - getAvailableKeys = () => value.keys() - value = value.get(nextKey) + } else if (typeof nextKey === "string" && nextKey.startsWith("_")) { + value = undefined + } else if (parent instanceof Map) { + getAvailableKeys = () => Array.from(parent.keys()) + value = parent.get(nextKey) } else { - getAvailableKeys = () => Object.keys(value).filter((k) => !k.startsWith("_")) - value = value[nextKey] + getAvailableKeys = () => { + return Object.keys(parent).filter((k) => !k.startsWith("_")) + } + value = parent[nextKey] } if (typeof value === "function") { diff --git a/core/test/unit/src/config/template-contexts/base.ts b/core/test/unit/src/config/template-contexts/base.ts index 54ff46fda5..8dac3d1d98 100644 --- a/core/test/unit/src/config/template-contexts/base.ts +++ b/core/test/unit/src/config/template-contexts/base.ts @@ -10,6 +10,7 @@ import { expect } from "chai" import stripAnsi from "strip-ansi" import { CONTEXT_RESOLVE_KEY_AVAILABLE_LATER, + CONTEXT_RESOLVE_KEY_NOT_FOUND, ContextKey, ContextResolveParams, } from "../../../../../src/config/template-contexts/base.js" @@ -48,10 +49,10 @@ describe("ConfigContext", () => { expect(resolveKey(c, ["basic"])).to.eql({ resolved: "value" }) }) - it("should return undefined for missing key", async () => { + it("should return CONTEXT_RESOLVE_KEY_NOT_FOUND for missing key", async () => { const c = new TestContext({}) const { resolved, getUnavailableReason: message } = resolveKey(c, ["basic"]) - expect(resolved).to.be.undefined + expect(resolved).to.be.equal(CONTEXT_RESOLVE_KEY_NOT_FOUND) expect(stripAnsi(message!())).to.include("Could not find key basic") }) @@ -88,12 +89,12 @@ describe("ConfigContext", () => { expect(resolveKey(c, ["nested", "key"])).eql({ resolved: "value" }) }) - it("should return undefined for missing keys on nested context", async () => { + it("should return CONTEXT_RESOLVE_KEY_NOT_FOUND for missing keys on nested context", async () => { const c = new TestContext({ nested: new TestContext({ key: "value" }), }) const { resolved, getUnavailableReason: message } = resolveKey(c, ["basic", "bla"]) - expect(resolved).to.be.undefined + expect(resolved).to.be.equal(CONTEXT_RESOLVE_KEY_NOT_FOUND) expect(stripAnsi(message!())).to.equal("Could not find key basic. Available keys: nested.") }) From dc1205ef09e28c710cfacd2eb47500758801a1db Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Wed, 4 Dec 2024 11:33:14 +0100 Subject: [PATCH 20/43] test: fix remaining ConfigContext tests --- core/src/config/template-contexts/base.ts | 10 ++++++---- core/src/exceptions.ts | 2 ++ core/src/template-string/ast.ts | 4 ++++ core/test/unit/src/config/template-contexts/base.ts | 2 +- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index 624a573be7..02822089b5 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -111,7 +111,7 @@ export abstract class ConfigContext { if (opts.stack.has(fullPath)) { // Circular dependency error is critical, throwing here. throw new ContextResolveError({ - message: `Circular reference detected when resolving key ${path} (${new Array(opts.stack || []).join(" -> ")})`, + message: `Circular reference detected when resolving key ${path} (${Array.from(opts.stack || []).join(" -> ")})`, }) } @@ -127,9 +127,11 @@ export abstract class ConfigContext { for (let p = 0; p < key.length; p++) { nextKey = key[p] + nestedNodePath = nodePath.concat(key.slice(0, p + 1)) const getRemainder = () => key.slice(p + 1) - const getNestedNodePath = () => nodePath.concat(key.slice(0, p + 1)) - const getStackEntry = () => renderKeyPath(getNestedNodePath()) + + const capturedNestedNodePath = nestedNodePath + const getStackEntry = () => renderKeyPath(capturedNestedNodePath) getAvailableKeys = undefined const parent: CollectionOrValue | ConfigContext | Function = value @@ -155,7 +157,7 @@ export abstract class ConfigContext { if (opts.stack?.has(stackEntry)) { // Circular dependency error is critical, throwing here. throw new ContextResolveError({ - message: `Circular reference detected when resolving key ${stackEntry} (from ${new Array(opts.stack || []).join(" -> ")})`, + message: `Circular reference detected when resolving key ${stackEntry} (from ${Array.from(opts.stack || []).join(" -> ")})`, }) } diff --git a/core/src/exceptions.ts b/core/src/exceptions.ts index 18ab111c79..f8269682f4 100644 --- a/core/src/exceptions.ts +++ b/core/src/exceptions.ts @@ -311,6 +311,7 @@ export class TemplateStringError extends GardenError { loc?: Location rawTemplateString: string + originalMessage: string constructor(params: GardenErrorParams & { rawTemplateString: string; loc?: Location }) { // TODO: figure out how to get correct path @@ -325,6 +326,7 @@ export class TemplateStringError extends GardenError { super({ ...params, message }) this.loc = params.loc this.rawTemplateString = params.rawTemplateString + this.originalMessage = params.message } } diff --git a/core/src/template-string/ast.ts b/core/src/template-string/ast.ts index ff095c3fa9..9d72e84bf2 100644 --- a/core/src/template-string/ast.ts +++ b/core/src/template-string/ast.ts @@ -22,6 +22,7 @@ import type { Collection, CollectionOrValue } from "../util/objects.js" import type { ConfigSource } from "../config/validation.js" import { validateSchema } from "../config/validation.js" import type { TemplateExpressionGenerator } from "./static-analysis.js" +import { instance } from "testdouble" type EvaluateArgs = { context: ConfigContext @@ -667,6 +668,9 @@ export class ContextLookupExpression extends TemplateExpression { if (e instanceof ContextResolveError) { throw new TemplateStringError({ message: e.message, rawTemplateString, loc: this.loc }) } + if (e instanceof TemplateStringError) { + throw new TemplateStringError({ message: e.originalMessage, rawTemplateString, loc: this.loc }) + } throw e } } diff --git a/core/test/unit/src/config/template-contexts/base.ts b/core/test/unit/src/config/template-contexts/base.ts index 8dac3d1d98..eab122e5a5 100644 --- a/core/test/unit/src/config/template-contexts/base.ts +++ b/core/test/unit/src/config/template-contexts/base.ts @@ -274,7 +274,7 @@ describe("ConfigContext", () => { c.addValues({ nested }) await expectError(() => resolveKey(c, ["nested", "key"]), { contains: - "Invalid template string (${'${nested.key}'}): Invalid template string (${nested.key}): Circular reference detected when resolving key nested.key (nested -> nested.key)", + "Invalid template string (${'${nested.key}'}): Circular reference detected when resolving key nested.key (nested -> nested.key)", }) }) }) From 1829f266f952a3124167395444914410d7e26b6c Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Wed, 4 Dec 2024 12:32:55 +0100 Subject: [PATCH 21/43] wip: use static analysis instead of scancontext ScanContext does not work with AST, because it does not eagerly resolve all variables anymore. we have to walk the AST intead. --- core/src/template-string/static-analysis.ts | 102 +++++++++---- core/src/template-string/template-string.ts | 157 +++++++++++--------- core/test/unit/src/template-string.ts | 4 +- 3 files changed, 161 insertions(+), 102 deletions(-) diff --git a/core/src/template-string/static-analysis.ts b/core/src/template-string/static-analysis.ts index 5c3fd894d5..2f13a17239 100644 --- a/core/src/template-string/static-analysis.ts +++ b/core/src/template-string/static-analysis.ts @@ -17,30 +17,47 @@ import { TemplateExpression, } from "./ast.js" import type { TemplatePrimitive } from "./types.js" +import { parseTemplateString } from "./template-string.js" +import { ConfigContext, ScanContext } from "../config/template-contexts/base.js" export type TemplateExpressionGenerator = Generator -export function* visitAll( +export function* visitAll({ + value, + parseTemplateStrings = false, +}: { value: CollectionOrValue -): TemplateExpressionGenerator { + parseTemplateStrings?: boolean +}): TemplateExpressionGenerator { if (isArray(value)) { for (const [_k, v] of value.entries()) { - yield* visitAll(v) + yield* visitAll({ value: v, parseTemplateStrings }) } } else if (isPlainObject(value)) { for (const k of Object.keys(value)) { - yield* visitAll(value[k]) + yield* visitAll({ value: value[k], parseTemplateStrings }) } } else { - yield value + if (parseTemplateStrings && typeof value === "string") { + const parsed = parseTemplateString({ + rawTemplateString: value, + unescape: false, + }) - if (value instanceof TemplateExpression) { + if (typeof parsed === "string") { + yield parsed + } else { + yield* parsed.visitAll() + } + } else if (value instanceof TemplateExpression) { yield* value.visitAll() + } else { + yield value } } } -export function containsTemplateExpression(value: CollectionOrValue): boolean { - for (const node of visitAll(value)) { +export function containsTemplateExpression(generator: TemplateExpressionGenerator): boolean { + for (const node of generator) { if (node instanceof TemplateExpression) { return true } @@ -50,48 +67,75 @@ export function containsTemplateExpression(value: CollectionOrValue, - path: (string | number)[] + generator: TemplateExpressionGenerator, ): boolean { - for (const keyPath of getContextLookupReferences(value)) { - // TODO: What if the key name contains dots? We should compare arrays instead of comparing joined strings. - if (startsWith(`${keyPath.join(".")}.`, `${path.join(".")}.`)) { - return true - } + for (const finding of getContextLookupReferences(generator)) { + return true } return false } +export type ContextLookupReferenceFinding = + | { + type: "static" + keyPath: (string | number)[] + } + | { + type: "dynamic" + keyPath: (string | number | TemplateExpression)[] + } + | { + type: "invalid" + keyPath: unknown[] + } + export function* getContextLookupReferences( - value: CollectionOrValue> -): Generator<(string | number)[], void, undefined> { - for (const expression of visitAll(value)) { + generator: TemplateExpressionGenerator +): Generator { + for (const expression of generator) { if (expression instanceof ContextLookupExpression) { - const keyPath: (string | number)[] = [] + let type: ContextLookupReferenceFinding["type"] | undefined = undefined + const keyPath: any[] = [] for (const v of expression.keyPath.values()) { if (v instanceof IdentifierExpression) { keyPath.push(v.name) } else if (v instanceof MemberExpression) { - if (v.innerExpression instanceof LiteralExpression) { - if (isString(v.innerExpression.literal) || isNumber(v.innerExpression.literal)) { - keyPath.push(v.innerExpression.literal) - } else { - // only strings and numbers are valid here - break + if (containsContextLookupReferences(v.innerExpression.visitAll())) { + // do not override invalid + if (type !== "invalid") { + type = "dynamic" } + keyPath.push(v.innerExpression) } else { - // it's a dynamic key, so we can't statically analyse the value - break + // can be evaluated statically + const result = v.innerExpression.evaluate({ + context: new ScanContext(), + rawTemplateString: "", + opts: {}, + }) + + if (isString(result) || isNumber(result)) { + keyPath.push(result) + type ||= "static" + } else { + keyPath.push(result) + // if it's invalid, we override to invalid + type = "invalid" + } + } } else { v satisfies never } } - if (keyPath.length > 0) { - yield keyPath + if (type && keyPath.length > 0) { + yield { + keyPath, + type, + } } } } diff --git a/core/src/template-string/template-string.ts b/core/src/template-string/template-string.ts index 988729afb6..b96022af59 100644 --- a/core/src/template-string/template-string.ts +++ b/core/src/template-string/template-string.ts @@ -7,7 +7,13 @@ */ import type { GardenErrorParams } from "../exceptions.js" -import { ConfigurationError, GardenError, InternalError, TemplateStringError } from "../exceptions.js" +import { + ConfigurationError, + GardenError, + InternalError, + NotImplementedError, + TemplateStringError, +} from "../exceptions.js" import type { ConfigContext, ContextKeySegment, ContextResolveOpts } from "../config/template-contexts/base.js" import { CONTEXT_RESOLVE_KEY_AVAILABLE_LATER } from "../config/template-contexts/base.js" import { CONTEXT_RESOLVE_KEY_NOT_FOUND, GenericContext, ScanContext } from "../config/template-contexts/base.js" @@ -40,6 +46,7 @@ import type { ObjectPath } from "../config/base.js" import type { TemplatePrimitive } from "./types.js" import * as ast from "./ast.js" import { LRUCache } from "lru-cache" +import { ContextLookupReferenceFinding, getContextLookupReferences, visitAll } from "./static-analysis.js" const escapePrefix = "$${" @@ -563,10 +570,7 @@ export function mayContainTemplateString(obj: any): boolean { * Scans for all template strings in the given object and lists the referenced keys. */ export function collectTemplateReferences(obj: object): ContextKeySegment[][] { - // TODO: Statically analyse AST instead of using ScanContext - const context = new ScanContext() - resolveTemplateStrings({ value: obj, context, contextOpts: { allowPartial: true }, source: undefined }) - return uniq(context.foundKeys.entries()).sort() + throw new NotImplementedError({ message: "TODO: Remove this function" }) } export function getRuntimeTemplateReferences(obj: object) { @@ -584,95 +588,106 @@ interface ActionTemplateReference extends ActionReference { * * TODO-0.13.1: Allow such nested references in certain cases, e.g. if resolvable with a ProjectConfigContext. */ -export function getActionTemplateReferences(config: object): ActionTemplateReference[] { - const rawRefs = collectTemplateReferences(config) - - // ${action.*} - const refs: ActionTemplateReference[] = rawRefs - .filter((ref) => ref[0] === "actions") - .map((ref) => { - if (!ref[1]) { - throw new ConfigurationError({ - message: `Found invalid action reference (missing kind).`, - }) - } - if (!isString(ref[1])) { - throw new ConfigurationError({ - message: `Found invalid action reference (kind is not a string).`, - }) - } - if (!actionKindsLower.includes(ref[1])) { - throw new ConfigurationError({ - message: `Found invalid action reference (invalid kind '${ref[1]}')`, - }) - } - - if (!ref[2]) { - throw new ConfigurationError({ - message: "Found invalid action reference (missing name)", - }) - } - if (!isString(ref[2])) { - throw new ConfigurationError({ - message: "Found invalid action reference (name is not a string)", - }) - } - - return { - kind: titleize(ref[1]), - name: ref[2], - fullRef: ref, - } - }) - - // ${runtime.*} - for (const ref of rawRefs) { - if (ref[0] !== "runtime") { +export function* getActionTemplateReferences(config: object): Generator { + const generator = getContextLookupReferences( + visitAll({ value: config as CollectionOrValue, parseTemplateStrings: true }) + ) + + for (const finding of generator) { + // we are interested in ${action.*} + if (finding.keyPath[0] !== "actions") { continue } - let kind: ActionKind - - if (!ref[1]) { + if (finding.type !== "static") { throw new ConfigurationError({ - message: "Found invalid runtime reference (missing kind)", + message: `Found invalid action reference (resolving variables in actions is not allowed).`, }) } - if (!isString(ref[1])) { + + if (!finding.keyPath[1]) { throw new ConfigurationError({ - message: "Found invalid runtime reference (kind is not a string)", + message: `Found invalid action reference (missing kind).`, }) } - if (ref[1] === "services") { - kind = "Deploy" - } else if (ref[1] === "tasks") { - kind = "Run" - } else { + if (!isString(finding.keyPath[1])) { + throw new ConfigurationError({ + message: `Found invalid action reference (kind is not a string).`, + }) + } + if (!actionKindsLower.includes(finding.keyPath[1])) { throw new ConfigurationError({ - message: `Found invalid runtime reference (invalid kind '${ref[1]}')`, + message: `Found invalid action reference (invalid kind '${finding.keyPath[1]}')`, }) } - if (!ref[2]) { + if (!finding.keyPath[2]) { throw new ConfigurationError({ - message: `Found invalid runtime reference (missing name)`, + message: "Found invalid action reference (missing name)", }) } - if (!isString(ref[2])) { + if (!isString(finding.keyPath[2])) { throw new ConfigurationError({ - message: "Found invalid runtime reference (name is not a string)", + message: "Found invalid action reference (name is not a string)", }) } - refs.push({ - kind, - name: ref[2], - fullRef: ref, - }) + yield { + kind: titleize(finding.keyPath[1]), + name: finding.keyPath[2], + fullRef: finding.keyPath, + } } - return refs + // // ${runtime.*} + // for (const ref of rawRefs) { + // if (ref[0] !== "runtime") { + // continue + // } + + // let kind: ActionKind + + // if (!ref[1]) { + // throw new ConfigurationError({ + // message: "Found invalid runtime reference (missing kind)", + // }) + // } + // if (!isString(ref[1])) { + // throw new ConfigurationError({ + // message: "Found invalid runtime reference (kind is not a string)", + // }) + // } + + // if (ref[1] === "services") { + // kind = "Deploy" + // } else if (ref[1] === "tasks") { + // kind = "Run" + // } else { + // throw new ConfigurationError({ + // message: `Found invalid runtime reference (invalid kind '${ref[1]}')`, + // }) + // } + + // if (!ref[2]) { + // throw new ConfigurationError({ + // message: `Found invalid runtime reference (missing name)`, + // }) + // } + // if (!isString(ref[2])) { + // throw new ConfigurationError({ + // message: "Found invalid runtime reference (name is not a string)", + // }) + // } + + // refs.push({ + // kind, + // name: ref[2], + // fullRef: ref, + // }) + // } + + // return refs } export function getModuleTemplateReferences(obj: object, context: ModuleConfigContext) { diff --git a/core/test/unit/src/template-string.ts b/core/test/unit/src/template-string.ts index 1c80825d09..81b16d6eeb 100644 --- a/core/test/unit/src/template-string.ts +++ b/core/test/unit/src/template-string.ts @@ -2197,7 +2197,7 @@ describe("getActionTemplateReferences", () => { run: '${actions["run"].run-a}', test: '${actions["test"].test-a}', } - const actionTemplateReferences = getActionTemplateReferences(config) + const actionTemplateReferences = Array.from(getActionTemplateReferences(config)) expect(actionTemplateReferences).to.eql([ { kind: "Build", @@ -2253,7 +2253,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: "${actions[foo.bar].some-name}", } - void expectError(() => getActionTemplateReferences(config), { + void expectError(() => Array.from(getActionTemplateReferences(config)), { contains: "Found invalid action reference (invalid kind '${foo.bar}')", }) }) From 6fa3feae8079355d33e6c339ba069b6ce50726b0 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Wed, 4 Dec 2024 12:52:19 +0100 Subject: [PATCH 22/43] chore: lint fixes --- core/src/config/template-contexts/base.ts | 5 +++-- core/src/template-string/ast.ts | 1 - core/src/template-string/static-analysis.ts | 18 +++++------------- core/src/template-string/template-string.ts | 6 +++--- core/src/util/testing.ts | 1 - .../unit/src/config/template-contexts/base.ts | 3 +-- 6 files changed, 12 insertions(+), 22 deletions(-) diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index 02822089b5..a931c2a608 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -16,8 +16,9 @@ import { KeyedSet } from "../../util/keyed-set.js" import { naturalList } from "../../util/string.js" import { styles } from "../../logger/styles.js" import { Profile } from "../../util/profiling.js" -import { CollectionOrValue } from "../../util/objects.js" -import { isTemplatePrimitive, TemplatePrimitive } from "../../template-string/types.js" +import type { CollectionOrValue } from "../../util/objects.js" +import type { TemplatePrimitive } from "../../template-string/types.js" +import { isTemplatePrimitive } from "../../template-string/types.js" export type ContextKeySegment = string | number export type ContextKey = ContextKeySegment[] diff --git a/core/src/template-string/ast.ts b/core/src/template-string/ast.ts index 9d72e84bf2..10cefb9ba8 100644 --- a/core/src/template-string/ast.ts +++ b/core/src/template-string/ast.ts @@ -22,7 +22,6 @@ import type { Collection, CollectionOrValue } from "../util/objects.js" import type { ConfigSource } from "../config/validation.js" import { validateSchema } from "../config/validation.js" import type { TemplateExpressionGenerator } from "./static-analysis.js" -import { instance } from "testdouble" type EvaluateArgs = { context: ConfigContext diff --git a/core/src/template-string/static-analysis.ts b/core/src/template-string/static-analysis.ts index 2f13a17239..6bfddf357a 100644 --- a/core/src/template-string/static-analysis.ts +++ b/core/src/template-string/static-analysis.ts @@ -6,21 +6,16 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import { isNumber, isString, startsWith } from "lodash-es" +import { isNumber, isString } from "lodash-es" import type { CollectionOrValue } from "../util/objects.js" import { isArray, isPlainObject } from "../util/objects.js" -import { - ContextLookupExpression, - IdentifierExpression, - LiteralExpression, - MemberExpression, - TemplateExpression, -} from "./ast.js" +import { ContextLookupExpression, IdentifierExpression, MemberExpression, TemplateExpression } from "./ast.js" import type { TemplatePrimitive } from "./types.js" import { parseTemplateString } from "./template-string.js" -import { ConfigContext, ScanContext } from "../config/template-contexts/base.js" +import { ScanContext } from "../config/template-contexts/base.js" export type TemplateExpressionGenerator = Generator + export function* visitAll({ value, parseTemplateStrings = false, @@ -66,9 +61,7 @@ export function containsTemplateExpression(generator: TemplateExpressionGenerato return false } -export function containsContextLookupReferences( - generator: TemplateExpressionGenerator, -): boolean { +export function containsContextLookupReferences(generator: TemplateExpressionGenerator): boolean { for (const finding of getContextLookupReferences(generator)) { return true } @@ -124,7 +117,6 @@ export function* getContextLookupReferences( // if it's invalid, we override to invalid type = "invalid" } - } } else { v satisfies never diff --git a/core/src/template-string/template-string.ts b/core/src/template-string/template-string.ts index b96022af59..fd936c3e46 100644 --- a/core/src/template-string/template-string.ts +++ b/core/src/template-string/template-string.ts @@ -16,9 +16,9 @@ import { } from "../exceptions.js" import type { ConfigContext, ContextKeySegment, ContextResolveOpts } from "../config/template-contexts/base.js" import { CONTEXT_RESOLVE_KEY_AVAILABLE_LATER } from "../config/template-contexts/base.js" -import { CONTEXT_RESOLVE_KEY_NOT_FOUND, GenericContext, ScanContext } from "../config/template-contexts/base.js" +import { CONTEXT_RESOLVE_KEY_NOT_FOUND, GenericContext } from "../config/template-contexts/base.js" import cloneDeep from "fast-copy" -import { difference, isPlainObject, isString, uniq } from "lodash-es" +import { difference, isPlainObject, isString } from "lodash-es" import type { ActionReference, Primitive, StringMap } from "../config/common.js" import { arrayConcatKey, @@ -46,7 +46,7 @@ import type { ObjectPath } from "../config/base.js" import type { TemplatePrimitive } from "./types.js" import * as ast from "./ast.js" import { LRUCache } from "lru-cache" -import { ContextLookupReferenceFinding, getContextLookupReferences, visitAll } from "./static-analysis.js" +import { getContextLookupReferences, visitAll } from "./static-analysis.js" const escapePrefix = "$${" diff --git a/core/src/util/testing.ts b/core/src/util/testing.ts index ede7c589bc..78983f7fbf 100644 --- a/core/src/util/testing.ts +++ b/core/src/util/testing.ts @@ -49,7 +49,6 @@ import fsExtra, { exists } from "fs-extra" const { mkdirp, remove } = fsExtra import { GlobalConfigStore } from "../config-store/global.js" import { isPromise } from "./objects.js" -import { styles } from "../logger/styles.js" import type { ConfigTemplateConfig } from "../config/config-template.js" import type { PluginToolSpec, ToolBuildSpec } from "../plugin/tools.js" import { fileURLToPath, parse } from "url" diff --git a/core/test/unit/src/config/template-contexts/base.ts b/core/test/unit/src/config/template-contexts/base.ts index eab122e5a5..d6c2ec74ed 100644 --- a/core/test/unit/src/config/template-contexts/base.ts +++ b/core/test/unit/src/config/template-contexts/base.ts @@ -8,11 +8,10 @@ import { expect } from "chai" import stripAnsi from "strip-ansi" +import type { ContextKey, ContextResolveParams } from "../../../../../src/config/template-contexts/base.js" import { CONTEXT_RESOLVE_KEY_AVAILABLE_LATER, CONTEXT_RESOLVE_KEY_NOT_FOUND, - ContextKey, - ContextResolveParams, } from "../../../../../src/config/template-contexts/base.js" import { ConfigContext, schema, ScanContext } from "../../../../../src/config/template-contexts/base.js" import { expectError } from "../../../../helpers.js" From 2cf06d32283ef2dc06a07400c62663d02aecee87 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Wed, 4 Dec 2024 12:57:38 +0100 Subject: [PATCH 23/43] chore: replace `ScanContext` with `NoOpContext` --- core/src/config/template-contexts/base.ts | 19 ------------ core/src/template-string/static-analysis.ts | 10 ++++-- .../unit/src/config/template-contexts/base.ts | 31 +------------------ 3 files changed, 9 insertions(+), 51 deletions(-) diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index a931c2a608..8969401af7 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -12,7 +12,6 @@ import { ConfigurationError, GardenError } from "../../exceptions.js" import { resolveTemplateString } from "../../template-string/template-string.js" import type { CustomObjectSchema } from "../common.js" import { joi, joiIdentifier } from "../common.js" -import { KeyedSet } from "../../util/keyed-set.js" import { naturalList } from "../../util/string.js" import { styles } from "../../logger/styles.js" import { Profile } from "../../util/profiling.js" @@ -254,24 +253,6 @@ export class GenericContext extends ConfigContext { } } -/** - * This is a utility context, used to extract all template references from a template. - */ -export class ScanContext extends ConfigContext { - foundKeys: KeyedSet - - constructor() { - super() - this.foundKeys = new KeyedSet((v) => renderKeyPath(v)) - } - - override resolve({ key, nodePath }: ContextResolveParams) { - const fullKey = nodePath.concat(key) - this.foundKeys.add(fullKey) - return { resolved: CONTEXT_RESOLVE_KEY_AVAILABLE_LATER, partial: true } - } -} - export class EnvironmentContext extends ConfigContext { @schema( joi diff --git a/core/src/template-string/static-analysis.ts b/core/src/template-string/static-analysis.ts index 6bfddf357a..425cfb0853 100644 --- a/core/src/template-string/static-analysis.ts +++ b/core/src/template-string/static-analysis.ts @@ -12,7 +12,7 @@ import { isArray, isPlainObject } from "../util/objects.js" import { ContextLookupExpression, IdentifierExpression, MemberExpression, TemplateExpression } from "./ast.js" import type { TemplatePrimitive } from "./types.js" import { parseTemplateString } from "./template-string.js" -import { ScanContext } from "../config/template-contexts/base.js" +import { ConfigContext, CONTEXT_RESOLVE_KEY_AVAILABLE_LATER } from "../config/template-contexts/base.js" export type TemplateExpressionGenerator = Generator @@ -104,7 +104,7 @@ export function* getContextLookupReferences( } else { // can be evaluated statically const result = v.innerExpression.evaluate({ - context: new ScanContext(), + context: new NoOpContext(), rawTemplateString: "", opts: {}, }) @@ -132,3 +132,9 @@ export function* getContextLookupReferences( } } } + +class NoOpContext extends ConfigContext { + override resolve() { + return { resolved: CONTEXT_RESOLVE_KEY_AVAILABLE_LATER, partial: true } + } +} diff --git a/core/test/unit/src/config/template-contexts/base.ts b/core/test/unit/src/config/template-contexts/base.ts index d6c2ec74ed..3b6495f49e 100644 --- a/core/test/unit/src/config/template-contexts/base.ts +++ b/core/test/unit/src/config/template-contexts/base.ts @@ -13,10 +13,9 @@ import { CONTEXT_RESOLVE_KEY_AVAILABLE_LATER, CONTEXT_RESOLVE_KEY_NOT_FOUND, } from "../../../../../src/config/template-contexts/base.js" -import { ConfigContext, schema, ScanContext } from "../../../../../src/config/template-contexts/base.js" +import { ConfigContext, schema } from "../../../../../src/config/template-contexts/base.js" import { expectError } from "../../../../helpers.js" import { joi } from "../../../../../src/config/common.js" -import { resolveTemplateStrings } from "../../../../../src/template-string/template-string.js" type TestValue = string | ConfigContext | TestValues | TestValueFunction type TestValueFunction = () => TestValue | Promise @@ -314,31 +313,3 @@ describe("ConfigContext", () => { }) }) }) - -describe("ScanContext", () => { - it("should collect found keys in an object", () => { - const context = new ScanContext() - const obj = { - a: "some ${templated.string}", - b: "${more.stuff}", - } - resolveTemplateStrings({ value: obj, context, source: undefined }) - expect(context.foundKeys.entries()).to.eql([ - ["templated", "string"], - ["more", "stuff"], - ]) - }) - - it("should handle keys with dots correctly", () => { - const context = new ScanContext() - const obj = { - a: "some ${templated['key.with.dots']}", - b: "${more.stuff}", - } - resolveTemplateStrings({ value: obj, context, source: undefined }) - expect(context.foundKeys.entries()).to.eql([ - ["templated", "key.with.dots"], - ["more", "stuff"], - ]) - }) -}) From d25d7616c64d56c257a21c933e101a954e4c0f40 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Wed, 4 Dec 2024 13:15:12 +0100 Subject: [PATCH 24/43] test: consume generator in tests --- core/test/unit/src/template-string.ts | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/core/test/unit/src/template-string.ts b/core/test/unit/src/template-string.ts index 81b16d6eeb..4067b1fb2e 100644 --- a/core/test/unit/src/template-string.ts +++ b/core/test/unit/src/template-string.ts @@ -2197,7 +2197,7 @@ describe("getActionTemplateReferences", () => { run: '${actions["run"].run-a}', test: '${actions["test"].test-a}', } - const actionTemplateReferences = Array.from(getActionTemplateReferences(config)) + const actionTemplateReferences = Array.from(Array.from(getActionTemplateReferences(config))) expect(actionTemplateReferences).to.eql([ { kind: "Build", @@ -2226,7 +2226,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: "${actions}", } - void expectError(() => getActionTemplateReferences(config), { + void expectError(() => Array.from(getActionTemplateReferences(config)), { contains: "Found invalid action reference (missing kind)", }) }) @@ -2235,7 +2235,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: '${actions["badkind"].some-name}', } - void expectError(() => getActionTemplateReferences(config), { + void expectError(() => Array.from(getActionTemplateReferences(config)), { contains: "Found invalid action reference (invalid kind 'badkind')", }) }) @@ -2244,7 +2244,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: "${actions[123]}", } - void expectError(() => getActionTemplateReferences(config), { + void expectError(() => Array.from(getActionTemplateReferences(config)), { contains: "Found invalid action reference (kind is not a string)", }) }) @@ -2262,7 +2262,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: '${actions["build"]}', } - void expectError(() => getActionTemplateReferences(config), { + void expectError(() => Array.from(getActionTemplateReferences(config)), { contains: "Found invalid action reference (missing name)", }) }) @@ -2271,7 +2271,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: '${actions["build"].123}', } - void expectError(() => getActionTemplateReferences(config), { + void expectError(() => Array.from(getActionTemplateReferences(config)), { contains: "Found invalid action reference (name is not a string)", }) }) @@ -2283,7 +2283,7 @@ describe("getActionTemplateReferences", () => { services: '${runtime["services"].service-a}', tasks: '${runtime["tasks"].task-a}', } - const actionTemplateReferences = getActionTemplateReferences(config) + const actionTemplateReferences = Array.from(getActionTemplateReferences(config)) expect(actionTemplateReferences).to.eql([ { kind: "Deploy", @@ -2302,7 +2302,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: "${runtime}", } - void expectError(() => getActionTemplateReferences(config), { + void expectError(() => Array.from(getActionTemplateReferences(config)), { contains: "Found invalid runtime reference (missing kind)", }) }) @@ -2311,7 +2311,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: '${runtime["badkind"].some-name}', } - void expectError(() => getActionTemplateReferences(config), { + void expectError(() => Array.from(getActionTemplateReferences(config)), { contains: "Found invalid runtime reference (invalid kind 'badkind')", }) }) @@ -2320,7 +2320,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: "${runtime[123]}", } - void expectError(() => getActionTemplateReferences(config), { + void expectError(() => Array.from(getActionTemplateReferences(config)), { contains: "Found invalid runtime reference (kind is not a string)", }) }) @@ -2329,7 +2329,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: "${runtime[foo.bar].some-name}", } - void expectError(() => getActionTemplateReferences(config), { + void expectError(() => Array.from(getActionTemplateReferences(config)), { contains: "Found invalid runtime reference (invalid kind '${foo.bar}')", }) }) @@ -2338,7 +2338,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: '${runtime["tasks"]}', } - void expectError(() => getActionTemplateReferences(config), { + void expectError(() => Array.from(getActionTemplateReferences(config)), { contains: "Found invalid runtime reference (missing name)", }) }) @@ -2347,7 +2347,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: '${runtime["tasks"].123}', } - void expectError(() => getActionTemplateReferences(config), { + void expectError(() => Array.from(getActionTemplateReferences(config)), { contains: "Found invalid runtime reference (name is not a string)", }) }) From e122a37619947ab4a634e088d10219b52efd0bbf Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Wed, 4 Dec 2024 13:49:11 +0100 Subject: [PATCH 25/43] test: fix "throws if action kind is not resolvable" --- core/src/exceptions.ts | 15 ++-- core/src/graph/actions.ts | 2 +- core/src/template-string/ast.ts | 30 ++----- core/src/template-string/parser.pegjs | 22 +++-- core/src/template-string/static-analysis.ts | 95 +++++++++++---------- core/src/template-string/template-string.ts | 26 ++++-- core/test/unit/src/template-string.ts | 32 +++---- 7 files changed, 107 insertions(+), 115 deletions(-) diff --git a/core/src/exceptions.ts b/core/src/exceptions.ts index f8269682f4..00ad86d960 100644 --- a/core/src/exceptions.ts +++ b/core/src/exceptions.ts @@ -309,23 +309,22 @@ export class CloudApiError extends GardenError { export class TemplateStringError extends GardenError { type = "template-string" - loc?: Location - rawTemplateString: string + loc: Location originalMessage: string - constructor(params: GardenErrorParams & { rawTemplateString: string; loc?: Location }) { + constructor(params: GardenErrorParams & { loc: Location }) { // TODO: figure out how to get correct path - const path = params.loc?.source?.basePath - - const pathDescription = path ? ` at path ${styles.accent(path.join("."))}` : "" + // const path = params.loc?.source?.basePath + // const path: string[] | undefined = undefined + // const pathDescription = path ? ` at path ${styles.accent(path.join("."))}` : "" + const pathDescription = "" const prefix = `Invalid template string (${styles.accent( - truncate(params.rawTemplateString, { length: 200 }).replace(/\n/g, "\\n") + truncate(params.loc.source, { length: 200 }).replace(/\n/g, "\\n") )})${pathDescription}: ` const message = params.message.startsWith(prefix) ? params.message : prefix + params.message super({ ...params, message }) this.loc = params.loc - this.rawTemplateString = params.rawTemplateString this.originalMessage = params.message } } diff --git a/core/src/graph/actions.ts b/core/src/graph/actions.ts index 0d4a7a9ba3..3a96e21646 100644 --- a/core/src/graph/actions.ts +++ b/core/src/graph/actions.ts @@ -1035,7 +1035,7 @@ function dependenciesFromActionConfig({ // -> We avoid depending on action execution when referencing static output keys const staticOutputKeys = definition?.staticOutputsSchema ? describeSchema(definition.staticOutputsSchema).keys : [] - for (const ref of getActionTemplateReferences(config)) { + for (const ref of getActionTemplateReferences(config, templateContext)) { let needsExecuted = false const outputKey = ref.fullRef[4] as string diff --git a/core/src/template-string/ast.ts b/core/src/template-string/ast.ts index 10cefb9ba8..5a3ab9324b 100644 --- a/core/src/template-string/ast.ts +++ b/core/src/template-string/ast.ts @@ -26,7 +26,6 @@ import type { TemplateExpressionGenerator } from "./static-analysis.js" type EvaluateArgs = { context: ConfigContext opts: ContextResolveOpts - rawTemplateString: string /** * Whether or not to throw an error if ContextLookupExpression fails to resolve variable. @@ -49,7 +48,7 @@ export type Location = { line: number column: number } - source?: ConfigSource + source: string } export type TemplateEvaluationResult = @@ -64,13 +63,13 @@ function* astVisitAll(e: TemplateExpression): TemplateExpressionGenerator { } const propertyValue = e[key] if (propertyValue instanceof TemplateExpression) { - yield propertyValue yield* astVisitAll(propertyValue) + yield propertyValue } else if (Array.isArray(propertyValue)) { for (const item of propertyValue) { if (item instanceof TemplateExpression) { - yield item yield* astVisitAll(item) + yield item } } } @@ -349,7 +348,6 @@ export class AddExpression extends BinaryExpression { } else { throw new TemplateStringError({ message: `Both terms need to be either arrays or strings or numbers for + operator (got ${typeof left} and ${typeof right}).`, - rawTemplateString: args.rawTemplateString, loc: this.loc, }) } @@ -365,7 +363,6 @@ export class ContainsExpression extends BinaryExpression { if (!isTemplatePrimitive(element)) { throw new TemplateStringError({ message: `The right-hand side of a 'contains' operator must be a string, number, boolean or null (got ${typeof element}).`, - rawTemplateString: args.rawTemplateString, loc: this.loc, }) } @@ -384,7 +381,6 @@ export class ContainsExpression extends BinaryExpression { throw new TemplateStringError({ message: `The left-hand side of a 'contains' operator must be a string, array or object (got ${collection}).`, - rawTemplateString: args.rawTemplateString, loc: this.loc, }) } @@ -402,7 +398,6 @@ export abstract class BinaryExpressionOnNumbers extends BinaryExpression { message: `Both terms need to be numbers for ${ this.operator } operator (got ${typeof left} and ${typeof right}).`, - rawTemplateString: args.rawTemplateString, loc: this.loc, }) } @@ -589,7 +584,6 @@ export class MemberExpression extends TemplateExpression { if (typeof inner !== "string" && typeof inner !== "number") { throw new TemplateStringError({ message: `Expression in brackets must resolve to a string or number (got ${typeof inner}).`, - rawTemplateString: args.rawTemplateString, loc: this.loc, }) } @@ -610,21 +604,20 @@ export class ContextLookupExpression extends TemplateExpression { context, opts, optional, - rawTemplateString, }: EvaluateArgs): | CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { const keyPath: (string | number)[] = [] for (const k of this.keyPath) { - const evaluated = k.evaluate({ context, opts, optional, rawTemplateString }) + const evaluated = k.evaluate({ context, opts, optional }) if (typeof evaluated === "symbol") { return evaluated } keyPath.push(evaluated) } - const { resolved, getUnavailableReason } = this.resolveContext(context, keyPath, opts, rawTemplateString) + const { resolved, getUnavailableReason } = this.resolveContext(context, keyPath, opts) // if context returns key available later, then we do not need to throw, because partial mode is enabled. if (resolved === CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) { @@ -640,7 +633,6 @@ export class ContextLookupExpression extends TemplateExpression { throw new TemplateStringError({ message: getUnavailableReason?.() || `Could not find key ${renderKeyPath(keyPath)}`, - rawTemplateString, loc: this.loc, }) } @@ -652,7 +644,6 @@ export class ContextLookupExpression extends TemplateExpression { context: ConfigContext, keyPath: (string | number)[], opts: ContextResolveOpts, - rawTemplateString: string ) { try { return context.resolve({ @@ -665,10 +656,10 @@ export class ContextLookupExpression extends TemplateExpression { }) } catch (e) { if (e instanceof ContextResolveError) { - throw new TemplateStringError({ message: e.message, rawTemplateString, loc: this.loc }) + throw new TemplateStringError({ message: e.message, loc: this.loc }) } if (e instanceof TemplateStringError) { - throw new TemplateStringError({ message: e.originalMessage, rawTemplateString, loc: this.loc }) + throw new TemplateStringError({ message: e.originalMessage, loc: this.loc }) } throw e } @@ -704,7 +695,6 @@ export class FunctionCallExpression extends TemplateExpression { const result: CollectionOrValue = this.callHelperFunction({ functionName, args: functionArgs, - text: args.rawTemplateString, context: args.context, opts: args.opts, }) @@ -715,11 +705,9 @@ export class FunctionCallExpression extends TemplateExpression { callHelperFunction({ functionName, args, - text, }: { functionName: string args: CollectionOrValue[] - text: string context: ConfigContext opts: ContextResolveOpts }): CollectionOrValue { @@ -730,7 +718,6 @@ export class FunctionCallExpression extends TemplateExpression { const availableFns = Object.keys(helperFunctions).join(", ") throw new TemplateStringError({ message: `Could not find helper function '${functionName}'. Available helper functions: ${availableFns}`, - rawTemplateString: text, loc: this.loc, }) } @@ -744,7 +731,6 @@ export class FunctionCallExpression extends TemplateExpression { if (value === undefined && schemaDescription.flags?.presence === "required") { throw new TemplateStringError({ message: `Missing argument '${argName}' (at index ${i}) for ${functionName} helper function.`, - rawTemplateString: text, loc: this.loc, }) } @@ -755,7 +741,6 @@ export class FunctionCallExpression extends TemplateExpression { constructor({ message }: { message: string }) { super({ message, - rawTemplateString: text, loc, }) } @@ -773,7 +758,6 @@ export class FunctionCallExpression extends TemplateExpression { } catch (error) { throw new TemplateStringError({ message: `Error from helper function ${functionName}: ${error}`, - rawTemplateString: text, loc: this.loc, }) } diff --git a/core/src/template-string/parser.pegjs b/core/src/template-string/parser.pegjs index c5f9d61cf7..3b578bfc1b 100644 --- a/core/src/template-string/parser.pegjs +++ b/core/src/template-string/parser.pegjs @@ -7,8 +7,6 @@ */ { - const rawTemplateString = input - const { ast, escapePrefix, @@ -67,7 +65,7 @@ case "%": return new ast.ModuloExpression(location(), operator, left, right) default: - throw new TemplateStringError({ message: `Unrecognized logical operator: ${operator}`, rawTemplateString, loc: location() }) + throw new TemplateStringError({ message: `Unrecognized logical operator: ${operator}`, loc: location() }) } }, head); } @@ -88,7 +86,7 @@ case "||": return new ast.LogicalOrExpression(location(), operator, left, right) default: - throw new TemplateStringError({ message: `Unrecognized logical operator: ${operator}`, rawTemplateString, loc: location() }) + throw new TemplateStringError({ message: `Unrecognized logical operator: ${operator}`, loc: location() }) } }, head); } @@ -127,10 +125,10 @@ } } else if (e instanceof ast.ElseBlockExpression) { if (currentCondition === undefined) { - throw new TemplateStringError({ message: "Found ${else} block without a preceding ${if...} block.", rawTemplateString, loc: location() }) + throw new TemplateStringError({ message: "Found ${else} block without a preceding ${if...} block.", loc: location() }) } if (encounteredElse && nestingLevel === 0) { - throw new TemplateStringError({ message: "Encountered multiple ${else} blocks on the same ${if...} block nesting level.", rawTemplateString, loc: location() }) + throw new TemplateStringError({ message: "Encountered multiple ${else} blocks on the same ${if...} block nesting level.", loc: location() }) } if (currentCondition && nestingLevel === 0) { @@ -140,7 +138,7 @@ } } else if (e instanceof ast.EndIfBlockExpression) { if (currentCondition === undefined) { - throw new TemplateStringError({ message: "Found ${endif} block without a preceding ${if...} block.", rawTemplateString, loc: location() }) + throw new TemplateStringError({ message: "Found ${endif} block without a preceding ${if...} block.", loc: location() }) } if (nestingLevel === 0) { currentCondition.ifTrue = buildConditionalTree(...ifTrue) @@ -158,7 +156,7 @@ } if (currentCondition) { - throw new TemplateStringError({ message: "Missing ${endif} after ${if ...} block.", rawTemplateString, loc: location() }) + throw new TemplateStringError({ message: "Missing ${endif} after ${if ...} block.", loc: location() }) } if (rootExpressions.length === 0) { @@ -221,7 +219,7 @@ FormatString case "endif": return new ast.EndIfBlockExpression(location()) default: - throw new TemplateStringError({ message: `Unrecognized block operator: ${op}`, rawTemplateString, loc: location() }) + throw new TemplateStringError({ message: `Unrecognized block operator: ${op}`, loc: location() }) } } / pre:FormatStartWithEscape blockOperator:(ExpressionBlockOperator __)* e:Expression end:FormatEndWithOptional { @@ -238,7 +236,7 @@ FormatString if (blockOperator && blockOperator.length > 0) { if (isOptional) { - throw new TemplateStringError({ message: "Cannot specify optional suffix in if-block.", rawTemplateString, loc: location() }) + throw new TemplateStringError({ message: "Cannot specify optional suffix in if-block.", loc: location() }) } // ifTrue and ifFalse will be filled in by `buildConditionalTree` @@ -252,7 +250,7 @@ FormatString UnclosedFormatString = Prefix? FormatStart .* { - throw new TemplateStringError({ message: "Unable to parse as valid template string.", rawTemplateString, loc: location() }) + throw new TemplateStringError({ message: "Unable to parse as valid template string.", loc: location() }) } EscapeStart @@ -389,7 +387,7 @@ UnaryExpression case "!": return new ast.NotExpression(location(), argument) default: - throw new TemplateStringError({ message: `Unrecognized unary operator: ${operator}`, rawTemplateString, loc: location() }) + throw new TemplateStringError({ message: `Unrecognized unary operator: ${operator}`, loc: location() }) } } diff --git a/core/src/template-string/static-analysis.ts b/core/src/template-string/static-analysis.ts index 425cfb0853..ead777e6d3 100644 --- a/core/src/template-string/static-analysis.ts +++ b/core/src/template-string/static-analysis.ts @@ -6,13 +6,16 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import { isNumber, isString } from "lodash-es" import type { CollectionOrValue } from "../util/objects.js" import { isArray, isPlainObject } from "../util/objects.js" -import { ContextLookupExpression, IdentifierExpression, MemberExpression, TemplateExpression } from "./ast.js" +import { + ContextLookupExpression, + TemplateExpression, +} from "./ast.js" import type { TemplatePrimitive } from "./types.js" import { parseTemplateString } from "./template-string.js" import { ConfigContext, CONTEXT_RESOLVE_KEY_AVAILABLE_LATER } from "../config/template-contexts/base.js" +import { GardenError, InternalError } from "../exceptions.js" export type TemplateExpressionGenerator = Generator @@ -62,7 +65,7 @@ export function containsTemplateExpression(generator: TemplateExpressionGenerato } export function containsContextLookupReferences(generator: TemplateExpressionGenerator): boolean { - for (const finding of getContextLookupReferences(generator)) { + for (const finding of getContextLookupReferences(generator, new NoOpContext())) { return true } @@ -71,63 +74,61 @@ export function containsContextLookupReferences(generator: TemplateExpressionGen export type ContextLookupReferenceFinding = | { - type: "static" + type: "resolvable" keyPath: (string | number)[] } | { - type: "dynamic" - keyPath: (string | number | TemplateExpression)[] + type: "unresolvable" + keyPath: (string | number | { getError: () => GardenError })[] } - | { - type: "invalid" - keyPath: unknown[] + +function captureError(arg: () => void): () => GardenError { + return () => { + try { + arg() + } catch (e) { + if (e instanceof GardenError) { + return e + } + throw e } + throw new InternalError({ + message: `captureError: function did not throw: ${arg}`, + }) + } +} export function* getContextLookupReferences( - generator: TemplateExpressionGenerator + generator: TemplateExpressionGenerator, + context: ConfigContext ): Generator { for (const expression of generator) { if (expression instanceof ContextLookupExpression) { - let type: ContextLookupReferenceFinding["type"] | undefined = undefined - const keyPath: any[] = [] - - for (const v of expression.keyPath.values()) { - if (v instanceof IdentifierExpression) { - keyPath.push(v.name) - } else if (v instanceof MemberExpression) { - if (containsContextLookupReferences(v.innerExpression.visitAll())) { - // do not override invalid - if (type !== "invalid") { - type = "dynamic" - } - keyPath.push(v.innerExpression) - } else { - // can be evaluated statically - const result = v.innerExpression.evaluate({ - context: new NoOpContext(), - rawTemplateString: "", - opts: {}, - }) - - if (isString(result) || isNumber(result)) { - keyPath.push(result) - type ||= "static" - } else { - keyPath.push(result) - // if it's invalid, we override to invalid - type = "invalid" - } + let isResolvable: boolean = true + const keyPath = expression.keyPath.map((keyPathExpression) => { + const key = keyPathExpression.evaluate({ context, opts: { allowPartial: true } }) + if (typeof key === "symbol") { + isResolvable = false + return { + getError: captureError(() => + // this will throw an error, because the key could not be resolved + keyPathExpression.evaluate({ context, opts: { allowPartial: false } }) + ), } - } else { - v satisfies never } - } + return key + }) - if (type && keyPath.length > 0) { - yield { - keyPath, - type, - } + if (keyPath.length > 0) { + yield isResolvable + ? { + type: "resolvable", + keyPath: keyPath as (string | number)[], + } + : { + type: "unresolvable", + keyPath, + } } } } diff --git a/core/src/template-string/template-string.ts b/core/src/template-string/template-string.ts index fd936c3e46..3af33add4b 100644 --- a/core/src/template-string/template-string.ts +++ b/core/src/template-string/template-string.ts @@ -121,6 +121,7 @@ export function parseTemplateString({ parseNested: (nested: string) => parseTemplateString({ rawTemplateString: nested, source, unescape }), TemplateStringError, unescape, + grammarSource: rawTemplateString, }, ]) @@ -164,7 +165,6 @@ export function resolveTemplateString({ } const result = parsed.evaluate({ - rawTemplateString: string, context, opts: contextOpts, }) @@ -585,12 +585,14 @@ interface ActionTemplateReference extends ActionReference { /** * Collects every reference to another action in the given config object, including translated runtime.* references. * An error is thrown if a reference is not resolvable, i.e. if a nested template is used as a reference. - * - * TODO-0.13.1: Allow such nested references in certain cases, e.g. if resolvable with a ProjectConfigContext. */ -export function* getActionTemplateReferences(config: object): Generator { +export function* getActionTemplateReferences( + config: object, + context: ConfigContext +): Generator { const generator = getContextLookupReferences( - visitAll({ value: config as CollectionOrValue, parseTemplateStrings: true }) + visitAll({ value: config as CollectionOrValue, parseTemplateStrings: true }), + context ) for (const finding of generator) { @@ -599,9 +601,17 @@ export function* getActionTemplateReferences(config: object): Generator { run: '${actions["run"].run-a}', test: '${actions["test"].test-a}', } - const actionTemplateReferences = Array.from(Array.from(getActionTemplateReferences(config))) + const actionTemplateReferences = Array.from(getActionTemplateReferences(config, new TestContext({}))) expect(actionTemplateReferences).to.eql([ { kind: "Build", @@ -2226,7 +2226,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: "${actions}", } - void expectError(() => Array.from(getActionTemplateReferences(config)), { + void expectError(() => Array.from(getActionTemplateReferences(config, new TestContext({}))), { contains: "Found invalid action reference (missing kind)", }) }) @@ -2235,7 +2235,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: '${actions["badkind"].some-name}', } - void expectError(() => Array.from(getActionTemplateReferences(config)), { + void expectError(() => Array.from(getActionTemplateReferences(config, new TestContext({}))), { contains: "Found invalid action reference (invalid kind 'badkind')", }) }) @@ -2244,7 +2244,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: "${actions[123]}", } - void expectError(() => Array.from(getActionTemplateReferences(config)), { + void expectError(() => Array.from(getActionTemplateReferences(config, new TestContext({}))), { contains: "Found invalid action reference (kind is not a string)", }) }) @@ -2253,8 +2253,8 @@ describe("getActionTemplateReferences", () => { const config = { foo: "${actions[foo.bar].some-name}", } - void expectError(() => Array.from(getActionTemplateReferences(config)), { - contains: "Found invalid action reference (invalid kind '${foo.bar}')", + void expectError(() => Array.from(getActionTemplateReferences(config, new TestContext({}))), { + contains: "found invalid action reference: invalid template string (${actions[foo.bar].some-name}): could not find key foo. available keys: (none)", }) }) @@ -2262,7 +2262,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: '${actions["build"]}', } - void expectError(() => Array.from(getActionTemplateReferences(config)), { + void expectError(() => Array.from(getActionTemplateReferences(config, new TestContext({}))), { contains: "Found invalid action reference (missing name)", }) }) @@ -2271,7 +2271,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: '${actions["build"].123}', } - void expectError(() => Array.from(getActionTemplateReferences(config)), { + void expectError(() => Array.from(getActionTemplateReferences(config, new TestContext({}))), { contains: "Found invalid action reference (name is not a string)", }) }) @@ -2283,7 +2283,7 @@ describe("getActionTemplateReferences", () => { services: '${runtime["services"].service-a}', tasks: '${runtime["tasks"].task-a}', } - const actionTemplateReferences = Array.from(getActionTemplateReferences(config)) + const actionTemplateReferences = Array.from(getActionTemplateReferences(config, new TestContext({}))) expect(actionTemplateReferences).to.eql([ { kind: "Deploy", @@ -2302,7 +2302,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: "${runtime}", } - void expectError(() => Array.from(getActionTemplateReferences(config)), { + void expectError(() => Array.from(getActionTemplateReferences(config, new TestContext({}))), { contains: "Found invalid runtime reference (missing kind)", }) }) @@ -2311,7 +2311,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: '${runtime["badkind"].some-name}', } - void expectError(() => Array.from(getActionTemplateReferences(config)), { + void expectError(() => Array.from(getActionTemplateReferences(config, new TestContext({}))), { contains: "Found invalid runtime reference (invalid kind 'badkind')", }) }) @@ -2320,7 +2320,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: "${runtime[123]}", } - void expectError(() => Array.from(getActionTemplateReferences(config)), { + void expectError(() => Array.from(getActionTemplateReferences(config, new TestContext({}))), { contains: "Found invalid runtime reference (kind is not a string)", }) }) @@ -2329,7 +2329,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: "${runtime[foo.bar].some-name}", } - void expectError(() => Array.from(getActionTemplateReferences(config)), { + void expectError(() => Array.from(getActionTemplateReferences(config, new TestContext({}))), { contains: "Found invalid runtime reference (invalid kind '${foo.bar}')", }) }) @@ -2338,7 +2338,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: '${runtime["tasks"]}', } - void expectError(() => Array.from(getActionTemplateReferences(config)), { + void expectError(() => Array.from(getActionTemplateReferences(config, new TestContext({}))), { contains: "Found invalid runtime reference (missing name)", }) }) @@ -2347,8 +2347,8 @@ describe("getActionTemplateReferences", () => { const config = { foo: '${runtime["tasks"].123}', } - void expectError(() => Array.from(getActionTemplateReferences(config)), { - contains: "Found invalid runtime reference (name is not a string)", + void expectError(() => Array.from(getActionTemplateReferences(config, new TestContext({}))), { + contains: "Found invalid runtime reference (name is not a string)", }) }) }) From 29152faa05a8440807d8e91a242fc57e83a7dba8 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Wed, 4 Dec 2024 14:55:35 +0100 Subject: [PATCH 26/43] feat: allow referencing variables and evaluating complex expressions in action references --- core/src/graph/actions.ts | 15 -- core/src/template-string/ast.ts | 13 +- core/src/template-string/parser.pegjs | 2 +- core/src/template-string/static-analysis.ts | 4 +- core/src/template-string/template-string.ts | 213 +++++++++++--------- core/test/unit/src/template-string.ts | 22 +- 6 files changed, 143 insertions(+), 126 deletions(-) diff --git a/core/src/graph/actions.ts b/core/src/graph/actions.ts index 3a96e21646..e6976f03f0 100644 --- a/core/src/graph/actions.ts +++ b/core/src/graph/actions.ts @@ -1040,21 +1040,6 @@ function dependenciesFromActionConfig({ const outputKey = ref.fullRef[4] as string - if (maybeTemplateString(ref.name)) { - try { - // TODO: validate that we actually resolve to a string - ref.name = resolveTemplateString({ - string: ref.name, - context: templateContext, - contextOpts: { allowPartial: false }, - }) as string - } catch (err) { - log.warn( - `Unable to infer dependency from action reference in ${description}, because template string '${ref.name}' could not be resolved. Either fix the dependency or specify it explicitly.` - ) - continue - } - } // also avoid execution when referencing the static output keys of the ref action type. // e.g. a helm deploy referencing container build static output deploymentImageName // ${actions.build.my-container.outputs.deploymentImageName} diff --git a/core/src/template-string/ast.ts b/core/src/template-string/ast.ts index 5a3ab9324b..d2545f8c9a 100644 --- a/core/src/template-string/ast.ts +++ b/core/src/template-string/ast.ts @@ -94,18 +94,19 @@ export abstract class TemplateExpression { export class IdentifierExpression extends TemplateExpression { constructor( loc: Location, - public readonly name: string + // in the template expression ${foo.123}, 123 is a valid identifier expression and is treated as a number. + public readonly identifier: string | number ) { - if (!isString(name)) { + if (!isString(identifier) && !isNumber(identifier)) { throw new InternalError({ - message: `IdentifierExpression name must be a string. Got: ${typeof name}`, + message: `identifier arg for IdentifierExpression must be a string or number. Got: ${typeof identifier}`, }) } super(loc) } - override evaluate(): string { - return this.name + override evaluate(): string | number { + return this.identifier } } @@ -693,7 +694,7 @@ export class FunctionCallExpression extends TemplateExpression { const functionName = this.functionName.evaluate() const result: CollectionOrValue = this.callHelperFunction({ - functionName, + functionName: functionName.toString(), args: functionArgs, context: args.context, opts: args.opts, diff --git a/core/src/template-string/parser.pegjs b/core/src/template-string/parser.pegjs index 3b578bfc1b..cdf4a42786 100644 --- a/core/src/template-string/parser.pegjs +++ b/core/src/template-string/parser.pegjs @@ -517,7 +517,7 @@ IdentifierName "identifier" return head + tail.join("") } / Integer { - return text(); + return parseInt(text()); } IdentifierStart diff --git a/core/src/template-string/static-analysis.ts b/core/src/template-string/static-analysis.ts index ead777e6d3..b0fb9544b8 100644 --- a/core/src/template-string/static-analysis.ts +++ b/core/src/template-string/static-analysis.ts @@ -106,13 +106,13 @@ export function* getContextLookupReferences( if (expression instanceof ContextLookupExpression) { let isResolvable: boolean = true const keyPath = expression.keyPath.map((keyPathExpression) => { - const key = keyPathExpression.evaluate({ context, opts: { allowPartial: true } }) + const key = keyPathExpression.evaluate({ context, opts: { allowPartial: false }, optional: true }) if (typeof key === "symbol") { isResolvable = false return { getError: captureError(() => // this will throw an error, because the key could not be resolved - keyPathExpression.evaluate({ context, opts: { allowPartial: false } }) + keyPathExpression.evaluate({ context, opts: { allowPartial: false }, optional: false }) ), } } diff --git a/core/src/template-string/template-string.ts b/core/src/template-string/template-string.ts index 3af33add4b..04a87e1133 100644 --- a/core/src/template-string/template-string.ts +++ b/core/src/template-string/template-string.ts @@ -46,7 +46,7 @@ import type { ObjectPath } from "../config/base.js" import type { TemplatePrimitive } from "./types.js" import * as ast from "./ast.js" import { LRUCache } from "lru-cache" -import { getContextLookupReferences, visitAll } from "./static-analysis.js" +import { ContextLookupReferenceFinding, getContextLookupReferences, visitAll } from "./static-analysis.js" const escapePrefix = "$${" @@ -582,6 +582,112 @@ interface ActionTemplateReference extends ActionReference { fullRef: ContextKeySegment[] } +function extractActionReference(finding: ContextLookupReferenceFinding): ActionTemplateReference { + if (finding.type === "unresolvable") { + for (const k of finding.keyPath) { + if (typeof k === "object" && "getError" in k) { + const err = k.getError() + throw new ConfigurationError({ + message: `Found invalid action reference: ${err.message}`, + }) + } + } + throw new InternalError({ + message: "No error found in unresolvable finding", + }) + } + + if (!finding.keyPath[1]) { + throw new ConfigurationError({ + message: `Found invalid action reference (missing kind).`, + }) + } + + if (!isString(finding.keyPath[1])) { + throw new ConfigurationError({ + message: `Found invalid action reference (kind is not a string).`, + }) + } + + if (!actionKindsLower.includes(finding.keyPath[1])) { + throw new ConfigurationError({ + message: `Found invalid action reference (invalid kind '${finding.keyPath[1]}')`, + }) + } + + if (!finding.keyPath[2]) { + throw new ConfigurationError({ + message: "Found invalid action reference (missing name)", + }) + } + if (!isString(finding.keyPath[2])) { + throw new ConfigurationError({ + message: "Found invalid action reference (name is not a string)", + }) + } + + return { + kind: titleize(finding.keyPath[1]), + name: finding.keyPath[2], + fullRef: finding.keyPath, + } +} + +function extractRuntimeReference(finding: ContextLookupReferenceFinding): ActionTemplateReference { + if (finding.type === "unresolvable") { + for (const k of finding.keyPath) { + if (typeof k === "object" && "getError" in k) { + const err = k.getError() + throw new ConfigurationError({ + message: `Found invalid action reference: ${err.message}`, + }) + } + } + throw new InternalError({ + message: "No error found in unresolvable finding", + }) + } + + if (!finding.keyPath[1]) { + throw new ConfigurationError({ + message: "Found invalid runtime reference (missing kind)", + }) + } + if (!isString(finding.keyPath[1])) { + throw new ConfigurationError({ + message: "Found invalid runtime reference (kind is not a string)", + }) + } + + let kind: ActionKind + if (finding.keyPath[1] === "services") { + kind = "Deploy" + } else if (finding.keyPath[1] === "tasks") { + kind = "Run" + } else { + throw new ConfigurationError({ + message: `Found invalid runtime reference (invalid kind '${finding.keyPath[1]}')`, + }) + } + + if (!finding.keyPath[2]) { + throw new ConfigurationError({ + message: `Found invalid runtime reference (missing name)`, + }) + } + if (!isString(finding.keyPath[2])) { + throw new ConfigurationError({ + message: "Found invalid runtime reference (name is not a string)", + }) + } + + return { + kind, + name: finding.keyPath[2], + fullRef: finding.keyPath, + } +} + /** * Collects every reference to another action in the given config object, including translated runtime.* references. * An error is thrown if a reference is not resolvable, i.e. if a nested template is used as a reference. @@ -596,108 +702,15 @@ export function* getActionTemplateReferences( ) for (const finding of generator) { - // we are interested in ${action.*} - if (finding.keyPath[0] !== "actions") { - continue - } - - if (finding.type === "unresolvable") { - for (const k of finding.keyPath) { - if (typeof k === "object" && "getError" in k) { - const err = k.getError() - throw new ConfigurationError({ - message: `Found invalid action reference: ${err.message}` - }) - } - } - throw new InternalError({ - message: "No error found in unresolvable finding", - }) - } - - if (!finding.keyPath[1]) { - throw new ConfigurationError({ - message: `Found invalid action reference (missing kind).`, - }) + // ${action.*} + if (finding.keyPath[0] === "actions") { + yield extractActionReference(finding) } - - if (!isString(finding.keyPath[1])) { - throw new ConfigurationError({ - message: `Found invalid action reference (kind is not a string).`, - }) - } - if (!actionKindsLower.includes(finding.keyPath[1])) { - throw new ConfigurationError({ - message: `Found invalid action reference (invalid kind '${finding.keyPath[1]}')`, - }) - } - - if (!finding.keyPath[2]) { - throw new ConfigurationError({ - message: "Found invalid action reference (missing name)", - }) - } - if (!isString(finding.keyPath[2])) { - throw new ConfigurationError({ - message: "Found invalid action reference (name is not a string)", - }) - } - - yield { - kind: titleize(finding.keyPath[1]), - name: finding.keyPath[2], - fullRef: finding.keyPath, + // ${runtime.*} + if (finding.keyPath[0] === "runtime") { + yield extractRuntimeReference(finding) } } - - // // ${runtime.*} - // for (const ref of rawRefs) { - // if (ref[0] !== "runtime") { - // continue - // } - - // let kind: ActionKind - - // if (!ref[1]) { - // throw new ConfigurationError({ - // message: "Found invalid runtime reference (missing kind)", - // }) - // } - // if (!isString(ref[1])) { - // throw new ConfigurationError({ - // message: "Found invalid runtime reference (kind is not a string)", - // }) - // } - - // if (ref[1] === "services") { - // kind = "Deploy" - // } else if (ref[1] === "tasks") { - // kind = "Run" - // } else { - // throw new ConfigurationError({ - // message: `Found invalid runtime reference (invalid kind '${ref[1]}')`, - // }) - // } - - // if (!ref[2]) { - // throw new ConfigurationError({ - // message: `Found invalid runtime reference (missing name)`, - // }) - // } - // if (!isString(ref[2])) { - // throw new ConfigurationError({ - // message: "Found invalid runtime reference (name is not a string)", - // }) - // } - - // refs.push({ - // kind, - // name: ref[2], - // fullRef: ref, - // }) - // } - - // return refs } export function getModuleTemplateReferences(obj: object, context: ModuleConfigContext) { diff --git a/core/test/unit/src/template-string.ts b/core/test/unit/src/template-string.ts index 76627d71b3..bf09052219 100644 --- a/core/test/unit/src/template-string.ts +++ b/core/test/unit/src/template-string.ts @@ -2258,6 +2258,15 @@ describe("getActionTemplateReferences", () => { }) }) + it("throws if dynamic action kind is invalid", () => { + const config = { + foo: "${actions[foo.bar || \"hello\"].some-name}", + } + void expectError(() => Array.from(getActionTemplateReferences(config, new TestContext({}))), { + contains: "found invalid action reference (invalid kind 'hello')", + }) + }) + it("throws if action ref has no name", () => { const config = { foo: '${actions["build"]}', @@ -2267,7 +2276,7 @@ describe("getActionTemplateReferences", () => { }) }) - it("throws if action name is not a string", () => { + it("throws if action name is not a string in identifier expression", () => { const config = { foo: '${actions["build"].123}', } @@ -2275,6 +2284,15 @@ describe("getActionTemplateReferences", () => { contains: "Found invalid action reference (name is not a string)", }) }) + + it("throws if action name is not a string in member expression", () => { + const config = { + foo: '${actions["build"][123]}', + } + void expectError(() => Array.from(getActionTemplateReferences(config, new TestContext({}))), { + contains: "Found invalid action reference (name is not a string)", + }) + }) }) context("runtime.*", () => { @@ -2330,7 +2348,7 @@ describe("getActionTemplateReferences", () => { foo: "${runtime[foo.bar].some-name}", } void expectError(() => Array.from(getActionTemplateReferences(config, new TestContext({}))), { - contains: "Found invalid runtime reference (invalid kind '${foo.bar}')", + contains: "found invalid action reference: invalid template string (${runtime[foo.bar].some-name}): could not find key foo. available keys: (none).", }) }) From fc231be0428097ace1a53d9a3ff069f760261207 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Wed, 4 Dec 2024 16:39:40 +0100 Subject: [PATCH 27/43] refactor: get rid of function `collectTemplateReferences` Co-authored-by: Steffen Neubauer --- core/src/config/provider.ts | 49 +++++++---- core/src/config/template-contexts/base.ts | 6 ++ core/src/graph/actions.ts | 4 - core/src/outputs.ts | 66 ++++++++------- core/src/template-string/ast.ts | 23 ++--- core/src/template-string/static-analysis.ts | 16 +--- core/src/template-string/template-string.ts | 93 +++++++++++++-------- core/test/unit/src/template-string.ts | 52 ++++++++++-- 8 files changed, 185 insertions(+), 124 deletions(-) diff --git a/core/src/config/provider.ts b/core/src/config/provider.ts index 42115f5908..4933478399 100644 --- a/core/src/config/provider.ts +++ b/core/src/config/provider.ts @@ -8,19 +8,18 @@ import { deline } from "../util/string.js" import { - joiIdentifier, - joiUserIdentifier, - joiArray, + createSchema, joi, + joiArray, + joiIdentifier, joiIdentifierMap, joiSparseArray, - createSchema, + joiUserIdentifier, } from "./common.js" -import { collectTemplateReferences } from "../template-string/template-string.js" import { ConfigurationError } from "../exceptions.js" import type { ModuleConfig } from "./module.js" import { moduleConfigSchema } from "./module.js" -import { memoize, uniq } from "lodash-es" +import { isNumber, isString, memoize, uniq } from "lodash-es" import type { GardenPluginSpec } from "../plugin/plugin.js" import type { EnvironmentStatus } from "../plugin/handlers/Provider/getEnvironmentStatus.js" import { environmentStatusSchema } from "./status.js" @@ -30,6 +29,8 @@ import type { ActionState } from "../actions/types.js" import type { ValidResultType } from "../tasks/base.js" import { uuidv4 } from "../util/random.js" import { s } from "./zod.js" +import { NoOpContext } from "./template-contexts/base.js" +import { getContextLookupReferences, visitAll } from "../template-string/static-analysis.js" // TODO: dedupe from the joi schema below export const baseProviderConfigSchemaZod = s.object({ @@ -176,22 +177,36 @@ export function getAllProviderDependencyNames(plugin: GardenPluginSpec, config: * Given a provider config, return implicit dependencies based on template strings. */ export function getProviderTemplateReferences(config: GenericProviderConfig) { - const references = collectTemplateReferences(config) const deps: string[] = [] - for (const key of references) { - if (key[0] === "providers") { - const providerName = key[1] as string - if (!providerName) { - throw new ConfigurationError({ - message: deline` - Invalid template key '${key.join(".")}' in configuration for provider '${config.name}'. You must + const generator = getContextLookupReferences( + visitAll({ value: config, parseTemplateStrings: true }), + new NoOpContext() + ) + for (const finding of generator) { + const keyPath = finding.keyPath + if (keyPath[0] !== "providers") { + continue + } + + const providerName = keyPath[1] + if (!providerName || isNumber(providerName)) { + throw new ConfigurationError({ + message: deline`s + Invalid template key '${keyPath.join(".")}' in configuration for provider '${config.name}'. You must specify a provider name as well (e.g. \${providers.my-provider}). `, - }) - } - deps.push(providerName) + }) } + + if (!isString(providerName)) { + const err = providerName.getError() + throw new ConfigurationError({ + message: `Found invalid provider reference: ${err.message}`, + }) + } + + deps.push(providerName) } return uniq(deps).sort() diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index 8969401af7..bcc0e929e7 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -253,6 +253,12 @@ export class GenericContext extends ConfigContext { } } +export class NoOpContext extends ConfigContext { + override resolve() { + return { resolved: CONTEXT_RESOLVE_KEY_AVAILABLE_LATER, partial: true } + } +} + export class EnvironmentContext extends ConfigContext { @schema( joi diff --git a/core/src/graph/actions.ts b/core/src/graph/actions.ts index e6976f03f0..76e331a75f 100644 --- a/core/src/graph/actions.ts +++ b/core/src/graph/actions.ts @@ -52,7 +52,6 @@ import { ResolveActionTask } from "../tasks/resolve-action.js" import { getActionTemplateReferences, maybeTemplateString, - resolveTemplateString, resolveTemplateStrings, } from "../template-string/template-string.js" import { dedent, deline, naturalList } from "../util/string.js" @@ -928,7 +927,6 @@ export const preprocessActionConfig = profileAsync(async function preprocessActi } const dependencies = dependenciesFromActionConfig({ - log, config, configsByKey, definition, @@ -946,14 +944,12 @@ export const preprocessActionConfig = profileAsync(async function preprocessActi }) function dependenciesFromActionConfig({ - log, config, configsByKey, definition, templateContext, actionTypes, }: { - log: Log config: ActionConfig configsByKey: ActionConfigsByKey definition: MaybeUndefined> diff --git a/core/src/outputs.ts b/core/src/outputs.ts index 4f0b388f89..d1e5a46fe1 100644 --- a/core/src/outputs.ts +++ b/core/src/outputs.ts @@ -7,13 +7,19 @@ */ import type { Garden } from "./garden.js" -import { collectTemplateReferences, resolveTemplateStrings } from "./template-string/template-string.js" +import { + extractActionReference, + extractRuntimeReference, + resolveTemplateStrings, +} from "./template-string/template-string.js" import { OutputConfigContext } from "./config/template-contexts/module.js" import type { Log } from "./logger/log-entry.js" import type { OutputSpec } from "./config/project.js" import type { ActionReference } from "./config/common.js" -import type { ActionKind } from "./plugin/action-types.js" import { GraphResults } from "./graph/results.js" +import { getContextLookupReferences, visitAll } from "./template-string/static-analysis.js" +import { isString } from "lodash-es" +import type { ObjectWithName } from "./util/util.js" /** * Resolves all declared project outputs. If necessary, this will resolve providers and modules, and ensure services @@ -29,29 +35,37 @@ export async function resolveProjectOutputs(garden: Garden, log: Log): Promiseref[1], name: ref[2] as string }) + const keyPath = finding.keyPath + if (!keyPath[1]) { + continue + } + + if (keyPath[0] === "providers" && isString(keyPath[1])) { + needProviders.push(keyPath[1]) + } else if (keyPath[0] === "modules" && isString(keyPath[1])) { + needModules.push(keyPath[1]) + } else if (keyPath[0] === "runtime") { + const runtimeRef = extractRuntimeReference(finding) + needActions.push(runtimeRef) + } else if (keyPath[0] === "actions") { + const actionRef = extractActionReference(finding) + needActions.push(actionRef) } } @@ -63,13 +77,7 @@ export async function resolveProjectOutputs(garden: Garden, log: Log): Promise, - right: CollectionOrValue, - args: EvaluateArgs + right: CollectionOrValue ): CollectionOrValue } @@ -335,8 +333,7 @@ export class NotEqualExpression extends BinaryExpression { export class AddExpression extends BinaryExpression { override transform( left: CollectionOrValue, - right: CollectionOrValue, - args: EvaluateArgs + right: CollectionOrValue ): CollectionOrValue { if (isNumber(left) && isNumber(right)) { return left + right @@ -358,8 +355,7 @@ export class AddExpression extends BinaryExpression { export class ContainsExpression extends BinaryExpression { override transform( collection: CollectionOrValue, - element: CollectionOrValue, - args: EvaluateArgs + element: CollectionOrValue ): boolean { if (!isTemplatePrimitive(element)) { throw new TemplateStringError({ @@ -390,8 +386,7 @@ export class ContainsExpression extends BinaryExpression { export abstract class BinaryExpressionOnNumbers extends BinaryExpression { override transform( left: CollectionOrValue, - right: CollectionOrValue, - args: EvaluateArgs + right: CollectionOrValue ): CollectionOrValue { // All other operators require numbers to make sense (we're not gonna allow random JS weirdness) if (!isNumber(left) || !isNumber(right)) { @@ -641,11 +636,7 @@ export class ContextLookupExpression extends TemplateExpression { return resolved } - private resolveContext( - context: ConfigContext, - keyPath: (string | number)[], - opts: ContextResolveOpts, - ) { + private resolveContext(context: ConfigContext, keyPath: (string | number)[], opts: ContextResolveOpts) { try { return context.resolve({ key: keyPath, diff --git a/core/src/template-string/static-analysis.ts b/core/src/template-string/static-analysis.ts index b0fb9544b8..ea1b9de5e9 100644 --- a/core/src/template-string/static-analysis.ts +++ b/core/src/template-string/static-analysis.ts @@ -8,13 +8,11 @@ import type { CollectionOrValue } from "../util/objects.js" import { isArray, isPlainObject } from "../util/objects.js" -import { - ContextLookupExpression, - TemplateExpression, -} from "./ast.js" +import { ContextLookupExpression, TemplateExpression } from "./ast.js" import type { TemplatePrimitive } from "./types.js" import { parseTemplateString } from "./template-string.js" -import { ConfigContext, CONTEXT_RESOLVE_KEY_AVAILABLE_LATER } from "../config/template-contexts/base.js" +import type { ConfigContext } from "../config/template-contexts/base.js" +import { NoOpContext } from "../config/template-contexts/base.js" import { GardenError, InternalError } from "../exceptions.js" export type TemplateExpressionGenerator = Generator @@ -65,7 +63,7 @@ export function containsTemplateExpression(generator: TemplateExpressionGenerato } export function containsContextLookupReferences(generator: TemplateExpressionGenerator): boolean { - for (const finding of getContextLookupReferences(generator, new NoOpContext())) { + for (const _ of getContextLookupReferences(generator, new NoOpContext())) { return true } @@ -133,9 +131,3 @@ export function* getContextLookupReferences( } } } - -class NoOpContext extends ConfigContext { - override resolve() { - return { resolved: CONTEXT_RESOLVE_KEY_AVAILABLE_LATER, partial: true } - } -} diff --git a/core/src/template-string/template-string.ts b/core/src/template-string/template-string.ts index 04a87e1133..c47c4e39d5 100644 --- a/core/src/template-string/template-string.ts +++ b/core/src/template-string/template-string.ts @@ -7,18 +7,16 @@ */ import type { GardenErrorParams } from "../exceptions.js" -import { - ConfigurationError, - GardenError, - InternalError, - NotImplementedError, - TemplateStringError, -} from "../exceptions.js" +import { ConfigurationError, GardenError, InternalError, TemplateStringError } from "../exceptions.js" import type { ConfigContext, ContextKeySegment, ContextResolveOpts } from "../config/template-contexts/base.js" -import { CONTEXT_RESOLVE_KEY_AVAILABLE_LATER } from "../config/template-contexts/base.js" -import { CONTEXT_RESOLVE_KEY_NOT_FOUND, GenericContext } from "../config/template-contexts/base.js" +import { + CONTEXT_RESOLVE_KEY_AVAILABLE_LATER, + CONTEXT_RESOLVE_KEY_NOT_FOUND, + GenericContext, + NoOpContext, +} from "../config/template-contexts/base.js" import cloneDeep from "fast-copy" -import { difference, isPlainObject, isString } from "lodash-es" +import { difference, isNumber, isPlainObject, isString } from "lodash-es" import type { ActionReference, Primitive, StringMap } from "../config/common.js" import { arrayConcatKey, @@ -46,7 +44,9 @@ import type { ObjectPath } from "../config/base.js" import type { TemplatePrimitive } from "./types.js" import * as ast from "./ast.js" import { LRUCache } from "lru-cache" -import { ContextLookupReferenceFinding, getContextLookupReferences, visitAll } from "./static-analysis.js" +import type { ContextLookupReferenceFinding } from "./static-analysis.js" +import { getContextLookupReferences, visitAll } from "./static-analysis.js" +import type { ModuleConfig } from "../config/module.js" const escapePrefix = "$${" @@ -566,23 +566,11 @@ export function mayContainTemplateString(obj: any): boolean { return out } -/** - * Scans for all template strings in the given object and lists the referenced keys. - */ -export function collectTemplateReferences(obj: object): ContextKeySegment[][] { - throw new NotImplementedError({ message: "TODO: Remove this function" }) -} - -export function getRuntimeTemplateReferences(obj: object) { - const refs = collectTemplateReferences(obj) - return refs.filter((ref) => ref[0] === "runtime") -} - interface ActionTemplateReference extends ActionReference { fullRef: ContextKeySegment[] } -function extractActionReference(finding: ContextLookupReferenceFinding): ActionTemplateReference { +export function extractActionReference(finding: ContextLookupReferenceFinding): ActionTemplateReference { if (finding.type === "unresolvable") { for (const k of finding.keyPath) { if (typeof k === "object" && "getError" in k) { @@ -620,6 +608,7 @@ function extractActionReference(finding: ContextLookupReferenceFinding): ActionT message: "Found invalid action reference (missing name)", }) } + if (!isString(finding.keyPath[2])) { throw new ConfigurationError({ message: "Found invalid action reference (name is not a string)", @@ -633,7 +622,7 @@ function extractActionReference(finding: ContextLookupReferenceFinding): ActionT } } -function extractRuntimeReference(finding: ContextLookupReferenceFinding): ActionTemplateReference { +export function extractRuntimeReference(finding: ContextLookupReferenceFinding): ActionTemplateReference { if (finding.type === "unresolvable") { for (const k of finding.keyPath) { if (typeof k === "object" && "getError" in k) { @@ -713,12 +702,34 @@ export function* getActionTemplateReferences( } } -export function getModuleTemplateReferences(obj: object, context: ModuleConfigContext) { - const refs = collectTemplateReferences(obj) - const moduleNames = refs.filter((ref) => ref[0] === "modules" && ref.length > 1) - // Resolve template strings in name refs. This would ideally be done ahead of this function, but is currently - // necessary to resolve templated module name references in ModuleTemplates. - return resolveTemplateStrings({ value: moduleNames, context, source: undefined }) +export function getModuleTemplateReferences(obj: ModuleConfig, context: ModuleConfigContext) { + const moduleNames: string[] = [] + const generator = getContextLookupReferences( + visitAll({ + value: obj as ObjectWithName, + parseTemplateStrings: true, + }), + context + ) + + for (const finding of generator) { + const keyPath = finding.keyPath + if (keyPath[0] !== "modules") { + continue + } + + const moduleName = keyPath[1] + if (isString(moduleName) || isNumber(moduleName)) { + moduleNames.push(moduleName.toString()) + } else { + const err = moduleName.getError() + throw new ConfigurationError({ + message: `Found invalid module reference: ${err.message}`, + }) + } + } + + return moduleNames } /** @@ -726,8 +737,6 @@ export function getModuleTemplateReferences(obj: object, context: ModuleConfigCo * blank values) in the provided secrets map. * * Prefix should be e.g. "Module" or "Provider" (used when generating error messages). - * - * TODO: We've disabled this for now. Re-introduce once we've removed get config command call from GE! */ export function throwOnMissingSecretKeys(configs: ObjectWithName[], secrets: StringMap, prefix: string, log?: Log) { const allMissing: [string, ContextKeySegment[]][] = [] // [[key, missing keys]] @@ -779,10 +788,20 @@ export function throwOnMissingSecretKeys(configs: ObjectWithName[], secrets: Str * Collects template references to secrets in obj, and returns an array of any secret keys referenced in it that * aren't present (or have blank values) in the provided secrets map. */ -export function detectMissingSecretKeys(obj: object, secrets: StringMap): ContextKeySegment[] { - const referencedKeys = collectTemplateReferences(obj) - .filter((ref) => ref[0] === "secrets") - .map((ref) => ref[1]) +export function detectMissingSecretKeys(obj: ObjectWithName, secrets: StringMap): ContextKeySegment[] { + const referencedKeys: ContextKeySegment[] = [] + const generator = getContextLookupReferences(visitAll({ value: obj, parseTemplateStrings: true }), new NoOpContext()) + for (const finding of generator) { + const keyPath = finding.keyPath + if (keyPath[0] !== "secrets") { + continue + } + + if (isString(keyPath[1])) { + referencedKeys.push(keyPath[1]) + } + } + /** * Secret keys with empty values should have resulted in an error by this point, but we filter on keys with * values for good measure. diff --git a/core/test/unit/src/template-string.ts b/core/test/unit/src/template-string.ts index bf09052219..25bcb4dcbd 100644 --- a/core/test/unit/src/template-string.ts +++ b/core/test/unit/src/template-string.ts @@ -10,17 +10,18 @@ import { expect } from "chai" import { resolveTemplateString, resolveTemplateStrings, - collectTemplateReferences, throwOnMissingSecretKeys, getActionTemplateReferences, } from "../../../src/template-string/template-string.js" -import { ConfigContext } from "../../../src/config/template-contexts/base.js" +import { ConfigContext, NoOpContext } from "../../../src/config/template-contexts/base.js" import type { TestGarden } from "../../helpers.js" import { expectError, getDataDir, makeTestGarden } from "../../helpers.js" import { dedent } from "../../../src/util/string.js" import stripAnsi from "strip-ansi" import { TemplateStringError } from "../../../src/exceptions.js" import repeat from "lodash-es/repeat.js" +import { getContextLookupReferences, visitAll } from "../../../src/template-string/static-analysis.js" +import type { ObjectWithName } from "../../../src/util/util.js" class TestContext extends ConfigContext { constructor(context) { @@ -2173,8 +2174,8 @@ describe("resolveTemplateStrings", () => { }) }) -describe("collectTemplateReferences", () => { - it("should return and sort all template string references in an object", () => { +describe("getContextLookupReferences", () => { + it("should return all template string references in an object", () => { const obj = { foo: "${my.reference}", nested: { @@ -2184,7 +2185,38 @@ describe("collectTemplateReferences", () => { }, } - expect(collectTemplateReferences(obj)).to.eql([["banana", "rama", "llama"], ["moo"], ["my", "reference"]]) + const result = Array.from( + getContextLookupReferences( + visitAll({ + value: obj, + parseTemplateStrings: true, + }), + new NoOpContext() + ) + ) + const expected = [ + { + keyPath: ["my", "reference"], + type: "resolvable", + }, + { + keyPath: ["moo"], + type: "resolvable", + }, + { + keyPath: ["moo"], + type: "resolvable", + }, + { + keyPath: ["moo"], + type: "resolvable", + }, + { + keyPath: ["banana", "rama", "llama"], + type: "resolvable", + }, + ] + expect(result).to.eql(expected) }) }) @@ -2254,13 +2286,14 @@ describe("getActionTemplateReferences", () => { foo: "${actions[foo.bar].some-name}", } void expectError(() => Array.from(getActionTemplateReferences(config, new TestContext({}))), { - contains: "found invalid action reference: invalid template string (${actions[foo.bar].some-name}): could not find key foo. available keys: (none)", + contains: + "found invalid action reference: invalid template string (${actions[foo.bar].some-name}): could not find key foo. available keys: (none)", }) }) it("throws if dynamic action kind is invalid", () => { const config = { - foo: "${actions[foo.bar || \"hello\"].some-name}", + foo: '${actions[foo.bar || "hello"].some-name}', } void expectError(() => Array.from(getActionTemplateReferences(config, new TestContext({}))), { contains: "found invalid action reference (invalid kind 'hello')", @@ -2348,7 +2381,8 @@ describe("getActionTemplateReferences", () => { foo: "${runtime[foo.bar].some-name}", } void expectError(() => Array.from(getActionTemplateReferences(config, new TestContext({}))), { - contains: "found invalid action reference: invalid template string (${runtime[foo.bar].some-name}): could not find key foo. available keys: (none).", + contains: + "found invalid action reference: invalid template string (${runtime[foo.bar].some-name}): could not find key foo. available keys: (none).", }) }) @@ -2366,7 +2400,7 @@ describe("getActionTemplateReferences", () => { foo: '${runtime["tasks"].123}', } void expectError(() => Array.from(getActionTemplateReferences(config, new TestContext({}))), { - contains: "Found invalid runtime reference (name is not a string)", + contains: "Found invalid runtime reference (name is not a string)", }) }) }) From 498dc30ce9249d9cf4d983826e4a8cb44cecde6f Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Thu, 5 Dec 2024 11:50:47 +0100 Subject: [PATCH 28/43] refactor: introduce local variables --- core/src/config/provider.ts | 1 - core/src/outputs.ts | 16 +++++++++------- core/src/template-string/template-string.ts | 16 +++++++++------- core/test/unit/src/template-string.ts | 1 - 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/core/src/config/provider.ts b/core/src/config/provider.ts index 4933478399..f3af7ea0d9 100644 --- a/core/src/config/provider.ts +++ b/core/src/config/provider.ts @@ -198,7 +198,6 @@ export function getProviderTemplateReferences(config: GenericProviderConfig) { `, }) } - if (!isString(providerName)) { const err = providerName.getError() throw new ConfigurationError({ diff --git a/core/src/outputs.ts b/core/src/outputs.ts index d1e5a46fe1..b7889232de 100644 --- a/core/src/outputs.ts +++ b/core/src/outputs.ts @@ -52,18 +52,20 @@ export async function resolveProjectOutputs(garden: Garden, log: Log): Promise Date: Thu, 5 Dec 2024 12:42:46 +0100 Subject: [PATCH 29/43] fix: module dependencies resolition --- core/src/resolve-module.ts | 3 +-- core/src/template-string/template-string.ts | 8 ++++++-- core/test/unit/src/garden.ts | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/core/src/resolve-module.ts b/core/src/resolve-module.ts index d322cf60b6..a57870d32b 100644 --- a/core/src/resolve-module.ts +++ b/core/src/resolve-module.ts @@ -529,8 +529,7 @@ export class ModuleResolver { const configContext = new ModuleConfigContext(contextParams) - const templateRefs = getModuleTemplateReferences(rawConfig, configContext) - const templateDeps = templateRefs.filter((d) => d[1] !== rawConfig.name).map((d) => d[1]) + const templateDeps = getModuleTemplateReferences(rawConfig, configContext) // This is a bit of a hack, but we need to store the template dependencies on the raw config so we can check // them later when deciding whether to resolve a module inline or not. diff --git a/core/src/template-string/template-string.ts b/core/src/template-string/template-string.ts index 29b5c13330..cc102cdc07 100644 --- a/core/src/template-string/template-string.ts +++ b/core/src/template-string/template-string.ts @@ -703,11 +703,11 @@ export function* getActionTemplateReferences( } } -export function getModuleTemplateReferences(obj: ModuleConfig, context: ModuleConfigContext) { +export function getModuleTemplateReferences(config: ModuleConfig, context: ModuleConfigContext) { const moduleNames: string[] = [] const generator = getContextLookupReferences( visitAll({ - value: obj as ObjectWithName, + value: config as ObjectWithName, parseTemplateStrings: true, }), context @@ -727,6 +727,10 @@ export function getModuleTemplateReferences(obj: ModuleConfig, context: ModuleCo }) } + // if (config.name === moduleName) { + // continue + // } + moduleNames.push(moduleName.toString()) } diff --git a/core/test/unit/src/garden.ts b/core/test/unit/src/garden.ts index 8b8003ed3e..0a0241729d 100644 --- a/core/test/unit/src/garden.ts +++ b/core/test/unit/src/garden.ts @@ -3411,7 +3411,7 @@ describe("Garden", () => { expect(module.spec.bla).to.eql({ nested: { key: "my value" } }) }) - it("should pass through runtime template strings when no runtimeContext is provider", async () => { + it("should pass through runtime template strings when no runtimeContext is provided", async () => { const test = createGardenPlugin({ name: "test", createModuleTypes: [ From 4b4b78775bba523bf7add956a878c7e95a88e936 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Thu, 5 Dec 2024 14:47:52 +0100 Subject: [PATCH 30/43] fix: restore module name filter --- core/src/template-string/template-string.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/template-string/template-string.ts b/core/src/template-string/template-string.ts index cc102cdc07..8de099db33 100644 --- a/core/src/template-string/template-string.ts +++ b/core/src/template-string/template-string.ts @@ -727,9 +727,9 @@ export function getModuleTemplateReferences(config: ModuleConfig, context: Modul }) } - // if (config.name === moduleName) { - // continue - // } + if (config.name === moduleName) { + continue + } moduleNames.push(moduleName.toString()) } From b40f5e6cd912baab5efee313c83f56d0f93e7db4 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Thu, 5 Dec 2024 14:48:01 +0100 Subject: [PATCH 31/43] test: update assertion for circular dep error --- core/test/unit/src/config/template-contexts/base.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/test/unit/src/config/template-contexts/base.ts b/core/test/unit/src/config/template-contexts/base.ts index 3b6495f49e..4a90215770 100644 --- a/core/test/unit/src/config/template-contexts/base.ts +++ b/core/test/unit/src/config/template-contexts/base.ts @@ -272,7 +272,7 @@ describe("ConfigContext", () => { c.addValues({ nested }) await expectError(() => resolveKey(c, ["nested", "key"]), { contains: - "Invalid template string (${'${nested.key}'}): Circular reference detected when resolving key nested.key (nested -> nested.key)", + "Invalid template string (${nested.key}): Circular reference detected when resolving key nested.key (nested -> nested.key)", }) }) }) From 1f7d99cfe390da11eb11ce073a3fbdc776995f57 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Thu, 5 Dec 2024 14:56:14 +0100 Subject: [PATCH 32/43] chore: remove dead code --- core/src/template-string/static-analysis.ts | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/core/src/template-string/static-analysis.ts b/core/src/template-string/static-analysis.ts index ea1b9de5e9..9109a9f398 100644 --- a/core/src/template-string/static-analysis.ts +++ b/core/src/template-string/static-analysis.ts @@ -52,24 +52,6 @@ export function* visitAll({ } } -export function containsTemplateExpression(generator: TemplateExpressionGenerator): boolean { - for (const node of generator) { - if (node instanceof TemplateExpression) { - return true - } - } - - return false -} - -export function containsContextLookupReferences(generator: TemplateExpressionGenerator): boolean { - for (const _ of getContextLookupReferences(generator, new NoOpContext())) { - return true - } - - return false -} - export type ContextLookupReferenceFinding = | { type: "resolvable" From 03006d1ddfc73ba3f44330f1a8d0577f40b2caf7 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Thu, 5 Dec 2024 15:18:59 +0100 Subject: [PATCH 33/43] fix: handle all kinds of `GardenError` on the AST level --- core/src/template-string/ast.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/core/src/template-string/ast.ts b/core/src/template-string/ast.ts index b6de8ae8db..13d175fa38 100644 --- a/core/src/template-string/ast.ts +++ b/core/src/template-string/ast.ts @@ -13,9 +13,8 @@ import { renderKeyPath, type ConfigContext, type ContextResolveOpts, - ContextResolveError, } from "../config/template-contexts/base.js" -import { InternalError, TemplateStringError } from "../exceptions.js" +import { GardenError, InternalError, TemplateStringError } from "../exceptions.js" import { getHelperFunctions } from "./functions.js" import { isTemplatePrimitive, type TemplatePrimitive } from "./types.js" import type { Collection, CollectionOrValue } from "../util/objects.js" @@ -647,12 +646,12 @@ export class ContextLookupExpression extends TemplateExpression { }, }) } catch (e) { - if (e instanceof ContextResolveError) { - throw new TemplateStringError({ message: e.message, loc: this.loc }) - } if (e instanceof TemplateStringError) { throw new TemplateStringError({ message: e.originalMessage, loc: this.loc }) } + if (e instanceof GardenError) { + throw new TemplateStringError({ message: e.message, loc: this.loc }) + } throw e } } From 4e515881b6c38212832961794ce20d79158571f2 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Thu, 5 Dec 2024 16:00:46 +0100 Subject: [PATCH 34/43] test: fix partial runtime resolution tests --- core/src/config/template-contexts/base.ts | 4 ++-- core/src/outputs.ts | 24 +++++++++++++---------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index bcc0e929e7..69621ce3ed 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -214,8 +214,8 @@ export abstract class ConfigContext { } } - if (typeof resolved === "symbol") { - return { resolved, getUnavailableReason } + if (typeof value === "symbol") { + return { resolved: value, getUnavailableReason } } // If we're allowing partial strings, we throw the error immediately to end the resolution flow. The error diff --git a/core/src/outputs.ts b/core/src/outputs.ts index b7889232de..bd71df1dd6 100644 --- a/core/src/outputs.ts +++ b/core/src/outputs.ts @@ -35,17 +35,15 @@ export async function resolveProjectOutputs(garden: Garden, log: Log): Promise Date: Thu, 5 Dec 2024 17:39:50 +0100 Subject: [PATCH 35/43] fix: make sure that yaml path is included in template string errors Co-authored-by: Vladimir Vagaytsev --- core/src/commands/custom.ts | 8 +- core/src/commands/workflow.ts | 4 +- core/src/config/config-template.ts | 6 +- core/src/config/project.ts | 4 +- core/src/config/provider.ts | 9 +- core/src/config/render-template.ts | 8 +- core/src/config/template-contexts/base.ts | 1 + core/src/config/validation.ts | 6 +- core/src/config/workflow.ts | 6 +- core/src/exceptions.ts | 12 +- core/src/garden.ts | 4 +- core/src/graph/actions.ts | 6 +- core/src/outputs.ts | 11 +- core/src/tasks/resolve-action.ts | 8 +- core/src/tasks/resolve-provider.ts | 4 +- core/src/template-string/ast.ts | 76 ++++++--- core/src/template-string/static-analysis.ts | 72 ++++++-- core/src/template-string/template-string.ts | 177 +++++++++++++------- core/src/util/testing.ts | 12 +- core/test/unit/src/config/validation.ts | 12 +- core/test/unit/src/template-string.ts | 57 ++++--- 21 files changed, 342 insertions(+), 161 deletions(-) diff --git a/core/src/commands/custom.ts b/core/src/commands/custom.ts index 52976e8592..b0310ba955 100644 --- a/core/src/commands/custom.ts +++ b/core/src/commands/custom.ts @@ -119,7 +119,7 @@ export class CustomCommandWrapper extends Command { const commandVariables = resolveTemplateStrings({ value: this.spec.variables, context: variablesContext, - source: { yamlDoc, basePath: ["variables"] }, + source: { yamlDoc, path: ["variables"] }, }) const variables: any = jsonMerge(cloneDeep(garden.variables), commandVariables) @@ -137,7 +137,7 @@ export class CustomCommandWrapper extends Command { config: resolveTemplateStrings({ value: this.spec.exec, context: commandContext, - source: { yamlDoc, basePath: ["exec"] }, + source: { yamlDoc, path: ["exec"] }, }), schema: customCommandExecSchema(), path: this.spec.internal.basePath, @@ -189,7 +189,7 @@ export class CustomCommandWrapper extends Command { config: resolveTemplateStrings({ value: this.spec.gardenCommand, context: commandContext, - source: { yamlDoc, basePath: ["gardenCommand"] }, + source: { yamlDoc, path: ["gardenCommand"] }, }), schema: customCommandGardenCommandSchema(), path: this.spec.internal.basePath, @@ -291,7 +291,7 @@ export async function getCustomCommands(log: Log, projectRoot: string) { path: (config).internal.basePath, projectRoot, configType: `custom Command '${config.name}'`, - source: { yamlDoc: (config).internal.yamlDoc }, + source: { yamlDoc: (config).internal.yamlDoc, path: [] }, }) ) diff --git a/core/src/commands/workflow.ts b/core/src/commands/workflow.ts index 5bdd2f7df1..05fbbce815 100644 --- a/core/src/commands/workflow.ts +++ b/core/src/commands/workflow.ts @@ -90,7 +90,7 @@ export class WorkflowCommand extends Command { const files = resolveTemplateStrings({ value: workflow.files || [], context: templateContext, - source: { yamlDoc, basePath: ["files"] }, + source: { yamlDoc, path: ["files"] }, }) // Write all the configured files for the workflow @@ -175,7 +175,7 @@ export class WorkflowCommand extends Command { step.command = resolveTemplateStrings({ value: step.command, context: stepTemplateContext, - source: { yamlDoc, basePath: ["steps", index, "command"] }, + source: { yamlDoc, path: ["steps", index, "command"] }, }).filter((arg) => !!arg) stepResult = await runStepCommand(stepParams) diff --git a/core/src/config/config-template.ts b/core/src/config/config-template.ts index 725f2d534c..82a99e682f 100644 --- a/core/src/config/config-template.ts +++ b/core/src/config/config-template.ts @@ -67,7 +67,11 @@ export async function resolveConfigTemplate( const loggedIn = garden.isLoggedIn() const enterpriseDomain = garden.cloudApi?.domain const context = new ProjectConfigContext({ ...garden, loggedIn, enterpriseDomain }) - const resolved = resolveTemplateStrings({ value: partial, context, source: { yamlDoc: resource.internal.yamlDoc } }) + const resolved = resolveTemplateStrings({ + value: partial, + context, + source: { yamlDoc: resource.internal.yamlDoc, path: [] }, + }) const configPath = resource.internal.configFilePath // Validate the partial config diff --git a/core/src/config/project.ts b/core/src/config/project.ts index 9b0bc06ccd..cf33c8f930 100644 --- a/core/src/config/project.ts +++ b/core/src/config/project.ts @@ -501,7 +501,7 @@ export function resolveProjectConfig({ secrets, commandInfo, }), - source: { yamlDoc: config.internal.yamlDoc, basePath: [] }, + source: { yamlDoc: config.internal.yamlDoc, path: [] }, }) } catch (err) { log.error("Failed to resolve project configuration.") @@ -622,7 +622,7 @@ export const pickEnvironment = profileAsync(async function _pickEnvironment({ }) const projectVariables: DeepPrimitiveMap = merge(projectConfig.variables, projectVarfileVars) - const source = { yamlDoc: projectConfig.internal.yamlDoc, basePath: ["environments", index] } + const source = { yamlDoc: projectConfig.internal.yamlDoc, path: ["environments", index] } // Resolve template strings in the environment config, except providers environmentConfig = resolveTemplateStrings({ diff --git a/core/src/config/provider.ts b/core/src/config/provider.ts index f3af7ea0d9..5711d6319e 100644 --- a/core/src/config/provider.ts +++ b/core/src/config/provider.ts @@ -180,7 +180,14 @@ export function getProviderTemplateReferences(config: GenericProviderConfig) { const deps: string[] = [] const generator = getContextLookupReferences( - visitAll({ value: config, parseTemplateStrings: true }), + visitAll({ + value: config, + parseTemplateStrings: true, + // TODO: get proper source + source: { + path: [], + }, + }), new NoOpContext() ) for (const finding of generator) { diff --git a/core/src/config/render-template.ts b/core/src/config/render-template.ts index 37d5f4e1d2..29676df8c0 100644 --- a/core/src/config/render-template.ts +++ b/core/src/config/render-template.ts @@ -132,7 +132,7 @@ export async function renderConfigTemplate({ const resolvedWithoutInputs = resolveTemplateStrings({ value: { ...omit(config, "inputs") }, context: templateContext, - source: { yamlDoc }, + source: { yamlDoc, path: [] }, }) const partiallyResolvedInputs = resolveTemplateStrings({ value: config.inputs || {}, @@ -140,7 +140,7 @@ export async function renderConfigTemplate({ contextOpts: { allowPartial: true, }, - source: { yamlDoc, basePath: ["inputs"] }, + source: { yamlDoc, path: ["inputs"] }, }) let resolved: RenderTemplateConfig = { ...resolvedWithoutInputs, @@ -214,7 +214,7 @@ async function renderModules({ value: m, context, contextOpts: { allowPartial: true }, - source: { yamlDoc, basePath: ["modules", i] }, + source: { yamlDoc, path: ["modules", i] }, }) const renderConfigPath = renderConfig.internal.configFilePath || renderConfig.internal.basePath @@ -279,7 +279,7 @@ async function renderConfigs({ value: templateConfigs, context, contextOpts: { allowPartial: true }, - source: { yamlDoc: template.internal.yamlDoc, basePath: ["inputs"] }, + source: { yamlDoc: template.internal.yamlDoc, path: ["inputs"] }, }) return Promise.all( diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index 69621ce3ed..de7f4653b8 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -118,6 +118,7 @@ export abstract class ConfigContext { // keep track of which resolvers have been called, in order to detect circular references let getAvailableKeys: (() => string[]) | undefined = undefined + // eslint-disable-next-line @typescript-eslint/no-this-alias let value: CollectionOrValue | ConfigContext | Function = this let partial = false let nextKey = key[0] diff --git a/core/src/config/validation.ts b/core/src/config/validation.ts index 5783e52b43..0442268955 100644 --- a/core/src/config/validation.ts +++ b/core/src/config/validation.ts @@ -40,8 +40,8 @@ const joiOptions: Joi.ValidationOptions = { } export interface ConfigSource { + path: ObjectPath yamlDoc?: YamlDocumentWithSource - basePath?: ObjectPath } export interface ValidateOptions { @@ -113,7 +113,7 @@ export function validateConfig(params: ValidateCon return validateSchema(config, schema, { context: context.trim(), - source: config.internal.yamlDoc ? { yamlDoc: config.internal.yamlDoc, basePath: yamlDocBasePath } : undefined, + source: config.internal.yamlDoc ? { yamlDoc: config.internal.yamlDoc, path: yamlDocBasePath } : undefined, ErrorClass, }) } @@ -132,7 +132,7 @@ export function validateSchema( const yamlDoc = source?.yamlDoc const rawYaml = yamlDoc?.source - const yamlBasePath = source?.basePath || [] + const yamlBasePath = source?.path || [] const errorDetails = error.details.map((e) => { e.message = diff --git a/core/src/config/workflow.ts b/core/src/config/workflow.ts index cc572cb988..856feeb2b4 100644 --- a/core/src/config/workflow.ts +++ b/core/src/config/workflow.ts @@ -362,7 +362,11 @@ export function resolveWorkflowConfig(garden: Garden, config: WorkflowConfig) { } let resolvedPartialConfig: WorkflowConfig = { - ...resolveTemplateStrings({ value: partialConfig, context, source: { yamlDoc: config.internal.yamlDoc } }), + ...resolveTemplateStrings({ + value: partialConfig, + context, + source: { yamlDoc: config.internal.yamlDoc, path: [] }, + }), name: config.name, } diff --git a/core/src/exceptions.ts b/core/src/exceptions.ts index 00ad86d960..8c97c9631e 100644 --- a/core/src/exceptions.ts +++ b/core/src/exceptions.ts @@ -18,6 +18,7 @@ import dns from "node:dns" import { styles } from "./logger/styles.js" import type { ExecaError } from "execa" import type { Location } from "./template-string/ast.js" +import type { ConfigSource } from "./config/validation.js" // Unfortunately, NodeJS does not provide a list of all error codes, so we have to maintain this list manually. // See https://nodejs.org/docs/latest-v18.x/api/dns.html#error-codes @@ -312,14 +313,11 @@ export class TemplateStringError extends GardenError { loc: Location originalMessage: string - constructor(params: GardenErrorParams & { loc: Location }) { - // TODO: figure out how to get correct path - // const path = params.loc?.source?.basePath - // const path: string[] | undefined = undefined - // const pathDescription = path ? ` at path ${styles.accent(path.join("."))}` : "" - const pathDescription = "" + constructor(params: GardenErrorParams & { loc: Location; yamlSource: ConfigSource }) { + const path = params.yamlSource.path + const pathDescription = path.length > 0 ? ` at path ${styles.accent(path.join("."))}` : "" const prefix = `Invalid template string (${styles.accent( - truncate(params.loc.source, { length: 200 }).replace(/\n/g, "\\n") + truncate(params.loc.source.rawTemplateString, { length: 200 }).replace(/\n/g, "\\n") )})${pathDescription}: ` const message = params.message.startsWith(prefix) ? params.message : prefix + params.message diff --git a/core/src/garden.ts b/core/src/garden.ts index 40bf3b0284..d1ebb00487 100644 --- a/core/src/garden.ts +++ b/core/src/garden.ts @@ -1655,7 +1655,7 @@ export class Garden { */ public getProjectSources() { const context = new RemoteSourceConfigContext(this, this.variables) - const source = { yamlDoc: this.projectConfig.internal.yamlDoc, basePath: ["sources"] } + const source = { yamlDoc: this.projectConfig.internal.yamlDoc, path: ["sources"] } const resolved = validateSchema( resolveTemplateStrings({ value: this.projectSources, context, source }), projectSourcesSchema(), @@ -1885,7 +1885,7 @@ export async function resolveGardenParamsPartial(currentDirectory: string, opts: configType: "project environments", path: config.path, projectRoot: config.path, - source: { yamlDoc: config.internal.yamlDoc, basePath: ["environments"] }, + source: { yamlDoc: config.internal.yamlDoc, path: ["environments"] }, }) const configDefaultEnvironment = resolveTemplateString({ diff --git a/core/src/graph/actions.ts b/core/src/graph/actions.ts index 76e331a75f..d7e3058552 100644 --- a/core/src/graph/actions.ts +++ b/core/src/graph/actions.ts @@ -813,7 +813,7 @@ export const preprocessActionConfig = profileAsync(async function preprocessActi contextOpts: { allowPartial: false, }, - source: { yamlDoc, basePath: [] }, + source: { yamlDoc, path: [] }, }) config = { ...config, ...resolvedBuiltin } const { spec = {} } = config @@ -831,7 +831,7 @@ export const preprocessActionConfig = profileAsync(async function preprocessActi name: config.name, path: config.internal.basePath, projectRoot: garden.projectRoot, - source: { yamlDoc }, + source: { yamlDoc, path: [] }, }) config = { ...config, variables: resolvedVariables, spec } @@ -844,7 +844,7 @@ export const preprocessActionConfig = profileAsync(async function preprocessActi contextOpts: { allowPartial: true, }, - source: { yamlDoc }, + source: { yamlDoc, path: [] }, }) config = { ...config, ...resolvedOther } } diff --git a/core/src/outputs.ts b/core/src/outputs.ts index bd71df1dd6..b0ac73a98c 100644 --- a/core/src/outputs.ts +++ b/core/src/outputs.ts @@ -36,7 +36,14 @@ export async function resolveProjectOutputs(garden: Garden, log: Log): Promise extends BaseActionTask extends BaseActionTask extends BaseActionTask extends BaseActionTask { }) } - const source = { yamlDoc, basePath: yamlDocBasePath } + const source = { yamlDoc, path: yamlDocBasePath } let resolvedConfig = resolveTemplateStrings({ value: this.config, context, source }) const providerName = resolvedConfig.name @@ -289,7 +289,7 @@ export class ResolveProviderTask extends BaseTask { projectRoot: this.garden.projectRoot, configType: `provider configuration (base schema from '${base.name}' plugin)`, ErrorClass: ConfigurationError, - source: { yamlDoc, basePath: yamlDocBasePath }, + source: { yamlDoc, path: yamlDocBasePath }, }) } diff --git a/core/src/template-string/ast.ts b/core/src/template-string/ast.ts index 13d175fa38..9af51dc1e6 100644 --- a/core/src/template-string/ast.ts +++ b/core/src/template-string/ast.ts @@ -18,12 +18,13 @@ import { GardenError, InternalError, TemplateStringError } from "../exceptions.j import { getHelperFunctions } from "./functions.js" import { isTemplatePrimitive, type TemplatePrimitive } from "./types.js" import type { Collection, CollectionOrValue } from "../util/objects.js" -import { validateSchema } from "../config/validation.js" +import { type ConfigSource, validateSchema } from "../config/validation.js" import type { TemplateExpressionGenerator } from "./static-analysis.js" type EvaluateArgs = { context: ConfigContext opts: ContextResolveOpts + yamlSource: ConfigSource /** * Whether or not to throw an error if ContextLookupExpression fails to resolve variable. @@ -32,6 +33,10 @@ type EvaluateArgs = { optional?: boolean } +export type TemplateStringSource = { + rawTemplateString: string +} + /** * Returned by the `location()` helper in PEG.js. */ @@ -46,7 +51,7 @@ export type Location = { line: number column: number } - source: string + source: TemplateStringSource } export type TemplateEvaluationResult = @@ -54,20 +59,26 @@ export type TemplateEvaluationResult = | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER -function* astVisitAll(e: TemplateExpression): TemplateExpressionGenerator { +function* astVisitAll(e: TemplateExpression, source: ConfigSource): TemplateExpressionGenerator { for (const key in e) { if (key === "loc") { continue } const propertyValue = e[key] if (propertyValue instanceof TemplateExpression) { - yield* astVisitAll(propertyValue) - yield propertyValue + yield* astVisitAll(propertyValue, source) + yield { + value: propertyValue, + yamlSource: source, + } } else if (Array.isArray(propertyValue)) { for (const item of propertyValue) { if (item instanceof TemplateExpression) { - yield* astVisitAll(item) - yield item + yield* astVisitAll(item, source) + yield { + value: item, + yamlSource: source, + } } } } @@ -77,8 +88,8 @@ function* astVisitAll(e: TemplateExpression): TemplateExpressionGenerator { export abstract class TemplateExpression { constructor(public readonly loc: Location) {} - *visitAll(): TemplateExpressionGenerator { - yield* astVisitAll(this) + *visitAll(source: ConfigSource): TemplateExpressionGenerator { + yield* astVisitAll(this, source) } abstract evaluate( @@ -308,12 +319,13 @@ export abstract class BinaryExpression extends TemplateExpression { return CONTEXT_RESOLVE_KEY_NOT_FOUND } - return this.transform(left, right) + return this.transform(left, right, args) } abstract transform( left: CollectionOrValue, - right: CollectionOrValue + right: CollectionOrValue, + params: EvaluateArgs ): CollectionOrValue } @@ -332,7 +344,8 @@ export class NotEqualExpression extends BinaryExpression { export class AddExpression extends BinaryExpression { override transform( left: CollectionOrValue, - right: CollectionOrValue + right: CollectionOrValue, + { yamlSource }: EvaluateArgs ): CollectionOrValue { if (isNumber(left) && isNumber(right)) { return left + right @@ -346,6 +359,7 @@ export class AddExpression extends BinaryExpression { throw new TemplateStringError({ message: `Both terms need to be either arrays or strings or numbers for + operator (got ${typeof left} and ${typeof right}).`, loc: this.loc, + yamlSource, }) } } @@ -354,12 +368,14 @@ export class AddExpression extends BinaryExpression { export class ContainsExpression extends BinaryExpression { override transform( collection: CollectionOrValue, - element: CollectionOrValue + element: CollectionOrValue, + { yamlSource }: EvaluateArgs ): boolean { if (!isTemplatePrimitive(element)) { throw new TemplateStringError({ message: `The right-hand side of a 'contains' operator must be a string, number, boolean or null (got ${typeof element}).`, loc: this.loc, + yamlSource, }) } @@ -378,6 +394,7 @@ export class ContainsExpression extends BinaryExpression { throw new TemplateStringError({ message: `The left-hand side of a 'contains' operator must be a string, array or object (got ${collection}).`, loc: this.loc, + yamlSource, }) } } @@ -385,7 +402,8 @@ export class ContainsExpression extends BinaryExpression { export abstract class BinaryExpressionOnNumbers extends BinaryExpression { override transform( left: CollectionOrValue, - right: CollectionOrValue + right: CollectionOrValue, + { yamlSource }: EvaluateArgs ): CollectionOrValue { // All other operators require numbers to make sense (we're not gonna allow random JS weirdness) if (!isNumber(left) || !isNumber(right)) { @@ -394,6 +412,7 @@ export abstract class BinaryExpressionOnNumbers extends BinaryExpression { this.operator } operator (got ${typeof left} and ${typeof right}).`, loc: this.loc, + yamlSource, }) } @@ -580,6 +599,7 @@ export class MemberExpression extends TemplateExpression { throw new TemplateStringError({ message: `Expression in brackets must resolve to a string or number (got ${typeof inner}).`, loc: this.loc, + yamlSource: args.yamlSource, }) } @@ -599,20 +619,21 @@ export class ContextLookupExpression extends TemplateExpression { context, opts, optional, + yamlSource, }: EvaluateArgs): | CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { const keyPath: (string | number)[] = [] for (const k of this.keyPath) { - const evaluated = k.evaluate({ context, opts, optional }) + const evaluated = k.evaluate({ context, opts, optional, yamlSource }) if (typeof evaluated === "symbol") { return evaluated } keyPath.push(evaluated) } - const { resolved, getUnavailableReason } = this.resolveContext(context, keyPath, opts) + const { resolved, getUnavailableReason } = this.resolveContext(context, keyPath, opts, yamlSource) // if context returns key available later, then we do not need to throw, because partial mode is enabled. if (resolved === CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) { @@ -629,13 +650,19 @@ export class ContextLookupExpression extends TemplateExpression { throw new TemplateStringError({ message: getUnavailableReason?.() || `Could not find key ${renderKeyPath(keyPath)}`, loc: this.loc, + yamlSource, }) } return resolved } - private resolveContext(context: ConfigContext, keyPath: (string | number)[], opts: ContextResolveOpts) { + private resolveContext( + context: ConfigContext, + keyPath: (string | number)[], + opts: ContextResolveOpts, + yamlSource: ConfigSource + ) { try { return context.resolve({ key: keyPath, @@ -647,10 +674,10 @@ export class ContextLookupExpression extends TemplateExpression { }) } catch (e) { if (e instanceof TemplateStringError) { - throw new TemplateStringError({ message: e.originalMessage, loc: this.loc }) + throw new TemplateStringError({ message: e.originalMessage, loc: this.loc, yamlSource }) } if (e instanceof GardenError) { - throw new TemplateStringError({ message: e.message, loc: this.loc }) + throw new TemplateStringError({ message: e.message, loc: this.loc, yamlSource }) } throw e } @@ -686,8 +713,7 @@ export class FunctionCallExpression extends TemplateExpression { const result: CollectionOrValue = this.callHelperFunction({ functionName: functionName.toString(), args: functionArgs, - context: args.context, - opts: args.opts, + yamlSource: args.yamlSource, }) return result @@ -696,11 +722,11 @@ export class FunctionCallExpression extends TemplateExpression { callHelperFunction({ functionName, args, + yamlSource, }: { functionName: string + yamlSource: ConfigSource args: CollectionOrValue[] - context: ConfigContext - opts: ContextResolveOpts }): CollectionOrValue { const helperFunctions = getHelperFunctions() const spec = helperFunctions[functionName] @@ -710,6 +736,7 @@ export class FunctionCallExpression extends TemplateExpression { throw new TemplateStringError({ message: `Could not find helper function '${functionName}'. Available helper functions: ${availableFns}`, loc: this.loc, + yamlSource, }) } @@ -723,6 +750,7 @@ export class FunctionCallExpression extends TemplateExpression { throw new TemplateStringError({ message: `Missing argument '${argName}' (at index ${i}) for ${functionName} helper function.`, loc: this.loc, + yamlSource, }) } @@ -733,6 +761,7 @@ export class FunctionCallExpression extends TemplateExpression { super({ message, loc, + yamlSource, }) } } @@ -749,6 +778,7 @@ export class FunctionCallExpression extends TemplateExpression { } catch (error) { throw new TemplateStringError({ message: `Error from helper function ${functionName}: ${error}`, + yamlSource, loc: this.loc, }) } diff --git a/core/src/template-string/static-analysis.ts b/core/src/template-string/static-analysis.ts index 9109a9f398..e44324d257 100644 --- a/core/src/template-string/static-analysis.ts +++ b/core/src/template-string/static-analysis.ts @@ -12,42 +12,72 @@ import { ContextLookupExpression, TemplateExpression } from "./ast.js" import type { TemplatePrimitive } from "./types.js" import { parseTemplateString } from "./template-string.js" import type { ConfigContext } from "../config/template-contexts/base.js" -import { NoOpContext } from "../config/template-contexts/base.js" import { GardenError, InternalError } from "../exceptions.js" +import { type ConfigSource } from "../config/validation.js" -export type TemplateExpressionGenerator = Generator +export type TemplateExpressionGenerator = Generator< + { + value: TemplatePrimitive | TemplateExpression + yamlSource: ConfigSource + }, + void, + undefined +> export function* visitAll({ value, parseTemplateStrings = false, + source, }: { value: CollectionOrValue parseTemplateStrings?: boolean + source: ConfigSource }): TemplateExpressionGenerator { if (isArray(value)) { - for (const [_k, v] of value.entries()) { - yield* visitAll({ value: v, parseTemplateStrings }) + for (const [k, v] of value.entries()) { + yield* visitAll({ + value: v, + parseTemplateStrings, + source: { + ...source, + path: [...source.path, k], + }, + }) } } else if (isPlainObject(value)) { for (const k of Object.keys(value)) { - yield* visitAll({ value: value[k], parseTemplateStrings }) + yield* visitAll({ + value: value[k], + parseTemplateStrings, + source: { + ...source, + path: [...source.path, k], + }, + }) } } else { if (parseTemplateStrings && typeof value === "string") { const parsed = parseTemplateString({ rawTemplateString: value, unescape: false, + source, }) if (typeof parsed === "string") { - yield parsed + yield { + value: parsed, + yamlSource: source, + } } else { - yield* parsed.visitAll() + yield* parsed.visitAll(source) } } else if (value instanceof TemplateExpression) { - yield* value.visitAll() + yield* value.visitAll(source) } else { - yield value + yield { + value, + yamlSource: source, + } } } } @@ -56,10 +86,12 @@ export type ContextLookupReferenceFinding = | { type: "resolvable" keyPath: (string | number)[] + yamlSource: ConfigSource } | { type: "unresolvable" keyPath: (string | number | { getError: () => GardenError })[] + yamlSource: ConfigSource } function captureError(arg: () => void): () => GardenError { @@ -82,17 +114,27 @@ export function* getContextLookupReferences( generator: TemplateExpressionGenerator, context: ConfigContext ): Generator { - for (const expression of generator) { - if (expression instanceof ContextLookupExpression) { + for (const { value, yamlSource } of generator) { + if (value instanceof ContextLookupExpression) { let isResolvable: boolean = true - const keyPath = expression.keyPath.map((keyPathExpression) => { - const key = keyPathExpression.evaluate({ context, opts: { allowPartial: false }, optional: true }) + const keyPath = value.keyPath.map((keyPathExpression) => { + const key = keyPathExpression.evaluate({ + context, + opts: { allowPartial: false }, + optional: true, + yamlSource, + }) if (typeof key === "symbol") { isResolvable = false return { getError: captureError(() => // this will throw an error, because the key could not be resolved - keyPathExpression.evaluate({ context, opts: { allowPartial: false }, optional: false }) + keyPathExpression.evaluate({ + context, + opts: { allowPartial: false }, + optional: false, + yamlSource, + }) ), } } @@ -104,10 +146,12 @@ export function* getContextLookupReferences( ? { type: "resolvable", keyPath: keyPath as (string | number)[], + yamlSource, } : { type: "unresolvable", keyPath, + yamlSource, } } } diff --git a/core/src/template-string/template-string.ts b/core/src/template-string/template-string.ts index 8de099db33..671685f7fa 100644 --- a/core/src/template-string/template-string.ts +++ b/core/src/template-string/template-string.ts @@ -34,7 +34,7 @@ import { dedent, deline, naturalList, titleize } from "../util/string.js" import type { ObjectWithName } from "../util/util.js" import type { Log } from "../logger/log-entry.js" import type { ModuleConfigContext } from "../config/template-contexts/module.js" -import type { ActionKind } from "../actions/types.js" +import type { ActionConfig, ActionKind } from "../actions/types.js" import { actionKindsLower } from "../actions/types.js" import type { CollectionOrValue } from "../util/objects.js" import { deepMap } from "../util/objects.js" @@ -86,12 +86,12 @@ const parseTemplateStringCache = new LRUCache parseTemplateString({ rawTemplateString: nested, source, unescape }), - TemplateStringError, + parseNested: (nested: string) => parseTemplateString({ rawTemplateString: nested, unescape, source }), + TemplateStringError: ParserError, unescape, - grammarSource: rawTemplateString, + grammarSource: templateStringSource, }, ]) @@ -142,21 +148,25 @@ export function resolveTemplateString({ string, context, contextOpts = {}, - // TODO: Path and source? what to do with path here? - path, source, }: { string: string context: ConfigContext contextOpts?: ContextResolveOpts source?: ConfigSource - path?: ObjectPath }): CollectionOrValue { + if (source === undefined) { + source = { + path: [], + yamlDoc: undefined, + } + } + const parsed = parseTemplateString({ rawTemplateString: string, - source, // TODO: remove unescape hacks. unescape: shouldUnescape(contextOpts), + source, }) // string does not contain @@ -167,6 +177,7 @@ export function resolveTemplateString({ const result = parsed.evaluate({ context, opts: contextOpts, + yamlSource: source, }) if (!contextOpts.allowPartial && result === CONTEXT_RESOLVE_KEY_NOT_FOUND) { @@ -194,13 +205,11 @@ export function resolveTemplateStrings({ value, context, contextOpts = {}, - path, source, }: { value: T context: ConfigContext contextOpts?: ContextResolveOpts - path?: ObjectPath source: ConfigSource | undefined }): T { if (value === null) { @@ -210,12 +219,17 @@ export function resolveTemplateStrings({ return undefined as T } - if (!path) { - path = [] + if (!source) { + source = { + path: [], + } + } + if (!source.path) { + source.path = [] } if (typeof value === "string") { - return resolveTemplateString({ string: value, context, path, source, contextOpts }) + return resolveTemplateString({ string: value, context, source, contextOpts }) } else if (Array.isArray(value)) { const output: unknown[] = [] @@ -230,7 +244,7 @@ export function resolveTemplateStrings({ ) throw new TemplateError({ message: `A list item with a ${arrayConcatKey} key cannot have any other keys (found ${extraKeys})`, - path, + path: source.path, value, resolved: undefined, }) @@ -243,8 +257,10 @@ export function resolveTemplateStrings({ contextOpts: { ...contextOpts, }, - path: path && [...path, arrayConcatKey], - source, + source: { + ...source, + path: [...source.path, arrayConcatKey], + }, }) if (Array.isArray(resolved)) { @@ -254,13 +270,23 @@ export function resolveTemplateStrings({ } else { throw new TemplateError({ message: `Value of ${arrayConcatKey} key must be (or resolve to) an array (got ${typeof resolved})`, - path, + path: source.path, value, resolved, }) } } else { - output.push(resolveTemplateStrings({ value: v, context, contextOpts, source, path: path && [...path, i] })) + output.push( + resolveTemplateStrings({ + value: v, + context, + contextOpts, + source: { + ...source, + path: [...source.path, i], + }, + }) + ) } } @@ -268,17 +294,25 @@ export function resolveTemplateStrings({ } else if (isPlainObject(value)) { if (value[arrayForEachKey] !== undefined) { // Handle $forEach loop - return handleForEachObject({ value, context, contextOpts, path, source }) + return handleForEachObject({ value, context, contextOpts, source }) } else if (value[conditionalKey] !== undefined) { // Handle $if conditional - return handleConditional({ value, context, contextOpts, path, source }) + return handleConditional({ value, context, contextOpts, source }) } else { // Resolve $merge keys, depth-first, leaves-first let output = {} for (const k in value as Record) { const v = value[k] - const resolved = resolveTemplateStrings({ value: v, context, contextOpts, source, path: path && [...path, k] }) + const resolved = resolveTemplateStrings({ + value: v, + context, + contextOpts, + source: { + ...source, + path: source.path && [...source.path, k], + }, + }) if (k === objectSpreadKey) { if (isPlainObject(resolved)) { @@ -288,7 +322,7 @@ export function resolveTemplateStrings({ } else { throw new TemplateError({ message: `Value of ${objectSpreadKey} key must be (or resolve to) a mapping object (got ${typeof resolved})`, - path: [...path, k], + path: [...source.path, k], value, resolved, }) @@ -311,14 +345,12 @@ function handleForEachObject({ value, context, contextOpts, - path, source, }: { value: any context: ConfigContext contextOpts: ContextResolveOpts - path: ObjectPath | undefined - source: ConfigSource | undefined + source: ConfigSource }) { // Validate input object if (value[arrayForEachReturnKey] === undefined) { @@ -326,7 +358,7 @@ function handleForEachObject({ message: `Missing ${arrayForEachReturnKey} field next to ${arrayForEachKey} field. Got ${naturalList( Object.keys(value) )}`, - path: path && [...path, arrayForEachKey], + path: source.path && [...source.path, arrayForEachKey], value, resolved: undefined, }) @@ -341,7 +373,7 @@ function handleForEachObject({ message: `Found one or more unexpected keys on ${arrayForEachKey} object: ${extraKeys}. Expected keys: ${naturalList( expectedForEachKeys )}`, - path, + path: source.path, value, resolved: undefined, }) @@ -352,8 +384,10 @@ function handleForEachObject({ value: value[arrayForEachKey], context, contextOpts, - source, - path: path && [...path, arrayForEachKey], + source: { + ...source, + path: source.path && [...source.path, arrayForEachKey], + }, }) const isObject = isPlainObject(resolvedInput) @@ -363,7 +397,7 @@ function handleForEachObject({ } else { throw new TemplateError({ message: `Value of ${arrayForEachKey} key must be (or resolve to) an array or mapping object (got ${typeof resolvedInput})`, - path: path && [...path, arrayForEachKey], + path: source.path && [...source.path, arrayForEachKey], value, resolved: resolvedInput, }) @@ -418,8 +452,10 @@ function handleForEachObject({ value: value[arrayForEachFilterKey], context: loopContext, contextOpts, - source, - path: path && [...path, arrayForEachFilterKey], + source: { + ...source, + path: source.path && [...source.path, arrayForEachFilterKey], + }, }) if (filterResult === false) { @@ -427,7 +463,7 @@ function handleForEachObject({ } else if (filterResult !== true) { throw new TemplateError({ message: `${arrayForEachFilterKey} clause in ${arrayForEachKey} loop must resolve to a boolean value (got ${typeof resolvedInput})`, - path: path && [...path, arrayForEachFilterKey], + path: source.path && [...source.path, arrayForEachFilterKey], value, resolved: undefined, }) @@ -439,14 +475,16 @@ function handleForEachObject({ value: value[arrayForEachReturnKey], context: loopContext, contextOpts, - source, - path: path && [...path, arrayForEachKey, i], + source: { + ...source, + path: source.path && [...source.path, arrayForEachKey, i], + }, }) ) } // Need to resolve once more to handle e.g. $concat expressions - return resolveTemplateStrings({ value: output, context, contextOpts, source, path }) + return resolveTemplateStrings({ value: output, context, contextOpts, source }) } const expectedConditionalKeys = [conditionalKey, conditionalThenKey, conditionalElseKey] @@ -455,14 +493,12 @@ function handleConditional({ value, context, contextOpts, - path, source, }: { value: any context: ConfigContext contextOpts: ContextResolveOpts - path: ObjectPath | undefined - source: ConfigSource | undefined + source: ConfigSource }) { // Validate input object const thenExpression = value[conditionalThenKey] @@ -473,7 +509,7 @@ function handleConditional({ message: `Missing ${conditionalThenKey} field next to ${conditionalKey} field. Got: ${naturalList( Object.keys(value) )}`, - path, + path: source.path, value, resolved: undefined, }) @@ -488,7 +524,7 @@ function handleConditional({ message: `Found one or more unexpected keys on ${conditionalKey} object: ${extraKeys}. Expected: ${naturalList( expectedConditionalKeys )}`, - path, + path: source.path, value, resolved: undefined, }) @@ -499,8 +535,10 @@ function handleConditional({ value: value[conditionalKey], context, contextOpts, - source, - path: path && [...path, conditionalKey], + source: { + ...source, + path: source.path && [...source.path, conditionalKey], + }, }) if (typeof resolvedConditional !== "boolean") { @@ -509,7 +547,7 @@ function handleConditional({ } else { throw new TemplateError({ message: `Value of ${conditionalKey} key must be (or resolve to) a boolean (got ${typeof resolvedConditional})`, - path: path && [...path, conditionalKey], + path: source.path && [...source.path, conditionalKey], value, resolved: resolvedConditional, }) @@ -521,16 +559,20 @@ function handleConditional({ const resolvedThen = resolveTemplateStrings({ value: thenExpression, context, - path: path && [...path, conditionalThenKey], contextOpts, - source, + source: { + ...source, + path: source.path && [...source.path, conditionalThenKey], + }, }) const resolvedElse = resolveTemplateStrings({ value: elseExpression, context, - path: path && [...path, conditionalElseKey], contextOpts, - source, + source: { + ...source, + path: source.path && [...source.path, conditionalElseKey], + }, }) if (!!resolvedConditional) { @@ -682,11 +724,18 @@ export function extractRuntimeReference(finding: ContextLookupReferenceFinding): * An error is thrown if a reference is not resolvable, i.e. if a nested template is used as a reference. */ export function* getActionTemplateReferences( - config: object, + config: ActionConfig, context: ConfigContext ): Generator { const generator = getContextLookupReferences( - visitAll({ value: config as CollectionOrValue, parseTemplateStrings: true }), + visitAll({ + value: config as ObjectWithName, + parseTemplateStrings: true, + source: { + yamlDoc: config.internal?.yamlDoc, + path: [], + }, + }), context ) @@ -709,6 +758,10 @@ export function getModuleTemplateReferences(config: ModuleConfig, context: Modul visitAll({ value: config as ObjectWithName, parseTemplateStrings: true, + // Note: We're not implementing the YAML source mapping for modules + source: { + path: [], + }, }), context ) @@ -795,7 +848,17 @@ export function throwOnMissingSecretKeys(configs: ObjectWithName[], secrets: Str */ export function detectMissingSecretKeys(obj: ObjectWithName, secrets: StringMap): ContextKeySegment[] { const referencedKeys: ContextKeySegment[] = [] - const generator = getContextLookupReferences(visitAll({ value: obj, parseTemplateStrings: true }), new NoOpContext()) + const generator = getContextLookupReferences( + visitAll({ + value: obj, + parseTemplateStrings: true, + // TODO: add real yaml source + source: { + path: [], + }, + }), + new NoOpContext() + ) for (const finding of generator) { const keyPath = finding.keyPath if (keyPath[0] !== "secrets") { diff --git a/core/src/util/testing.ts b/core/src/util/testing.ts index 78983f7fbf..dd75c95997 100644 --- a/core/src/util/testing.ts +++ b/core/src/util/testing.ts @@ -19,7 +19,7 @@ import type { WorkflowConfig, WorkflowConfigMap } from "../config/workflow.js" import { resolveMsg, type Log, type LogEntry } from "../logger/log-entry.js" import type { GardenModule, ModuleConfigMap } from "../types/module.js" import { findByName, getNames, hashString } from "./util.js" -import { GardenError, InternalError } from "../exceptions.js" +import { GardenError, InternalError, toGardenError } from "../exceptions.js" import type { EventName, Events } from "../events/events.js" import { EventBus } from "../events/events.js" import { dedent, naturalList } from "./string.js" @@ -460,7 +460,7 @@ export class TestGarden extends Garden { } } -export function expectFuzzyMatch(str: string, sample: string | string[]) { +export function expectFuzzyMatch(str: string, sample: string | string[], assertionMessage?: string) { const errorMessageNonAnsi = stripAnsi(str) const samples = typeof sample === "string" ? [sample] : sample const samplesNonAnsi = samples.map(stripAnsi) @@ -468,7 +468,7 @@ export function expectFuzzyMatch(str: string, sample: string | string[]) { const actualErrorMsgLowercase = errorMessageNonAnsi.toLowerCase() const expectedErrorSample = s.toLowerCase() try { - expect(actualErrorMsgLowercase).to.contain(expectedErrorSample) + expect(actualErrorMsgLowercase, assertionMessage).to.contain(expectedErrorSample) } catch (err) { // eslint-disable-next-line no-console console.log( @@ -527,7 +527,11 @@ export function expectError(fn: Function, assertion: ExpectErrorAssertion = {}) if (contains) { const errorMessage = (errorMessageGetter || defaultErrorMessageGetter)(err) - expectFuzzyMatch(errorMessage, contains) + expectFuzzyMatch( + errorMessage, + contains, + `Assertion failed: '${errorMessage}' does not contain '${contains}'.\n\nOriginal error: ${toGardenError(err).stack}` + ) } if (message) { diff --git a/core/test/unit/src/config/validation.ts b/core/test/unit/src/config/validation.ts index e2d195e9c3..e480081b5b 100644 --- a/core/test/unit/src/config/validation.ts +++ b/core/test/unit/src/config/validation.ts @@ -126,7 +126,7 @@ describe("validateSchema", () => { } void expectError( - () => validateSchema(config, schema, { source: { yamlDoc } }), + () => validateSchema(config, schema, { source: { yamlDoc, path: [] } }), (err) => expect(stripAnsi(err.message)).to.equal(dedent` Validation error: @@ -170,7 +170,7 @@ describe("validateSchema", () => { } void expectError( - () => validateSchema(config, schema, { source: { yamlDoc } }), + () => validateSchema(config, schema, { source: { yamlDoc, path: [] } }), (err) => expect(stripAnsi(err.message)).to.equal(dedent` Validation error: @@ -218,7 +218,7 @@ describe("validateSchema", () => { } void expectError( - () => validateSchema(config, schema, { source: { yamlDoc } }), + () => validateSchema(config, schema, { source: { yamlDoc, path: [] } }), (err) => expect(stripAnsi(err.message)).to.equal(dedent` Validation error: @@ -264,7 +264,7 @@ describe("validateSchema", () => { } void expectError( - () => validateSchema(config, schema, { source: { yamlDoc } }), + () => validateSchema(config, schema, { source: { yamlDoc, path: [] } }), (err) => expect(stripAnsi(err.message)).to.equal(dedent` Validation error: @@ -319,7 +319,7 @@ describe("validateSchema", () => { } void expectError( - () => validateSchema(config, schema, { source: { yamlDoc } }), + () => validateSchema(config, schema, { source: { yamlDoc, path: [] } }), (err) => expect(stripAnsi(err.message)).to.equal(dedent` Validation error: @@ -359,7 +359,7 @@ describe("validateSchema", () => { } void expectError( - () => validateSchema(config.spec, schema, { source: { yamlDoc, basePath: ["spec"] } }), + () => validateSchema(config.spec, schema, { source: { yamlDoc, path: ["spec"] } }), (err) => expect(stripAnsi(err.message)).to.equal(dedent` Validation error: diff --git a/core/test/unit/src/template-string.ts b/core/test/unit/src/template-string.ts index 6ea6efdbab..fbfff43ed1 100644 --- a/core/test/unit/src/template-string.ts +++ b/core/test/unit/src/template-string.ts @@ -20,6 +20,7 @@ import { dedent } from "../../../src/util/string.js" import stripAnsi from "strip-ansi" import { TemplateStringError } from "../../../src/exceptions.js" import repeat from "lodash-es/repeat.js" +import type { ContextLookupReferenceFinding } from "../../../src/template-string/static-analysis.js" import { getContextLookupReferences, visitAll } from "../../../src/template-string/static-analysis.js" class TestContext extends ConfigContext { @@ -2189,30 +2190,48 @@ describe("getContextLookupReferences", () => { visitAll({ value: obj, parseTemplateStrings: true, + source: { + path: [], + }, }), new NoOpContext() ) ) - const expected = [ + const expected: ContextLookupReferenceFinding[] = [ { keyPath: ["my", "reference"], type: "resolvable", + yamlSource: { + path: ["foo"], + }, }, { keyPath: ["moo"], type: "resolvable", + yamlSource: { + path: ["nested", "boo"], + }, }, { keyPath: ["moo"], type: "resolvable", + yamlSource: { + path: ["nested", "foo"], + }, }, { keyPath: ["moo"], type: "resolvable", + yamlSource: { + path: ["nested", "foo"], + }, }, { keyPath: ["banana", "rama", "llama"], type: "resolvable", + yamlSource: { + path: ["nested", "banana"], + }, }, ] expect(result).to.eql(expected) @@ -2228,7 +2247,7 @@ describe("getActionTemplateReferences", () => { run: '${actions["run"].run-a}', test: '${actions["test"].test-a}', } - const actionTemplateReferences = Array.from(getActionTemplateReferences(config, new TestContext({}))) + const actionTemplateReferences = Array.from(getActionTemplateReferences(config as any, new TestContext({}))) expect(actionTemplateReferences).to.eql([ { kind: "Build", @@ -2257,7 +2276,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: "${actions}", } - void expectError(() => Array.from(getActionTemplateReferences(config, new TestContext({}))), { + void expectError(() => Array.from(getActionTemplateReferences(config as any, new TestContext({}))), { contains: "Found invalid action reference (missing kind)", }) }) @@ -2266,7 +2285,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: '${actions["badkind"].some-name}', } - void expectError(() => Array.from(getActionTemplateReferences(config, new TestContext({}))), { + void expectError(() => Array.from(getActionTemplateReferences(config as any, new TestContext({}))), { contains: "Found invalid action reference (invalid kind 'badkind')", }) }) @@ -2275,7 +2294,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: "${actions[123]}", } - void expectError(() => Array.from(getActionTemplateReferences(config, new TestContext({}))), { + void expectError(() => Array.from(getActionTemplateReferences(config as any, new TestContext({}))), { contains: "Found invalid action reference (kind is not a string)", }) }) @@ -2284,9 +2303,9 @@ describe("getActionTemplateReferences", () => { const config = { foo: "${actions[foo.bar].some-name}", } - void expectError(() => Array.from(getActionTemplateReferences(config, new TestContext({}))), { + void expectError(() => Array.from(getActionTemplateReferences(config as any, new TestContext({}))), { contains: - "found invalid action reference: invalid template string (${actions[foo.bar].some-name}): could not find key foo. available keys: (none)", + "found invalid action reference: invalid template string (${actions[foo.bar].some-name}) at path foo: could not find key foo. available keys: (none)", }) }) @@ -2294,7 +2313,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: '${actions[foo.bar || "hello"].some-name}', } - void expectError(() => Array.from(getActionTemplateReferences(config, new TestContext({}))), { + void expectError(() => Array.from(getActionTemplateReferences(config as any, new TestContext({}))), { contains: "found invalid action reference (invalid kind 'hello')", }) }) @@ -2303,7 +2322,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: '${actions["build"]}', } - void expectError(() => Array.from(getActionTemplateReferences(config, new TestContext({}))), { + void expectError(() => Array.from(getActionTemplateReferences(config as any, new TestContext({}))), { contains: "Found invalid action reference (missing name)", }) }) @@ -2312,7 +2331,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: '${actions["build"].123}', } - void expectError(() => Array.from(getActionTemplateReferences(config, new TestContext({}))), { + void expectError(() => Array.from(getActionTemplateReferences(config as any, new TestContext({}))), { contains: "Found invalid action reference (name is not a string)", }) }) @@ -2321,7 +2340,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: '${actions["build"][123]}', } - void expectError(() => Array.from(getActionTemplateReferences(config, new TestContext({}))), { + void expectError(() => Array.from(getActionTemplateReferences(config as any, new TestContext({}))), { contains: "Found invalid action reference (name is not a string)", }) }) @@ -2333,7 +2352,7 @@ describe("getActionTemplateReferences", () => { services: '${runtime["services"].service-a}', tasks: '${runtime["tasks"].task-a}', } - const actionTemplateReferences = Array.from(getActionTemplateReferences(config, new TestContext({}))) + const actionTemplateReferences = Array.from(getActionTemplateReferences(config as any, new TestContext({}))) expect(actionTemplateReferences).to.eql([ { kind: "Deploy", @@ -2352,7 +2371,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: "${runtime}", } - void expectError(() => Array.from(getActionTemplateReferences(config, new TestContext({}))), { + void expectError(() => Array.from(getActionTemplateReferences(config as any, new TestContext({}))), { contains: "Found invalid runtime reference (missing kind)", }) }) @@ -2361,7 +2380,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: '${runtime["badkind"].some-name}', } - void expectError(() => Array.from(getActionTemplateReferences(config, new TestContext({}))), { + void expectError(() => Array.from(getActionTemplateReferences(config as any, new TestContext({}))), { contains: "Found invalid runtime reference (invalid kind 'badkind')", }) }) @@ -2370,7 +2389,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: "${runtime[123]}", } - void expectError(() => Array.from(getActionTemplateReferences(config, new TestContext({}))), { + void expectError(() => Array.from(getActionTemplateReferences(config as any, new TestContext({}))), { contains: "Found invalid runtime reference (kind is not a string)", }) }) @@ -2379,9 +2398,9 @@ describe("getActionTemplateReferences", () => { const config = { foo: "${runtime[foo.bar].some-name}", } - void expectError(() => Array.from(getActionTemplateReferences(config, new TestContext({}))), { + void expectError(() => Array.from(getActionTemplateReferences(config as any, new TestContext({}))), { contains: - "found invalid action reference: invalid template string (${runtime[foo.bar].some-name}): could not find key foo. available keys: (none).", + "found invalid action reference: invalid template string (${runtime[foo.bar].some-name}) at path foo: could not find key foo. available keys: (none).", }) }) @@ -2389,7 +2408,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: '${runtime["tasks"]}', } - void expectError(() => Array.from(getActionTemplateReferences(config, new TestContext({}))), { + void expectError(() => Array.from(getActionTemplateReferences(config as any, new TestContext({}))), { contains: "Found invalid runtime reference (missing name)", }) }) @@ -2398,7 +2417,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: '${runtime["tasks"].123}', } - void expectError(() => Array.from(getActionTemplateReferences(config, new TestContext({}))), { + void expectError(() => Array.from(getActionTemplateReferences(config as any, new TestContext({}))), { contains: "Found invalid runtime reference (name is not a string)", }) }) From 77f4e5bc9a44f351f11c7c685d841679a10de05e Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Thu, 5 Dec 2024 20:39:12 +0100 Subject: [PATCH 36/43] refactor: make code safer and easier to reason about --- core/src/config/render-template.ts | 20 ++-- core/src/graph/actions.ts | 53 +++++---- core/src/outputs.ts | 5 +- core/src/template-string/static-analysis.ts | 20 +++- core/src/template-string/template-string.ts | 101 ++++++++++-------- core/src/util/testing.ts | 12 ++- .../src/actions/action-configs-to-graph.ts | 3 - core/test/unit/src/template-string.ts | 50 +++++++-- 8 files changed, 168 insertions(+), 96 deletions(-) diff --git a/core/src/config/render-template.ts b/core/src/config/render-template.ts index 29676df8c0..a1ff67410e 100644 --- a/core/src/config/render-template.ts +++ b/core/src/config/render-template.ts @@ -273,29 +273,29 @@ async function renderConfigs({ context: RenderTemplateConfigContext renderConfig: RenderTemplateConfig }): Promise { + const source = { yamlDoc: template.internal.yamlDoc, path: ["configs"] } + const templateDescription = `${configTemplateKind} '${template.name}'` const templateConfigs = template.configs || [] const partiallyResolvedTemplateConfigs = resolveTemplateStrings({ value: templateConfigs, context, contextOpts: { allowPartial: true }, - source: { yamlDoc: template.internal.yamlDoc, path: ["inputs"] }, + source, }) return Promise.all( - partiallyResolvedTemplateConfigs.map(async (m) => { + partiallyResolvedTemplateConfigs.map(async (m, index) => { // Resolve just the name, which must be immediately resolvable let resolvedName = m.name try { - const result = resolveTemplateString({ string: m.name, context, contextOpts: { allowPartial: false } }) - if (typeof result === "string") { - resolvedName = result - } else { - throw new ConfigurationError({ - message: "must resolve to string", - }) - } + resolvedName = resolveTemplateString({ + string: m.name, + context, + contextOpts: { allowPartial: false }, + source: { ...source, path: [...source.path, index, "name"] }, + }) as string } catch (error) { throw new ConfigurationError({ message: `Could not resolve the \`name\` field (${m.name}) for a config in ${templateDescription}: ${error}\n\nNote that template strings in config names in must be fully resolvable at the time of scanning. This means that e.g. references to other actions, modules or runtime outputs cannot be used.`, diff --git a/core/src/graph/actions.ts b/core/src/graph/actions.ts index d7e3058552..a776920e60 100644 --- a/core/src/graph/actions.ts +++ b/core/src/graph/actions.ts @@ -7,7 +7,7 @@ */ import cloneDeep from "fast-copy" -import { isEqual, mapValues, memoize, omit, pick, uniq } from "lodash-es" +import { isEqual, isString, mapValues, memoize, omit, pick, uniq } from "lodash-es" import type { Action, ActionConfig, @@ -68,6 +68,7 @@ import { profileAsync } from "../util/profiling.js" import { uuidv4 } from "../util/random.js" import { getSourcePath } from "../vcs/vcs.js" import { styles } from "../logger/styles.js" +import { isUnresolvableValue } from "../template-string/static-analysis.js" function* sliceToBatches(dict: Record, batchSize: number) { const entries = Object.entries(dict) @@ -1034,27 +1035,37 @@ function dependenciesFromActionConfig({ for (const ref of getActionTemplateReferences(config, templateContext)) { let needsExecuted = false - const outputKey = ref.fullRef[4] as string + const outputType = ref.keyPath[0] + + if (isUnresolvableValue(outputType)) { + const err = outputType.getError() + throw new ConfigurationError({ + message: `Found invalid action reference: ${err}`, + }) + } + + const outputKey = ref.keyPath[1] - // also avoid execution when referencing the static output keys of the ref action type. - // e.g. a helm deploy referencing container build static output deploymentImageName - // ${actions.build.my-container.outputs.deploymentImageName} const refActionKey = actionReferenceToString(ref) const refActionType = configsByKey[refActionKey]?.type - let refStaticOutputKeys: string[] = [] - if (refActionType) { - const refActionSpec = actionTypes[ref.kind][refActionType]?.spec - refStaticOutputKeys = refActionSpec?.staticOutputsSchema - ? describeSchema(refActionSpec.staticOutputsSchema).keys - : [] - } - if ( - ref.fullRef[3] === "outputs" && - !staticOutputKeys.includes(outputKey) && - !refStaticOutputKeys.includes(outputKey) - ) { - needsExecuted = true + if (outputType === "outputs") { + let refStaticOutputKeys: string[] = [] + if (refActionType) { + const refActionSpec = actionTypes[ref.kind][refActionType]?.spec + refStaticOutputKeys = refActionSpec?.staticOutputsSchema + ? describeSchema(refActionSpec.staticOutputsSchema).keys + : [] + } + + // Avoid execution when referencing the static output keys of the ref action type. + // e.g. a helm deploy referencing container build static output deploymentImageName + // ${actions.build.my-container.outputs.deploymentImageName} + if (!isString(outputKey)) { + needsExecuted = true + } else { + needsExecuted = !staticOutputKeys.includes(outputKey) && !refStaticOutputKeys.includes(outputKey) + } } const refWithType = { @@ -1062,7 +1073,11 @@ function dependenciesFromActionConfig({ type: refActionType, } - addDep(refWithType, { explicit: false, needsExecutedOutputs: needsExecuted, needsStaticOutputs: !needsExecuted }) + addDep(omit(refWithType, ["keyPath"]), { + explicit: false, + needsExecutedOutputs: needsExecuted, + needsStaticOutputs: !needsExecuted, + }) } return deps diff --git a/core/src/outputs.ts b/core/src/outputs.ts index b0ac73a98c..af2e627930 100644 --- a/core/src/outputs.ts +++ b/core/src/outputs.ts @@ -53,12 +53,9 @@ export async function resolveProjectOutputs(garden: Garden, log: Log): Promise GardenError) {} +} + +export function isUnresolvableValue( + val: CollectionOrValue +): val is UnresolvableValue { + return val instanceof UnresolvableValue +} + export type ContextLookupReferenceFinding = | { type: "resolvable" @@ -90,7 +100,7 @@ export type ContextLookupReferenceFinding = } | { type: "unresolvable" - keyPath: (string | number | { getError: () => GardenError })[] + keyPath: (string | number | UnresolvableValue)[] yamlSource: ConfigSource } @@ -126,8 +136,8 @@ export function* getContextLookupReferences( }) if (typeof key === "symbol") { isResolvable = false - return { - getError: captureError(() => + return new UnresolvableValue( + captureError(() => // this will throw an error, because the key could not be resolved keyPathExpression.evaluate({ context, @@ -135,8 +145,8 @@ export function* getContextLookupReferences( optional: false, yamlSource, }) - ), - } + ) + ) } return key }) diff --git a/core/src/template-string/template-string.ts b/core/src/template-string/template-string.ts index 671685f7fa..13e4447f03 100644 --- a/core/src/template-string/template-string.ts +++ b/core/src/template-string/template-string.ts @@ -16,7 +16,7 @@ import { NoOpContext, } from "../config/template-contexts/base.js" import cloneDeep from "fast-copy" -import { difference, isNumber, isPlainObject, isString } from "lodash-es" +import { difference, isPlainObject, isString } from "lodash-es" import type { ActionReference, Primitive, StringMap } from "../config/common.js" import { arrayConcatKey, @@ -44,8 +44,8 @@ import type { ObjectPath } from "../config/base.js" import type { TemplatePrimitive } from "./types.js" import * as ast from "./ast.js" import { LRUCache } from "lru-cache" -import type { ContextLookupReferenceFinding } from "./static-analysis.js" -import { getContextLookupReferences, visitAll } from "./static-analysis.js" +import type { ContextLookupReferenceFinding, UnresolvableValue } from "./static-analysis.js" +import { getContextLookupReferences, isUnresolvableValue, visitAll } from "./static-analysis.js" import type { ModuleConfig } from "../config/module.js" const escapePrefix = "$${" @@ -609,104 +609,111 @@ export function mayContainTemplateString(obj: any): boolean { } interface ActionTemplateReference extends ActionReference { - fullRef: ContextKeySegment[] + keyPath: (ContextKeySegment | UnresolvableValue)[] } export function extractActionReference(finding: ContextLookupReferenceFinding): ActionTemplateReference { - if (finding.type === "unresolvable") { - for (const k of finding.keyPath) { - if (typeof k === "object" && "getError" in k) { - const err = k.getError() - throw new ConfigurationError({ - message: `Found invalid action reference: ${err.message}`, - }) - } - } - throw new InternalError({ - message: "No error found in unresolvable finding", + const kind = finding.keyPath[1] + if (!kind) { + throw new ConfigurationError({ + message: `Found invalid action reference (missing kind).`, }) } - if (!finding.keyPath[1]) { + if (isUnresolvableValue(kind)) { + const err = kind.getError() throw new ConfigurationError({ - message: `Found invalid action reference (missing kind).`, + message: `Found invalid action reference: ${err.message}`, }) } - if (!isString(finding.keyPath[1])) { + if (!isString(kind)) { throw new ConfigurationError({ message: `Found invalid action reference (kind is not a string).`, }) } - if (!actionKindsLower.includes(finding.keyPath[1])) { + if (!actionKindsLower.includes(kind)) { throw new ConfigurationError({ - message: `Found invalid action reference (invalid kind '${finding.keyPath[1]}')`, + message: `Found invalid action reference (invalid kind '${kind}')`, }) } - if (!finding.keyPath[2]) { + const name = finding.keyPath[2] + if (!name) { throw new ConfigurationError({ message: "Found invalid action reference (missing name)", }) } - if (!isString(finding.keyPath[2])) { + if (isUnresolvableValue(name)) { + const err = name.getError() + throw new ConfigurationError({ + message: `Found invalid action reference: ${err.message}`, + }) + } + + if (!isString(name)) { throw new ConfigurationError({ message: "Found invalid action reference (name is not a string)", }) } return { - kind: titleize(finding.keyPath[1]), - name: finding.keyPath[2], - fullRef: finding.keyPath, + kind: titleize(kind), + name, + keyPath: finding.keyPath.slice(3), } } export function extractRuntimeReference(finding: ContextLookupReferenceFinding): ActionTemplateReference { - if (finding.type === "unresolvable") { - for (const k of finding.keyPath) { - if (typeof k === "object" && "getError" in k) { - const err = k.getError() - throw new ConfigurationError({ - message: `Found invalid action reference: ${err.message}`, - }) - } - } - throw new InternalError({ - message: "No error found in unresolvable finding", + const runtimeKind = finding.keyPath[1] + if (!runtimeKind) { + throw new ConfigurationError({ + message: "Found invalid runtime reference (missing kind)", }) } - if (!finding.keyPath[1]) { + if (isUnresolvableValue(runtimeKind)) { + const err = runtimeKind.getError() throw new ConfigurationError({ - message: "Found invalid runtime reference (missing kind)", + message: `Found invalid runtime reference: ${err.message}`, }) } - if (!isString(finding.keyPath[1])) { + + if (!isString(runtimeKind)) { throw new ConfigurationError({ message: "Found invalid runtime reference (kind is not a string)", }) } let kind: ActionKind - if (finding.keyPath[1] === "services") { + if (runtimeKind === "services") { kind = "Deploy" - } else if (finding.keyPath[1] === "tasks") { + } else if (runtimeKind === "tasks") { kind = "Run" } else { throw new ConfigurationError({ - message: `Found invalid runtime reference (invalid kind '${finding.keyPath[1]}')`, + message: `Found invalid runtime reference (invalid kind '${runtimeKind}')`, }) } - if (!finding.keyPath[2]) { + const name = finding.keyPath[2] + + if (!name) { throw new ConfigurationError({ message: `Found invalid runtime reference (missing name)`, }) } - if (!isString(finding.keyPath[2])) { + + if (isUnresolvableValue(name)) { + const err = name.getError() + throw new ConfigurationError({ + message: `Found invalid action reference: ${err.message}`, + }) + } + + if (!isString(name)) { throw new ConfigurationError({ message: "Found invalid runtime reference (name is not a string)", }) @@ -714,8 +721,8 @@ export function extractRuntimeReference(finding: ContextLookupReferenceFinding): return { kind, - name: finding.keyPath[2], - fullRef: finding.keyPath, + name, + keyPath: finding.keyPath.slice(3), } } @@ -773,7 +780,7 @@ export function getModuleTemplateReferences(config: ModuleConfig, context: Modul } const moduleName = keyPath[1] - if (!isString(moduleName) && !isNumber(moduleName)) { + if (isUnresolvableValue(moduleName)) { const err = moduleName.getError() throw new ConfigurationError({ message: `Found invalid module reference: ${err.message}`, diff --git a/core/src/util/testing.ts b/core/src/util/testing.ts index dd75c95997..0b7967521e 100644 --- a/core/src/util/testing.ts +++ b/core/src/util/testing.ts @@ -530,7 +530,17 @@ export function expectError(fn: Function, assertion: ExpectErrorAssertion = {}) expectFuzzyMatch( errorMessage, contains, - `Assertion failed: '${errorMessage}' does not contain '${contains}'.\n\nOriginal error: ${toGardenError(err).stack}` + dedent` + Expected + + '${stripAnsi(errorMessage).toLowerCase()}' + + to contain + + '${typeof contains === "string" ? stripAnsi(contains).toLowerCase() : contains.map(stripAnsi).map((s) => s.toLowerCase())}' + + Original error: + ${stripAnsi(toGardenError(err).stack || "")}` ) } diff --git a/core/test/unit/src/actions/action-configs-to-graph.ts b/core/test/unit/src/actions/action-configs-to-graph.ts index d0953cc41c..33a35fa8e4 100644 --- a/core/test/unit/src/actions/action-configs-to-graph.ts +++ b/core/test/unit/src/actions/action-configs-to-graph.ts @@ -341,7 +341,6 @@ describe("actionConfigsToGraph", () => { kind: "Build", type: "test", name: "foo", - fullRef: ["actions", "build", "foo", "version"], needsExecutedOutputs: false, needsStaticOutputs: true, }, @@ -391,7 +390,6 @@ describe("actionConfigsToGraph", () => { kind: "Build", type: "test", name: "foo", - fullRef: ["actions", "build", "foo", "outputs", "bar"], needsExecutedOutputs: true, needsStaticOutputs: false, }, @@ -441,7 +439,6 @@ describe("actionConfigsToGraph", () => { kind: "Build", type: "container", name: "foo", - fullRef: ["actions", "build", "foo", "outputs", "deploymentImageName"], needsExecutedOutputs: false, needsStaticOutputs: true, }, diff --git a/core/test/unit/src/template-string.ts b/core/test/unit/src/template-string.ts index fbfff43ed1..c4e047e1f4 100644 --- a/core/test/unit/src/template-string.ts +++ b/core/test/unit/src/template-string.ts @@ -2243,31 +2243,55 @@ describe("getActionTemplateReferences", () => { it("returns valid action references", () => { const config = { build: '${actions["build"].build-a}', + buildFoo: '${actions["build"].build-a.outputs.foo}', deploy: '${actions["deploy"].deploy-a}', + deployFoo: '${actions["deploy"].deploy-a.outputs.foo}', run: '${actions["run"].run-a}', + runFoo: '${actions["run"].run-a.outputs.foo}', test: '${actions["test"].test-a}', + testFoo: '${actions["test"].test-a.outputs.foo}', } const actionTemplateReferences = Array.from(getActionTemplateReferences(config as any, new TestContext({}))) expect(actionTemplateReferences).to.eql([ { kind: "Build", name: "build-a", - fullRef: ["actions", "build", "build-a"], + keyPath: [], + }, + { + kind: "Build", + name: "build-a", + keyPath: ["outputs", "foo"], + }, + { + kind: "Deploy", + name: "deploy-a", + keyPath: [], }, { kind: "Deploy", name: "deploy-a", - fullRef: ["actions", "deploy", "deploy-a"], + keyPath: ["outputs", "foo"], + }, + { + kind: "Run", + name: "run-a", + keyPath: [], }, { kind: "Run", name: "run-a", - fullRef: ["actions", "run", "run-a"], + keyPath: ["outputs", "foo"], }, { kind: "Test", name: "test-a", - fullRef: ["actions", "test", "test-a"], + keyPath: [], + }, + { + kind: "Test", + name: "test-a", + keyPath: ["outputs", "foo"], }, ]) }) @@ -2350,19 +2374,31 @@ describe("getActionTemplateReferences", () => { it("returns valid runtime references", () => { const config = { services: '${runtime["services"].service-a}', + servicesFoo: '${runtime["services"].service-a.outputs.foo}', tasks: '${runtime["tasks"].task-a}', + tasksFoo: '${runtime["tasks"].task-a.outputs.foo}', } const actionTemplateReferences = Array.from(getActionTemplateReferences(config as any, new TestContext({}))) expect(actionTemplateReferences).to.eql([ { kind: "Deploy", name: "service-a", - fullRef: ["runtime", "services", "service-a"], + keyPath: [], + }, + { + kind: "Deploy", + name: "service-a", + keyPath: ["outputs", "foo"], + }, + { + kind: "Run", + name: "task-a", + keyPath: [], }, { kind: "Run", name: "task-a", - fullRef: ["runtime", "tasks", "task-a"], + keyPath: ["outputs", "foo"], }, ]) }) @@ -2400,7 +2436,7 @@ describe("getActionTemplateReferences", () => { } void expectError(() => Array.from(getActionTemplateReferences(config as any, new TestContext({}))), { contains: - "found invalid action reference: invalid template string (${runtime[foo.bar].some-name}) at path foo: could not find key foo. available keys: (none).", + "found invalid runtime reference: invalid template string (${runtime[foo.bar].some-name}) at path foo: could not find key foo. available keys: (none).", }) }) From 930b04a43e08256b39a2cae0a9ae53b7ce2ff8c7 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Thu, 5 Dec 2024 21:24:35 +0100 Subject: [PATCH 37/43] improvement: minor improvements --- core/src/config/template-contexts/base.ts | 4 +-- core/src/util/testing.ts | 41 +++++++++-------------- 2 files changed, 18 insertions(+), 27 deletions(-) diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index de7f4653b8..6387c977f1 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -95,8 +95,7 @@ export abstract class ConfigContext { } resolve({ key, nodePath, opts }: ContextResolveParams): ContextResolveOutput { - const path = renderKeyPath(key) - const fullPath = renderKeyPath(nodePath.concat(key)) + const path = key.join(".") // if the key has previously been resolved, return it directly const resolved = this._resolvedValues[path] @@ -108,6 +107,7 @@ export abstract class ConfigContext { // TODO: freeze opts object instead of using shallow copy opts.stack = new Set(opts.stack || []) + const fullPath = nodePath.concat(key).join(".") if (opts.stack.has(fullPath)) { // Circular dependency error is critical, throwing here. throw new ContextResolveError({ diff --git a/core/src/util/testing.ts b/core/src/util/testing.ts index 0b7967521e..d1cd49ae14 100644 --- a/core/src/util/testing.ts +++ b/core/src/util/testing.ts @@ -460,28 +460,27 @@ export class TestGarden extends Garden { } } -export function expectFuzzyMatch(str: string, sample: string | string[], assertionMessage?: string) { +export function expectFuzzyMatch(str: string, sample: string | string[], extraMessage?: string) { const errorMessageNonAnsi = stripAnsi(str) const samples = typeof sample === "string" ? [sample] : sample const samplesNonAnsi = samples.map(stripAnsi) for (const s of samplesNonAnsi) { const actualErrorMsgLowercase = errorMessageNonAnsi.toLowerCase() const expectedErrorSample = s.toLowerCase() - try { - expect(actualErrorMsgLowercase, assertionMessage).to.contain(expectedErrorSample) - } catch (err) { - // eslint-disable-next-line no-console - console.log( - "Expected string", - "\n", - `"${actualErrorMsgLowercase}"`, - "\n", - "to contain string", - "\n", - `"${expectedErrorSample}"` - ) - throw err - } + + const assertionMessage = dedent` + Expected string + + '${actualErrorMsgLowercase}' + + to contain string + + '${expectedErrorSample}' + + ${extraMessage || ""} + ` + + expect(actualErrorMsgLowercase, assertionMessage).to.contain(expectedErrorSample) } } @@ -531,15 +530,7 @@ export function expectError(fn: Function, assertion: ExpectErrorAssertion = {}) errorMessage, contains, dedent` - Expected - - '${stripAnsi(errorMessage).toLowerCase()}' - - to contain - - '${typeof contains === "string" ? stripAnsi(contains).toLowerCase() : contains.map(stripAnsi).map((s) => s.toLowerCase())}' - - Original error: + \nOriginal error: ${stripAnsi(toGardenError(err).stack || "")}` ) } From 3397b0eb41a9daf8a33b924e3bf3422dd6335979 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Thu, 5 Dec 2024 23:06:22 +0100 Subject: [PATCH 38/43] test: additional tests --- core/test/unit/src/template-string.ts | 62 ++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/core/test/unit/src/template-string.ts b/core/test/unit/src/template-string.ts index c4e047e1f4..2d9c2cd39d 100644 --- a/core/test/unit/src/template-string.ts +++ b/core/test/unit/src/template-string.ts @@ -21,7 +21,11 @@ import stripAnsi from "strip-ansi" import { TemplateStringError } from "../../../src/exceptions.js" import repeat from "lodash-es/repeat.js" import type { ContextLookupReferenceFinding } from "../../../src/template-string/static-analysis.js" -import { getContextLookupReferences, visitAll } from "../../../src/template-string/static-analysis.js" +import { + getContextLookupReferences, + UnresolvableValue, + visitAll, +} from "../../../src/template-string/static-analysis.js" class TestContext extends ConfigContext { constructor(context) { @@ -2236,6 +2240,62 @@ describe("getContextLookupReferences", () => { ] expect(result).to.eql(expected) }) + + it("should handle keys with dots correctly", () => { + const obj = { + a: "some ${templated['key.with.dots']}", + b: "${more.stuff}", + c: "${keyThatIs[unresolvable]}", + } + const foundKeys = Array.from( + getContextLookupReferences( + visitAll({ + value: obj, + parseTemplateStrings: true, + source: { + path: [], + }, + }), + new NoOpContext() + ) + ) + + const unresolvable = foundKeys[3].keyPath[1] + + expect(unresolvable).to.be.instanceOf(UnresolvableValue) + + const expected: ContextLookupReferenceFinding[] = [ + { + type: "resolvable", + keyPath: ["templated", "key.with.dots"], + yamlSource: { + path: ["a"], + }, + }, + { + type: "resolvable", + keyPath: ["more", "stuff"], + yamlSource: { + path: ["b"], + }, + }, + { + type: "resolvable", + keyPath: ["unresolvable"], + yamlSource: { + path: ["c"], + }, + }, + { + type: "unresolvable", + keyPath: ["keyThatIs", unresolvable], + yamlSource: { + path: ["c"], + }, + }, + ] + expect(foundKeys).to.deep.equals(expected) + }) }) describe("getActionTemplateReferences", () => { From e2200466c051d7948767850bef4364446567364a Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Fri, 6 Dec 2024 00:40:08 +0100 Subject: [PATCH 39/43] refactor: remove NoopContext --- core/src/config/provider.ts | 14 +- core/src/config/template-contexts/base.ts | 6 - core/src/garden.ts | 22 +- core/src/tasks/resolve-provider.ts | 6 +- core/src/template-string/template-string.ts | 19 +- core/test/unit/src/config/provider.ts | 10 +- .../unit/src/config/template-contexts/base.ts | 66 +- core/test/unit/src/template-string.ts | 633 ++++++++++-------- 8 files changed, 429 insertions(+), 347 deletions(-) diff --git a/core/src/config/provider.ts b/core/src/config/provider.ts index 5711d6319e..889cc10215 100644 --- a/core/src/config/provider.ts +++ b/core/src/config/provider.ts @@ -29,8 +29,8 @@ import type { ActionState } from "../actions/types.js" import type { ValidResultType } from "../tasks/base.js" import { uuidv4 } from "../util/random.js" import { s } from "./zod.js" -import { NoOpContext } from "./template-contexts/base.js" import { getContextLookupReferences, visitAll } from "../template-string/static-analysis.js" +import type { ConfigContext } from "./template-contexts/base.js" // TODO: dedupe from the joi schema below export const baseProviderConfigSchemaZod = s.object({ @@ -165,18 +165,22 @@ export function providerFromConfig({ * Given a plugin and its provider config, return a list of dependency names based on declared dependencies, * as well as implicit dependencies based on template strings. */ -export function getAllProviderDependencyNames(plugin: GardenPluginSpec, config: GenericProviderConfig) { +export function getAllProviderDependencyNames( + plugin: GardenPluginSpec, + config: GenericProviderConfig, + context: ConfigContext +) { return uniq([ ...(plugin.dependencies || []).map((d) => d.name), ...(config.dependencies || []), - ...getProviderTemplateReferences(config), + ...getProviderTemplateReferences(config, context), ]).sort() } /** * Given a provider config, return implicit dependencies based on template strings. */ -export function getProviderTemplateReferences(config: GenericProviderConfig) { +export function getProviderTemplateReferences(config: GenericProviderConfig, context: ConfigContext) { const deps: string[] = [] const generator = getContextLookupReferences( @@ -188,7 +192,7 @@ export function getProviderTemplateReferences(config: GenericProviderConfig) { path: [], }, }), - new NoOpContext() + context ) for (const finding of generator) { const keyPath = finding.keyPath diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index 6387c977f1..557bb48cc2 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -254,12 +254,6 @@ export class GenericContext extends ConfigContext { } } -export class NoOpContext extends ConfigContext { - override resolve() { - return { resolved: CONTEXT_RESOLVE_KEY_AVAILABLE_LATER, partial: true } - } -} - export class EnvironmentContext extends ConfigContext { @schema( joi diff --git a/core/src/garden.ts b/core/src/garden.ts index d1ebb00487..70f23f4f00 100644 --- a/core/src/garden.ts +++ b/core/src/garden.ts @@ -784,7 +784,13 @@ export class Garden { providerNames = getNames(rawConfigs) } - throwOnMissingSecretKeys(rawConfigs, this.secrets, "Provider", log) + throwOnMissingSecretKeys( + rawConfigs, + new RemoteSourceConfigContext(this, this.variables), + this.secrets, + "Provider", + log + ) // As an optimization, we return immediately if all requested providers are already resolved const alreadyResolvedProviders = providerNames.map((name) => this.resolvedProviders[name]).filter(Boolean) @@ -820,7 +826,11 @@ export class Garden { validationGraph.addNode(plugin.name) - for (const dep of getAllProviderDependencyNames(plugin!, config!)) { + for (const dep of getAllProviderDependencyNames( + plugin!, + config!, + new RemoteSourceConfigContext(this, this.variables) + )) { validationGraph.addNode(dep) validationGraph.addDependency(plugin.name, dep) } @@ -1411,7 +1421,13 @@ export class Garden { const groupedResources = groupBy(allResources, "kind") for (const [kind, configs] of Object.entries(groupedResources)) { - throwOnMissingSecretKeys(configs, this.secrets, kind, this.log) + throwOnMissingSecretKeys( + configs, + new RemoteSourceConfigContext(this, this.variables), + this.secrets, + kind, + this.log + ) } let rawModuleConfigs = [...((groupedResources.Module as ModuleConfig[]) || [])] diff --git a/core/src/tasks/resolve-provider.ts b/core/src/tasks/resolve-provider.ts index 40535a3af2..11b5412f93 100644 --- a/core/src/tasks/resolve-provider.ts +++ b/core/src/tasks/resolve-provider.ts @@ -34,6 +34,7 @@ import type { Log } from "../logger/log-entry.js" import { styles } from "../logger/styles.js" import type { ObjectPath } from "../config/base.js" import fsExtra from "fs-extra" +import { RemoteSourceConfigContext } from "../config/template-contexts/project.js" const { readFile, writeFile, ensureDir } = fsExtra @@ -116,7 +117,10 @@ export class ResolveProviderTask extends BaseTask { const pluginDeps = this.plugin.dependencies const explicitDeps = (this.config.dependencies || []).map((name) => ({ name })) - const implicitDeps = getProviderTemplateReferences(this.config).map((name) => ({ name })) + const implicitDeps = getProviderTemplateReferences( + this.config, + new RemoteSourceConfigContext(this.garden, this.garden.variables) + ).map((name) => ({ name })) const allDeps = uniq([...pluginDeps, ...explicitDeps, ...implicitDeps]) const rawProviderConfigs = this.garden.getRawProviderConfigs() diff --git a/core/src/template-string/template-string.ts b/core/src/template-string/template-string.ts index 13e4447f03..3b41fc5a59 100644 --- a/core/src/template-string/template-string.ts +++ b/core/src/template-string/template-string.ts @@ -13,7 +13,6 @@ import { CONTEXT_RESOLVE_KEY_AVAILABLE_LATER, CONTEXT_RESOLVE_KEY_NOT_FOUND, GenericContext, - NoOpContext, } from "../config/template-contexts/base.js" import cloneDeep from "fast-copy" import { difference, isPlainObject, isString } from "lodash-es" @@ -803,10 +802,16 @@ export function getModuleTemplateReferences(config: ModuleConfig, context: Modul * * Prefix should be e.g. "Module" or "Provider" (used when generating error messages). */ -export function throwOnMissingSecretKeys(configs: ObjectWithName[], secrets: StringMap, prefix: string, log?: Log) { +export function throwOnMissingSecretKeys( + configs: ObjectWithName[], + context: ConfigContext, + secrets: StringMap, + prefix: string, + log?: Log +) { const allMissing: [string, ContextKeySegment[]][] = [] // [[key, missing keys]] for (const config of configs) { - const missing = detectMissingSecretKeys(config, secrets) + const missing = detectMissingSecretKeys(config, context, secrets) if (missing.length > 0) { allMissing.push([config.name, missing]) } @@ -853,7 +858,11 @@ export function throwOnMissingSecretKeys(configs: ObjectWithName[], secrets: Str * Collects template references to secrets in obj, and returns an array of any secret keys referenced in it that * aren't present (or have blank values) in the provided secrets map. */ -export function detectMissingSecretKeys(obj: ObjectWithName, secrets: StringMap): ContextKeySegment[] { +export function detectMissingSecretKeys( + obj: ObjectWithName, + context: ConfigContext, + secrets: StringMap +): ContextKeySegment[] { const referencedKeys: ContextKeySegment[] = [] const generator = getContextLookupReferences( visitAll({ @@ -864,7 +873,7 @@ export function detectMissingSecretKeys(obj: ObjectWithName, secrets: StringMap) path: [], }, }), - new NoOpContext() + context ) for (const finding of generator) { const keyPath = finding.keyPath diff --git a/core/test/unit/src/config/provider.ts b/core/test/unit/src/config/provider.ts index 6701c31637..80d009a391 100644 --- a/core/test/unit/src/config/provider.ts +++ b/core/test/unit/src/config/provider.ts @@ -11,6 +11,7 @@ import type { GenericProviderConfig } from "../../../../src/config/provider.js" import { getAllProviderDependencyNames } from "../../../../src/config/provider.js" import { expectError } from "../../../helpers.js" import { createGardenPlugin } from "../../../../src/plugin/plugin.js" +import { GenericContext } from "../../../../src/config/template-contexts/base.js" describe("getProviderDependencies", () => { const plugin = createGardenPlugin({ @@ -23,7 +24,10 @@ describe("getProviderDependencies", () => { someKey: "${providers.other-provider.foo}", anotherKey: "foo-${providers.another-provider.bar}", } - expect(getAllProviderDependencyNames(plugin, config)).to.eql(["another-provider", "other-provider"]) + expect(getAllProviderDependencyNames(plugin, config, new GenericContext({}))).to.eql([ + "another-provider", + "other-provider", + ]) }) it("should ignore template strings that don't reference providers", async () => { @@ -32,7 +36,7 @@ describe("getProviderDependencies", () => { someKey: "${providers.other-provider.foo}", anotherKey: "foo-${some.other.ref}", } - expect(getAllProviderDependencyNames(plugin, config)).to.eql(["other-provider"]) + expect(getAllProviderDependencyNames(plugin, config, new GenericContext({}))).to.eql(["other-provider"]) }) it("should throw on provider-scoped template strings without a provider name", async () => { @@ -41,7 +45,7 @@ describe("getProviderDependencies", () => { someKey: "${providers}", } - await expectError(() => getAllProviderDependencyNames(plugin, config), { + await expectError(() => getAllProviderDependencyNames(plugin, config, new GenericContext({})), { contains: "Invalid template key 'providers' in configuration for provider 'my-provider'. You must specify a provider name as well (e.g. \\${providers.my-provider}).", }) diff --git a/core/test/unit/src/config/template-contexts/base.ts b/core/test/unit/src/config/template-contexts/base.ts index 4a90215770..0e5998dc09 100644 --- a/core/test/unit/src/config/template-contexts/base.ts +++ b/core/test/unit/src/config/template-contexts/base.ts @@ -25,7 +25,7 @@ interface TestValues { } describe("ConfigContext", () => { - class TestContext extends ConfigContext { + class GenericContext extends ConfigContext { constructor(obj: TestValues, root?: ConfigContext) { super(root) this.addValues(obj) @@ -43,12 +43,12 @@ describe("ConfigContext", () => { } it("should resolve simple keys", async () => { - const c = new TestContext({ basic: "value" }) + const c = new GenericContext({ basic: "value" }) expect(resolveKey(c, ["basic"])).to.eql({ resolved: "value" }) }) it("should return CONTEXT_RESOLVE_KEY_NOT_FOUND for missing key", async () => { - const c = new TestContext({}) + const c = new GenericContext({}) const { resolved, getUnavailableReason: message } = resolveKey(c, ["basic"]) expect(resolved).to.be.equal(CONTEXT_RESOLVE_KEY_NOT_FOUND) expect(stripAnsi(message!())).to.include("Could not find key basic") @@ -56,14 +56,14 @@ describe("ConfigContext", () => { context("allowPartial=true", () => { it("should return CONTEXT_RESOLVE_KEY_AVAILABLE_LATER symbol on missing key", async () => { - const c = new TestContext({}) + const c = new GenericContext({}) const result = resolveKey(c, ["basic"], { allowPartial: true }) expect(result.resolved).to.eql(CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) }) it("should return CONTEXT_RESOLVE_KEY_AVAILABLE_LATER symbol on missing key on nested context", async () => { - const c = new TestContext({ - nested: new TestContext({ key: "value" }), + const c = new GenericContext({ + nested: new GenericContext({ key: "value" }), }) const result = resolveKey(c, ["nested", "bla"], { allowPartial: true }) expect(result.resolved).to.eql(CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) @@ -71,25 +71,25 @@ describe("ConfigContext", () => { }) it("should throw when looking for nested value on primitive", async () => { - const c = new TestContext({ basic: "value" }) + const c = new GenericContext({ basic: "value" }) await expectError(() => resolveKey(c, ["basic", "nested"]), "context-resolve") }) it("should resolve nested keys", async () => { - const c = new TestContext({ nested: { key: "value" } }) + const c = new GenericContext({ nested: { key: "value" } }) expect(resolveKey(c, ["nested", "key"])).eql({ resolved: "value" }) }) it("should resolve keys on nested contexts", async () => { - const c = new TestContext({ - nested: new TestContext({ key: "value" }), + const c = new GenericContext({ + nested: new GenericContext({ key: "value" }), }) expect(resolveKey(c, ["nested", "key"])).eql({ resolved: "value" }) }) it("should return CONTEXT_RESOLVE_KEY_NOT_FOUND for missing keys on nested context", async () => { - const c = new TestContext({ - nested: new TestContext({ key: "value" }), + const c = new GenericContext({ + nested: new GenericContext({ key: "value" }), }) const { resolved, getUnavailableReason: message } = resolveKey(c, ["basic", "bla"]) expect(resolved).to.be.equal(CONTEXT_RESOLVE_KEY_NOT_FOUND) @@ -97,21 +97,21 @@ describe("ConfigContext", () => { }) it("should resolve keys with value behind callable", async () => { - const c = new TestContext({ basic: () => "value" }) + const c = new GenericContext({ basic: () => "value" }) expect(resolveKey(c, ["basic"])).to.eql({ resolved: "value" }) }) it("should resolve keys on nested contexts where context is behind callable", async () => { - const c = new TestContext({ - nested: () => new TestContext({ key: "value" }), + const c = new GenericContext({ + nested: () => new GenericContext({ key: "value" }), }) expect(resolveKey(c, ["nested", "key"])).to.eql({ resolved: "value" }) }) it("should cache resolved values", async () => { // eslint-disable-next-line @typescript-eslint/no-explicit-any - const nested: any = new TestContext({ key: "value" }) - const c = new TestContext({ + const nested: any = new GenericContext({ key: "value" }) + const c = new GenericContext({ nested, }) resolveKey(c, ["nested", "key"]) @@ -122,8 +122,8 @@ describe("ConfigContext", () => { }) it("should throw if resolving a key that's already in the lookup stack", async () => { - const c = new TestContext({ - nested: new TestContext({ key: "value" }), + const c = new GenericContext({ + nested: new GenericContext({ key: "value" }), }) const key = ["nested", "key"] const stack = new Set([key.join(".")]) @@ -139,7 +139,7 @@ describe("ConfigContext", () => { } } - const c = new TestContext({ + const c = new GenericContext({ nested: new NestedContext(), }) await expectError(() => resolveKey(c, ["nested", "bla"]), "context-resolve") @@ -210,65 +210,65 @@ describe("ConfigContext", () => { }) it("should resolve template strings", async () => { - const c = new TestContext({ + const c = new GenericContext({ foo: "value", }) // eslint-disable-next-line @typescript-eslint/no-explicit-any - const nested: any = new TestContext({ key: "${foo}" }, c) + const nested: any = new GenericContext({ key: "${foo}" }, c) c.addValues({ nested }) expect(resolveKey(c, ["nested", "key"])).to.eql({ resolved: "value" }) }) it("should resolve template strings with nested context", async () => { - const c = new TestContext({ + const c = new GenericContext({ foo: "bar", }) // eslint-disable-next-line @typescript-eslint/no-explicit-any - const nested: any = new TestContext({ key: "${nested.foo}", foo: "value" }, c) + const nested: any = new GenericContext({ key: "${nested.foo}", foo: "value" }, c) c.addValues({ nested }) expect(resolveKey(c, ["nested", "key"])).to.eql({ resolved: "value" }) }) it("should detect a self-reference when resolving a template string", async () => { - const c = new TestContext({ key: "${key}" }) + const c = new GenericContext({ key: "${key}" }) await expectError(() => resolveKey(c, ["key"]), "template-string") }) it("should detect a nested self-reference when resolving a template string", async () => { - const c = new TestContext({ + const c = new GenericContext({ foo: "bar", }) - const nested = new TestContext({ key: "${nested.key}" }, c) + const nested = new GenericContext({ key: "${nested.key}" }, c) c.addValues({ nested }) await expectError(() => resolveKey(c, ["nested", "key"]), "template-string") }) it("should detect a circular reference when resolving a template string", async () => { - const c = new TestContext({ + const c = new GenericContext({ foo: "bar", }) // eslint-disable-next-line @typescript-eslint/no-explicit-any - const nested: any = new TestContext({ key: "${nested.foo}", foo: "${nested.key}" }, c) + const nested: any = new GenericContext({ key: "${nested.foo}", foo: "${nested.key}" }, c) c.addValues({ nested }) await expectError(() => resolveKey(c, ["nested", "key"]), "template-string") }) it("should detect a circular reference when resolving a nested template string", async () => { - const c = new TestContext({ + const c = new GenericContext({ foo: "bar", }) // eslint-disable-next-line @typescript-eslint/no-explicit-any - const nested: any = new TestContext({ key: "${nested.foo}", foo: "${'${nested.key}'}" }, c) + const nested: any = new GenericContext({ key: "${nested.foo}", foo: "${'${nested.key}'}" }, c) c.addValues({ nested }) await expectError(() => resolveKey(c, ["nested", "key"]), "template-string") }) it("should detect a circular reference when nested template string resolves to self", async () => { - const c = new TestContext({ + const c = new GenericContext({ foo: "bar", }) // eslint-disable-next-line @typescript-eslint/no-explicit-any - const nested: any = new TestContext({ key: "${'${nested.key}'}" }, c) + const nested: any = new GenericContext({ key: "${'${nested.key}'}" }, c) c.addValues({ nested }) await expectError(() => resolveKey(c, ["nested", "key"]), { contains: diff --git a/core/test/unit/src/template-string.ts b/core/test/unit/src/template-string.ts index 2d9c2cd39d..90fc82408d 100644 --- a/core/test/unit/src/template-string.ts +++ b/core/test/unit/src/template-string.ts @@ -13,9 +13,9 @@ import { throwOnMissingSecretKeys, getActionTemplateReferences, } from "../../../src/template-string/template-string.js" -import { ConfigContext, NoOpContext } from "../../../src/config/template-contexts/base.js" +import { GenericContext } from "../../../src/config/template-contexts/base.js" import type { TestGarden } from "../../helpers.js" -import { expectError, getDataDir, makeTestGarden } from "../../helpers.js" +import { expectError, expectFuzzyMatch, getDataDir, makeTestGarden } from "../../helpers.js" import { dedent } from "../../../src/util/string.js" import stripAnsi from "strip-ansi" import { TemplateStringError } from "../../../src/exceptions.js" @@ -27,46 +27,39 @@ import { visitAll, } from "../../../src/template-string/static-analysis.js" -class TestContext extends ConfigContext { - constructor(context) { - super() - Object.assign(this, context) - } -} - describe("resolveTemplateString", () => { it("should return a non-templated string unchanged", () => { - const res = resolveTemplateString({ string: "somestring", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "somestring", context: new GenericContext({}) }) expect(res).to.equal("somestring") }) it("should resolve a key with a dash in it", () => { - const res = resolveTemplateString({ string: "${some-key}", context: new TestContext({ "some-key": "value" }) }) + const res = resolveTemplateString({ string: "${some-key}", context: new GenericContext({ "some-key": "value" }) }) expect(res).to.equal("value") }) it("should resolve a nested key with a dash in it", () => { const res = resolveTemplateString({ string: "${ctx.some-key}", - context: new TestContext({ ctx: { "some-key": "value" } }), + context: new GenericContext({ ctx: { "some-key": "value" } }), }) expect(res).to.equal("value") }) it("should correctly resolve if ? suffix is present but value exists", () => { - const res = resolveTemplateString({ string: "${foo}?", context: new TestContext({ foo: "bar" }) }) + const res = resolveTemplateString({ string: "${foo}?", context: new GenericContext({ foo: "bar" }) }) expect(res).to.equal("bar") }) it("should allow undefined values if ? suffix is present", () => { - const res = resolveTemplateString({ string: "${foo}?", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${foo}?", context: new GenericContext({}) }) expect(res).to.equal(undefined) }) it("should pass optional string through if allowPartial=true", () => { const res = resolveTemplateString({ string: "${foo}?", - context: new TestContext({}), + context: new GenericContext({}), contextOpts: { allowPartial: true }, }) expect(res).to.equal("${foo}?") @@ -75,7 +68,7 @@ describe("resolveTemplateString", () => { it("should not crash when variable in a member expression cannot be resolved", () => { const res = resolveTemplateString({ string: '${actions.run["${inputs.deployableTarget}-dummy"].var}', - context: new TestContext({ + context: new GenericContext({ actions: { run: {}, }, @@ -86,14 +79,14 @@ describe("resolveTemplateString", () => { }) it("should support a string literal in a template string as a means to escape it", () => { - const res = resolveTemplateString({ string: "${'$'}{bar}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${'$'}{bar}", context: new GenericContext({}) }) expect(res).to.equal("${bar}") }) it("should pass through a template string with a double $$ prefix if allowPartial=true", () => { const res = resolveTemplateString({ string: "$${bar}", - context: new TestContext({}), + context: new GenericContext({}), contextOpts: { allowPartial: true }, }) expect(res).to.equal("$${bar}") @@ -102,7 +95,7 @@ describe("resolveTemplateString", () => { it("should allow unescaping a template string with a double $$ prefix", () => { const res = resolveTemplateString({ string: "$${bar}", - context: new TestContext({}), + context: new GenericContext({}), contextOpts: { unescape: true }, }) expect(res).to.equal("${bar}") @@ -111,7 +104,7 @@ describe("resolveTemplateString", () => { it("should unescape a template string with a double $$ prefix if allowPartial=false", () => { const res = resolveTemplateString({ string: "$${bar}", - context: new TestContext({}), + context: new GenericContext({}), contextOpts: { allowPartial: false }, }) expect(res).to.equal("${bar}") @@ -120,7 +113,7 @@ describe("resolveTemplateString", () => { it("should allow nesting escaped strings within normal strings", () => { const res = resolveTemplateString({ string: "${foo == 'yes' ? '$${bar}' : 'fail' }", - context: new TestContext({ foo: "yes" }), + context: new GenericContext({ foo: "yes" }), contextOpts: { unescape: true }, }) expect(res).to.equal("${bar}") @@ -138,7 +131,7 @@ describe("resolveTemplateString", () => { it("for standalone env vars", () => { const res = resolveTemplateString({ string: "$${env" + envFormat.delimiter + "TEST_ENV}", - context: new TestContext({}), + context: new GenericContext({}), contextOpts: { unescape: true }, }) expect(res).to.equal("${env" + envFormat.delimiter + "TEST_ENV}") @@ -147,7 +140,7 @@ describe("resolveTemplateString", () => { it("for env vars in argument lists", () => { const res = resolveTemplateString({ string: "foo $${env" + envFormat.delimiter + "TEST_ENV} bar", - context: new TestContext({}), + context: new GenericContext({}), contextOpts: { unescape: true }, }) expect(res).to.equal("foo ${env" + envFormat.delimiter + "TEST_ENV} bar") @@ -156,7 +149,7 @@ describe("resolveTemplateString", () => { it("for env vars that are parts of another strings", () => { const res = resolveTemplateString({ string: "${foo}-$${env" + envFormat.delimiter + "TEST_ENV}", - context: new TestContext({ foo: "foo" }), + context: new GenericContext({ foo: "foo" }), contextOpts: { unescape: true }, }) expect(res).to.equal("foo-${env" + envFormat.delimiter + "TEST_ENV}") @@ -168,51 +161,57 @@ describe("resolveTemplateString", () => { it("should allow mixing normal and escaped strings", () => { const res = resolveTemplateString({ string: "${foo}-and-$${var.nope}", - context: new TestContext({ foo: "yes" }), + context: new GenericContext({ foo: "yes" }), contextOpts: { unescape: true }, }) expect(res).to.equal("yes-and-${var.nope}") }) it("should interpolate a format string with a prefix", () => { - const res = resolveTemplateString({ string: "prefix-${some}", context: new TestContext({ some: "value" }) }) + const res = resolveTemplateString({ string: "prefix-${some}", context: new GenericContext({ some: "value" }) }) expect(res).to.equal("prefix-value") }) it("should interpolate a format string with a suffix", () => { - const res = resolveTemplateString({ string: "${some}-suffix", context: new TestContext({ some: "value" }) }) + const res = resolveTemplateString({ string: "${some}-suffix", context: new GenericContext({ some: "value" }) }) expect(res).to.equal("value-suffix") }) it("should interpolate a format string with a prefix and a suffix", () => { - const res = resolveTemplateString({ string: "prefix-${some}-suffix", context: new TestContext({ some: "value" }) }) + const res = resolveTemplateString({ + string: "prefix-${some}-suffix", + context: new GenericContext({ some: "value" }), + }) expect(res).to.equal("prefix-value-suffix") }) it("should interpolate an optional format string with a prefix and a suffix", () => { - const res = resolveTemplateString({ string: "prefix-${some}?-suffix", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "prefix-${some}?-suffix", context: new GenericContext({}) }) expect(res).to.equal("prefix--suffix") }) it("should interpolate a format string with a prefix with whitespace", () => { - const res = resolveTemplateString({ string: "prefix ${some}", context: new TestContext({ some: "value" }) }) + const res = resolveTemplateString({ string: "prefix ${some}", context: new GenericContext({ some: "value" }) }) expect(res).to.equal("prefix value") }) it("should interpolate a format string with a suffix with whitespace", () => { - const res = resolveTemplateString({ string: "${some} suffix", context: new TestContext({ some: "value" }) }) + const res = resolveTemplateString({ string: "${some} suffix", context: new GenericContext({ some: "value" }) }) expect(res).to.equal("value suffix") }) it("should correctly interpolate a format string with surrounding whitespace", () => { - const res = resolveTemplateString({ string: "prefix ${some} suffix", context: new TestContext({ some: "value" }) }) + const res = resolveTemplateString({ + string: "prefix ${some} suffix", + context: new GenericContext({ some: "value" }), + }) expect(res).to.equal("prefix value suffix") }) it("should handle a nested key", () => { const res = resolveTemplateString({ string: "${some.nested}", - context: new TestContext({ some: { nested: "value" } }), + context: new GenericContext({ some: { nested: "value" } }), }) expect(res).to.equal("value") }) @@ -220,18 +219,18 @@ describe("resolveTemplateString", () => { it("should handle multiple format strings", () => { const res = resolveTemplateString({ string: "prefix-${a}-${b}-suffix", - context: new TestContext({ a: "value", b: "other" }), + context: new GenericContext({ a: "value", b: "other" }), }) expect(res).to.equal("prefix-value-other-suffix") }) it("should handle consecutive format strings", () => { - const res = resolveTemplateString({ string: "${a}${b}", context: new TestContext({ a: "value", b: "other" }) }) + const res = resolveTemplateString({ string: "${a}${b}", context: new GenericContext({ a: "value", b: "other" }) }) expect(res).to.equal("valueother") }) it("should throw when a key is not found", () => { - void expectError(() => resolveTemplateString({ string: "${some}", context: new TestContext({}) }), { + void expectError(() => resolveTemplateString({ string: "${some}", context: new GenericContext({}) }), { contains: "Invalid template string (${some}): Could not find key some", }) }) @@ -242,7 +241,7 @@ describe("resolveTemplateString", () => { () => resolveTemplateString({ string: `\${some} ${veryLongString} template string`, - context: new TestContext({}), + context: new GenericContext({}), }), (err) => expect(err.message.length).to.be.lessThan(350) ) @@ -250,7 +249,7 @@ describe("resolveTemplateString", () => { it("should replace line breaks in template strings in error messages", () => { void expectError( - () => resolveTemplateString({ string: "${some}\nmulti\nline\nstring", context: new TestContext({}) }), + () => resolveTemplateString({ string: "${some}\nmulti\nline\nstring", context: new GenericContext({}) }), { contains: "Invalid template string (${some}\\nmulti\\nline\\nstring): Could not find key some", } @@ -258,14 +257,17 @@ describe("resolveTemplateString", () => { }) it("should throw when a nested key is not found", () => { - void expectError(() => resolveTemplateString({ string: "${some.other}", context: new TestContext({ some: {} }) }), { - contains: "Invalid template string (${some.other}): Could not find key other under some", - }) + void expectError( + () => resolveTemplateString({ string: "${some.other}", context: new GenericContext({ some: {} }) }), + { + contains: "Invalid template string (${some.other}): Could not find key other under some", + } + ) }) it("should throw with an incomplete template string", () => { try { - resolveTemplateString({ string: "${some", context: new TestContext({ some: {} }) }) + resolveTemplateString({ string: "${some", context: new GenericContext({ some: {} }) }) } catch (err) { if (!(err instanceof TemplateStringError)) { expect.fail("Expected TemplateStringError") @@ -280,82 +282,82 @@ describe("resolveTemplateString", () => { }) it("should throw on nested format strings", () => { - void expectError(() => resolveTemplateString({ string: "${resol${part}ed}", context: new TestContext({}) }), { + void expectError(() => resolveTemplateString({ string: "${resol${part}ed}", context: new GenericContext({}) }), { contains: "Invalid template string (${resol${part}ed}): Unable to parse as valid template string.", }) }) it("should handle a single-quoted string", () => { - const res = resolveTemplateString({ string: "${'foo'}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${'foo'}", context: new GenericContext({}) }) expect(res).to.equal("foo") }) it("should handle a numeric literal and return it directly", () => { - const res = resolveTemplateString({ string: "${123}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${123}", context: new GenericContext({}) }) expect(res).to.equal(123) }) it("should handle a boolean true literal and return it directly", () => { - const res = resolveTemplateString({ string: "${true}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${true}", context: new GenericContext({}) }) expect(res).to.equal(true) }) it("should handle a boolean false literal and return it directly", () => { - const res = resolveTemplateString({ string: "${false}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${false}", context: new GenericContext({}) }) expect(res).to.equal(false) }) it("should handle a null literal and return it directly", () => { - const res = resolveTemplateString({ string: "${null}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${null}", context: new GenericContext({}) }) expect(res).to.equal(null) }) it("should handle a numeric literal in a logical OR and return it directly", () => { - const res = resolveTemplateString({ string: "${a || 123}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${a || 123}", context: new GenericContext({}) }) expect(res).to.equal(123) }) it("should handle a boolean true literal in a logical OR and return it directly", () => { - const res = resolveTemplateString({ string: "${a || true}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${a || true}", context: new GenericContext({}) }) expect(res).to.equal(true) }) it("should handle a boolean false literal in a logical OR and return it directly", () => { - const res = resolveTemplateString({ string: "${a || false}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${a || false}", context: new GenericContext({}) }) expect(res).to.equal(false) }) it("should handle a null literal in a logical OR and return it directly", () => { - const res = resolveTemplateString({ string: "${a || null}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${a || null}", context: new GenericContext({}) }) expect(res).to.equal(null) }) it("should handle a double-quoted string", () => { - const res = resolveTemplateString({ string: '${"foo"}', context: new TestContext({}) }) + const res = resolveTemplateString({ string: '${"foo"}', context: new GenericContext({}) }) expect(res).to.equal("foo") }) it("should throw on invalid single-quoted string", () => { - void expectError(() => resolveTemplateString({ string: "${'foo}", context: new TestContext({}) }), { + void expectError(() => resolveTemplateString({ string: "${'foo}", context: new GenericContext({}) }), { contains: "Invalid template string (${'foo}): Unable to parse as valid template string.", }) }) it("should throw on invalid double-quoted string", () => { - void expectError(() => resolveTemplateString({ string: '${"foo}', context: new TestContext({}) }), { + void expectError(() => resolveTemplateString({ string: '${"foo}', context: new GenericContext({}) }), { contains: 'Invalid template string (${"foo}): Unable to parse as valid template string.', }) }) it("should handle a logical OR between two identifiers", () => { - const res = resolveTemplateString({ string: "${a || b}", context: new TestContext({ a: undefined, b: "abc" }) }) + const res = resolveTemplateString({ string: "${a || b}", context: new GenericContext({ a: undefined, b: "abc" }) }) expect(res).to.equal("abc") }) it("should handle a logical OR between two nested identifiers", () => { const res = resolveTemplateString({ string: "${a.b || c.d}", - context: new TestContext({ + context: new GenericContext({ a: { b: undefined }, c: { d: "abc" }, }), @@ -366,7 +368,7 @@ describe("resolveTemplateString", () => { it("should handle a logical OR between two nested identifiers where the first resolves", () => { const res = resolveTemplateString({ string: "${a.b || c.d}", - context: new TestContext({ + context: new GenericContext({ a: { b: "abc" }, c: { d: undefined }, }), @@ -375,92 +377,92 @@ describe("resolveTemplateString", () => { }) it("should handle a logical OR between two identifiers without spaces with first value undefined", () => { - const res = resolveTemplateString({ string: "${a||b}", context: new TestContext({ a: undefined, b: "abc" }) }) + const res = resolveTemplateString({ string: "${a||b}", context: new GenericContext({ a: undefined, b: "abc" }) }) expect(res).to.equal("abc") }) it("should handle a logical OR between two identifiers with first value undefined and string fallback", () => { - const res = resolveTemplateString({ string: '${a || "foo"}', context: new TestContext({ a: undefined }) }) + const res = resolveTemplateString({ string: '${a || "foo"}', context: new GenericContext({ a: undefined }) }) expect(res).to.equal("foo") }) it("should handle a logical OR with undefined nested value and string fallback", () => { - const res = resolveTemplateString({ string: "${a.b || 'foo'}", context: new TestContext({ a: {} }) }) + const res = resolveTemplateString({ string: "${a.b || 'foo'}", context: new GenericContext({ a: {} }) }) expect(res).to.equal("foo") }) it("should handle chained logical OR with string fallback", () => { const res = resolveTemplateString({ string: "${a.b || c.d || e.f || 'foo'}", - context: new TestContext({ a: {}, c: {}, e: {} }), + context: new GenericContext({ a: {}, c: {}, e: {} }), }) expect(res).to.equal("foo") }) it("should handle a logical OR between two identifiers without spaces with first value set", () => { - const res = resolveTemplateString({ string: "${a||b}", context: new TestContext({ a: "abc", b: undefined }) }) + const res = resolveTemplateString({ string: "${a||b}", context: new GenericContext({ a: "abc", b: undefined }) }) expect(res).to.equal("abc") }) it("should throw if neither key in logical OR is valid", () => { - void expectError(() => resolveTemplateString({ string: "${a || b}", context: new TestContext({}) }), { + void expectError(() => resolveTemplateString({ string: "${a || b}", context: new GenericContext({}) }), { contains: "Invalid template string (${a || b}): Could not find key b", }) }) it("should throw on invalid logical OR string", () => { - void expectError(() => resolveTemplateString({ string: "${a || 'b}", context: new TestContext({}) }), { + void expectError(() => resolveTemplateString({ string: "${a || 'b}", context: new GenericContext({}) }), { contains: "Invalid template string (${a || 'b}): Unable to parse as valid template string.", }) }) it("should handle a logical OR between a string and a string", () => { - const res = resolveTemplateString({ string: "${'a' || 'b'}", context: new TestContext({ a: undefined }) }) + const res = resolveTemplateString({ string: "${'a' || 'b'}", context: new GenericContext({ a: undefined }) }) expect(res).to.equal("a") }) it("should handle a logical OR between an empty string and a string", () => { - const res = resolveTemplateString({ string: "${a || 'b'}", context: new TestContext({ a: "" }) }) + const res = resolveTemplateString({ string: "${a || 'b'}", context: new GenericContext({ a: "" }) }) expect(res).to.equal("b") }) context("logical AND (&& operator)", () => { it("true literal and true variable reference", () => { - const res = resolveTemplateString({ string: "${true && a}", context: new TestContext({ a: true }) }) + const res = resolveTemplateString({ string: "${true && a}", context: new GenericContext({ a: true }) }) expect(res).to.equal(true) }) it("two true variable references", () => { const res = resolveTemplateString({ string: "${var.a && var.b}", - context: new TestContext({ var: { a: true, b: true } }), + context: new GenericContext({ var: { a: true, b: true } }), }) expect(res).to.equal(true) }) it("first part is false but the second part is not resolvable", () => { // i.e. the 2nd clause should not need to be evaluated - const res = resolveTemplateString({ string: "${false && a}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${false && a}", context: new GenericContext({}) }) expect(res).to.equal(false) }) it("an empty string as the first clause", () => { - const res = resolveTemplateString({ string: "${'' && true}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${'' && true}", context: new GenericContext({}) }) expect(res).to.equal("") }) it("an empty string as the second clause", () => { - const res = resolveTemplateString({ string: "${true && ''}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${true && ''}", context: new GenericContext({}) }) expect(res).to.equal("") }) it("a missing reference as the first clause", () => { - const res = resolveTemplateString({ string: "${var.foo && 'a'}", context: new TestContext({ var: {} }) }) + const res = resolveTemplateString({ string: "${var.foo && 'a'}", context: new GenericContext({ var: {} }) }) expect(res).to.equal(false) }) it("a missing reference as the second clause", () => { - const res = resolveTemplateString({ string: "${'a' && var.foo}", context: new TestContext({ var: {} }) }) + const res = resolveTemplateString({ string: "${'a' && var.foo}", context: new GenericContext({ var: {} }) }) expect(res).to.equal(false) }) @@ -468,7 +470,7 @@ describe("resolveTemplateString", () => { it("a missing reference as the first clause returns the original template", () => { const res = resolveTemplateString({ string: "${var.foo && 'a'}", - context: new TestContext({ var: {} }), + context: new GenericContext({ var: {} }), contextOpts: { allowPartial: true }, }) expect(res).to.equal("${var.foo && 'a'}") @@ -477,7 +479,7 @@ describe("resolveTemplateString", () => { it("a missing reference as the second clause returns the original template", () => { const res = resolveTemplateString({ string: "${'a' && var.foo}", - context: new TestContext({ var: {} }), + context: new GenericContext({ var: {} }), contextOpts: { allowPartial: true }, }) expect(res).to.equal("${'a' && var.foo}") @@ -493,7 +495,7 @@ describe("resolveTemplateString", () => { it("a missing reference as the first clause returns the original template", () => { const res = resolveTemplateString({ string: `$\{var.foo ${operator} 2}`, - context: new TestContext({ var: {} }), + context: new GenericContext({ var: {} }), contextOpts: { allowPartial: true }, }) expect(res).to.equal(`$\{var.foo ${operator} 2}`) @@ -502,7 +504,7 @@ describe("resolveTemplateString", () => { it("a missing reference as the second clause returns the original template", () => { const res = resolveTemplateString({ string: `$\{2 ${operator} var.foo}`, - context: new TestContext({ var: {} }), + context: new GenericContext({ var: {} }), contextOpts: { allowPartial: true }, }) expect(res).to.equal(`$\{2 ${operator} var.foo}`) @@ -518,7 +520,7 @@ describe("resolveTemplateString", () => { it(`a missing reference as the first clause returns the original template`, () => { const res = resolveTemplateString({ string: `$\{var.foo ${operator} '2'}`, - context: new TestContext({ var: {} }), + context: new GenericContext({ var: {} }), contextOpts: { allowPartial: true }, }) expect(res).to.equal(`$\{var.foo ${operator} '2'}`) @@ -527,7 +529,7 @@ describe("resolveTemplateString", () => { it("a missing reference as the second clause returns the original template", () => { const res = resolveTemplateString({ string: `$\{2 ${operator} var.foo}`, - context: new TestContext({ var: {} }), + context: new GenericContext({ var: {} }), contextOpts: { allowPartial: true }, }) expect(res).to.equal(`$\{2 ${operator} var.foo}`) @@ -538,154 +540,154 @@ describe("resolveTemplateString", () => { }) it("should handle a positive equality comparison between equal resolved values", () => { - const res = resolveTemplateString({ string: "${a == b}", context: new TestContext({ a: "a", b: "a" }) }) + const res = resolveTemplateString({ string: "${a == b}", context: new GenericContext({ a: "a", b: "a" }) }) expect(res).to.equal(true) }) it("should handle a positive equality comparison between equal string literals", () => { - const res = resolveTemplateString({ string: "${'a' == 'a'}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${'a' == 'a'}", context: new GenericContext({}) }) expect(res).to.equal(true) }) it("should handle a positive equality comparison between equal numeric literals", () => { - const res = resolveTemplateString({ string: "${123 == 123}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${123 == 123}", context: new GenericContext({}) }) expect(res).to.equal(true) }) it("should handle a positive equality comparison between equal boolean literals", () => { - const res = resolveTemplateString({ string: "${true == true}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${true == true}", context: new GenericContext({}) }) expect(res).to.equal(true) }) it("should handle a positive equality comparison between different resolved values", () => { - const res = resolveTemplateString({ string: "${a == b}", context: new TestContext({ a: "a", b: "b" }) }) + const res = resolveTemplateString({ string: "${a == b}", context: new GenericContext({ a: "a", b: "b" }) }) expect(res).to.equal(false) }) it("should handle a positive equality comparison between different string literals", () => { - const res = resolveTemplateString({ string: "${'a' == 'b'}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${'a' == 'b'}", context: new GenericContext({}) }) expect(res).to.equal(false) }) it("should handle a positive equality comparison between different numeric literals", () => { - const res = resolveTemplateString({ string: "${123 == 456}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${123 == 456}", context: new GenericContext({}) }) expect(res).to.equal(false) }) it("should handle a positive equality comparison between different boolean literals", () => { - const res = resolveTemplateString({ string: "${true == false}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${true == false}", context: new GenericContext({}) }) expect(res).to.equal(false) }) it("should handle a negative equality comparison between equal resolved values", () => { - const res = resolveTemplateString({ string: "${a != b}", context: new TestContext({ a: "a", b: "a" }) }) + const res = resolveTemplateString({ string: "${a != b}", context: new GenericContext({ a: "a", b: "a" }) }) expect(res).to.equal(false) }) it("should handle a negative equality comparison between equal string literals", () => { - const res = resolveTemplateString({ string: "${'a' != 'a'}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${'a' != 'a'}", context: new GenericContext({}) }) expect(res).to.equal(false) }) it("should handle a negative equality comparison between equal numeric literals", () => { - const res = resolveTemplateString({ string: "${123 != 123}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${123 != 123}", context: new GenericContext({}) }) expect(res).to.equal(false) }) it("should handle a negative equality comparison between equal boolean literals", () => { - const res = resolveTemplateString({ string: "${false != false}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${false != false}", context: new GenericContext({}) }) expect(res).to.equal(false) }) it("should handle a negative equality comparison between different resolved values", () => { - const res = resolveTemplateString({ string: "${a != b}", context: new TestContext({ a: "a", b: "b" }) }) + const res = resolveTemplateString({ string: "${a != b}", context: new GenericContext({ a: "a", b: "b" }) }) expect(res).to.equal(true) }) it("should handle a negative equality comparison between different string literals", () => { - const res = resolveTemplateString({ string: "${'a' != 'b'}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${'a' != 'b'}", context: new GenericContext({}) }) expect(res).to.equal(true) }) it("should handle a negative equality comparison between different numeric literals", () => { - const res = resolveTemplateString({ string: "${123 != 456}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${123 != 456}", context: new GenericContext({}) }) expect(res).to.equal(true) }) it("should handle a negative equality comparison between different boolean literals", () => { - const res = resolveTemplateString({ string: "${true != false}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${true != false}", context: new GenericContext({}) }) expect(res).to.equal(true) }) it("should handle a positive equality comparison between different value types", () => { - const res = resolveTemplateString({ string: "${true == 'foo'}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${true == 'foo'}", context: new GenericContext({}) }) expect(res).to.equal(false) }) it("should handle a negative equality comparison between different value types", () => { - const res = resolveTemplateString({ string: "${123 != false}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${123 != false}", context: new GenericContext({}) }) expect(res).to.equal(true) }) it("should handle negations on booleans", () => { - const res = resolveTemplateString({ string: "${!true}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${!true}", context: new GenericContext({}) }) expect(res).to.equal(false) }) it("should handle negations on nulls", () => { - const res = resolveTemplateString({ string: "${!null}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${!null}", context: new GenericContext({}) }) expect(res).to.equal(true) }) it("should handle negations on empty strings", () => { - const res = resolveTemplateString({ string: "${!''}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${!''}", context: new GenericContext({}) }) expect(res).to.equal(true) }) it("should handle negations on resolved keys", () => { - const res = resolveTemplateString({ string: "${!a}", context: new TestContext({ a: false }) }) + const res = resolveTemplateString({ string: "${!a}", context: new GenericContext({ a: false }) }) expect(res).to.equal(true) }) it("should handle the typeof operator for resolved booleans", () => { - const res = resolveTemplateString({ string: "${typeof a}", context: new TestContext({ a: false }) }) + const res = resolveTemplateString({ string: "${typeof a}", context: new GenericContext({ a: false }) }) expect(res).to.equal("boolean") }) it("should handle the typeof operator for resolved numbers", () => { - const res = resolveTemplateString({ string: "${typeof foo}", context: new TestContext({ foo: 1234 }) }) + const res = resolveTemplateString({ string: "${typeof foo}", context: new GenericContext({ foo: 1234 }) }) expect(res).to.equal("number") }) it("should handle the typeof operator for strings", () => { - const res = resolveTemplateString({ string: "${typeof 'foo'}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${typeof 'foo'}", context: new GenericContext({}) }) expect(res).to.equal("string") }) it("should throw when using comparison operators on missing keys", () => { - void expectError(() => resolveTemplateString({ string: "${a >= b}", context: new TestContext({ a: 123 }) }), { + void expectError(() => resolveTemplateString({ string: "${a >= b}", context: new GenericContext({ a: 123 }) }), { contains: "Invalid template string (${a >= b}): Could not find key b. Available keys: a.", }) }) it("should concatenate two arrays", () => { - const res = resolveTemplateString({ string: "${a + b}", context: new TestContext({ a: [1], b: [2, 3] }) }) + const res = resolveTemplateString({ string: "${a + b}", context: new GenericContext({ a: [1], b: [2, 3] }) }) expect(res).to.eql([1, 2, 3]) }) it("should concatenate two strings", () => { - const res = resolveTemplateString({ string: "${a + b}", context: new TestContext({ a: "foo", b: "bar" }) }) + const res = resolveTemplateString({ string: "${a + b}", context: new GenericContext({ a: "foo", b: "bar" }) }) expect(res).to.eql("foobar") }) it("should add two numbers together", () => { - const res = resolveTemplateString({ string: "${1 + a}", context: new TestContext({ a: 2 }) }) + const res = resolveTemplateString({ string: "${1 + a}", context: new GenericContext({ a: 2 }) }) expect(res).to.equal(3) }) it("should throw when using + on number and array", () => { void expectError( - () => resolveTemplateString({ string: "${a + b}", context: new TestContext({ a: 123, b: ["a"] }) }), + () => resolveTemplateString({ string: "${a + b}", context: new GenericContext({ a: 123, b: ["a"] }) }), { contains: "Invalid template string (${a + b}): Both terms need to be either arrays or strings or numbers for + operator (got number and object).", @@ -694,19 +696,19 @@ describe("resolveTemplateString", () => { }) it("should correctly evaluate clauses in parentheses", () => { - const res = resolveTemplateString({ string: "${(1 + 2) * (3 + 4)}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${(1 + 2) * (3 + 4)}", context: new GenericContext({}) }) expect(res).to.equal(21) }) it("should handle member lookup with bracket notation", () => { - const res = resolveTemplateString({ string: "${foo['bar']}", context: new TestContext({ foo: { bar: true } }) }) + const res = resolveTemplateString({ string: "${foo['bar']}", context: new GenericContext({ foo: { bar: true } }) }) expect(res).to.equal(true) }) it("should handle member lookup with bracket notation, single quotes and dot in key name", () => { const res = resolveTemplateString({ string: "${foo['bar.baz']}", - context: new TestContext({ foo: { "bar.baz": true } }), + context: new GenericContext({ foo: { "bar.baz": true } }), }) expect(res).to.equal(true) }) @@ -714,20 +716,20 @@ describe("resolveTemplateString", () => { it("should handle member lookup with bracket notation, double quotes and dot in key name", () => { const res = resolveTemplateString({ string: '${foo.bar["bla.ble"]}', - context: new TestContext({ foo: { bar: { "bla.ble": 123 } } }), + context: new GenericContext({ foo: { bar: { "bla.ble": 123 } } }), }) expect(res).to.equal(123) }) it("should handle numeric member lookup with bracket notation", () => { - const res = resolveTemplateString({ string: "${foo[1]}", context: new TestContext({ foo: [false, true] }) }) + const res = resolveTemplateString({ string: "${foo[1]}", context: new GenericContext({ foo: [false, true] }) }) expect(res).to.equal(true) }) it("should handle consecutive member lookups with bracket notation", () => { const res = resolveTemplateString({ string: "${foo['bar']['baz']}", - context: new TestContext({ foo: { bar: { baz: true } } }), + context: new GenericContext({ foo: { bar: { baz: true } } }), }) expect(res).to.equal(true) }) @@ -735,7 +737,7 @@ describe("resolveTemplateString", () => { it("should handle dot member after bracket member", () => { const res = resolveTemplateString({ string: "${foo['bar'].baz}", - context: new TestContext({ foo: { bar: { baz: true } } }), + context: new GenericContext({ foo: { bar: { baz: true } } }), }) expect(res).to.equal(true) }) @@ -743,7 +745,7 @@ describe("resolveTemplateString", () => { it("should handle template expression within brackets", () => { const res = resolveTemplateString({ string: "${foo['${bar}']}", - context: new TestContext({ + context: new GenericContext({ foo: { baz: true }, bar: "baz", }), @@ -754,7 +756,7 @@ describe("resolveTemplateString", () => { it("should handle identifiers within brackets", () => { const res = resolveTemplateString({ string: "${foo[bar]}", - context: new TestContext({ + context: new GenericContext({ foo: { baz: true }, bar: "baz", }), @@ -765,7 +767,7 @@ describe("resolveTemplateString", () => { it("should handle nested identifiers within brackets", () => { const res = resolveTemplateString({ string: "${foo[a.b]}", - context: new TestContext({ + context: new GenericContext({ foo: { baz: true }, a: { b: "baz" }, }), @@ -775,7 +777,7 @@ describe("resolveTemplateString", () => { it("should throw if bracket expression resolves to a non-primitive", () => { void expectError( - () => resolveTemplateString({ string: "${foo[bar]}", context: new TestContext({ foo: {}, bar: {} }) }), + () => resolveTemplateString({ string: "${foo[bar]}", context: new GenericContext({ foo: {}, bar: {} }) }), { contains: "Invalid template string (${foo[bar]}): Expression in brackets must resolve to a string or number (got object).", @@ -785,7 +787,7 @@ describe("resolveTemplateString", () => { it("should throw if attempting to index a primitive with brackets", () => { void expectError( - () => resolveTemplateString({ string: "${foo[bar]}", context: new TestContext({ foo: 123, bar: "baz" }) }), + () => resolveTemplateString({ string: "${foo[bar]}", context: new GenericContext({ foo: 123, bar: "baz" }) }), { contains: 'Invalid template string (${foo[bar]}): Attempted to look up key "baz" on a number.', } @@ -794,7 +796,7 @@ describe("resolveTemplateString", () => { it("should throw when using >= on non-numeric terms", () => { void expectError( - () => resolveTemplateString({ string: "${a >= b}", context: new TestContext({ a: 123, b: "foo" }) }), + () => resolveTemplateString({ string: "${a >= b}", context: new GenericContext({ a: 123, b: "foo" }) }), { contains: "Invalid template string (${a >= b}): Both terms need to be numbers for >= operator (got number and string).", @@ -803,19 +805,19 @@ describe("resolveTemplateString", () => { }) it("should handle a positive ternary expression", () => { - const res = resolveTemplateString({ string: "${foo ? true : false}", context: new TestContext({ foo: true }) }) + const res = resolveTemplateString({ string: "${foo ? true : false}", context: new GenericContext({ foo: true }) }) expect(res).to.equal(true) }) it("should handle a negative ternary expression", () => { - const res = resolveTemplateString({ string: "${foo ? true : false}", context: new TestContext({ foo: false }) }) + const res = resolveTemplateString({ string: "${foo ? true : false}", context: new GenericContext({ foo: false }) }) expect(res).to.equal(false) }) it("should handle a ternary expression with an expression as a test", () => { const res = resolveTemplateString({ string: "${foo == 'bar' ? a : b}", - context: new TestContext({ foo: "bar", a: true, b: false }), + context: new GenericContext({ foo: "bar", a: true, b: false }), }) expect(res).to.equal(true) }) @@ -823,7 +825,7 @@ describe("resolveTemplateString", () => { it("should ignore errors in a value not returned by a ternary", () => { const res = resolveTemplateString({ string: "${var.foo ? replace(var.foo, ' ', ',') : null}", - context: new TestContext({ var: {} }), + context: new GenericContext({ var: {} }), }) expect(res).to.equal(null) }) @@ -831,7 +833,7 @@ describe("resolveTemplateString", () => { it("should handle a ternary expression with an object as a test", () => { const res = resolveTemplateString({ string: "${a ? a.value : b}", - context: new TestContext({ a: { value: true }, b: false }), + context: new GenericContext({ a: { value: true }, b: false }), }) expect(res).to.equal(true) }) @@ -839,25 +841,28 @@ describe("resolveTemplateString", () => { it("should handle a ternary expression with template key values", () => { const res = resolveTemplateString({ string: "${foo == 'bar' ? '=${foo}' : b}", - context: new TestContext({ foo: "bar", a: true, b: false }), + context: new GenericContext({ foo: "bar", a: true, b: false }), }) expect(res).to.equal("=bar") }) it("should handle an expression in parentheses", () => { - const res = resolveTemplateString({ string: "${foo || (a > 5)}", context: new TestContext({ foo: false, a: 10 }) }) + const res = resolveTemplateString({ + string: "${foo || (a > 5)}", + context: new GenericContext({ foo: false, a: 10 }), + }) expect(res).to.equal(true) }) it("should handle numeric indices on arrays", () => { - const res = resolveTemplateString({ string: "${foo.1}", context: new TestContext({ foo: [false, true] }) }) + const res = resolveTemplateString({ string: "${foo.1}", context: new GenericContext({ foo: [false, true] }) }) expect(res).to.equal(true) }) it("should resolve keys on objects in arrays", () => { const res = resolveTemplateString({ string: "${foo.1.bar}", - context: new TestContext({ foo: [{}, { bar: true }] }), + context: new GenericContext({ foo: [{}, { bar: true }] }), }) expect(res).to.equal(true) }) @@ -867,7 +872,7 @@ describe("resolveTemplateString", () => { () => resolveTemplateString({ string: "${nested.missing}", - context: new TestContext({ nested: new TestContext({ foo: 123, bar: 456, baz: 789 }) }), + context: new GenericContext({ nested: new GenericContext({ foo: 123, bar: 456, baz: 789 }) }), }), { contains: @@ -881,7 +886,7 @@ describe("resolveTemplateString", () => { () => resolveTemplateString({ string: "${nested.missing}", - context: new TestContext({ nested: { foo: 123, bar: 456 } }), + context: new GenericContext({ nested: { foo: 123, bar: 456 } }), }), { contains: @@ -891,7 +896,7 @@ describe("resolveTemplateString", () => { }) it("should correctly propagate errors when resolving key on object in nested context", () => { - const c = new TestContext({ nested: new TestContext({ deeper: {} }) }) + const c = new GenericContext({ nested: new GenericContext({ deeper: {} }) }) void expectError(() => resolveTemplateString({ string: "${nested.deeper.missing}", context: c }), { contains: "Invalid template string (${nested.deeper.missing}): Could not find key missing under nested.deeper.", @@ -899,7 +904,7 @@ describe("resolveTemplateString", () => { }) it("should correctly propagate errors from deeply nested contexts", () => { - const c = new TestContext({ nested: new TestContext({ deeper: new TestContext({}) }) }) + const c = new GenericContext({ nested: new GenericContext({ deeper: new GenericContext({}) }) }) void expectError(() => resolveTemplateString({ string: "${nested.deeper.missing}", context: c }), { contains: "Invalid template string (${nested.deeper.missing}): Could not find key missing under nested.deeper.", @@ -910,7 +915,7 @@ describe("resolveTemplateString", () => { it("passes through template strings with missing key", () => { const res = resolveTemplateString({ string: "${a}", - context: new TestContext({}), + context: new GenericContext({}), contextOpts: { allowPartial: true }, }) expect(res).to.equal("${a}") @@ -919,7 +924,7 @@ describe("resolveTemplateString", () => { it("passes through a template string with a missing key in an optional clause", () => { const res = resolveTemplateString({ string: "${a || b}", - context: new TestContext({ b: 123 }), + context: new GenericContext({ b: 123 }), contextOpts: { allowPartial: true }, }) expect(res).to.equal("${a || b}") @@ -928,7 +933,7 @@ describe("resolveTemplateString", () => { it("passes through a template string with a missing key in a ternary", () => { const res = resolveTemplateString({ string: "${a ? b : 123}", - context: new TestContext({ b: 123 }), + context: new GenericContext({ b: 123 }), contextOpts: { allowPartial: true }, }) expect(res).to.equal("${a ? b : 123}") @@ -937,54 +942,54 @@ describe("resolveTemplateString", () => { context("when the template string is the full input string", () => { it("should return a resolved number directly", () => { - const res = resolveTemplateString({ string: "${a}", context: new TestContext({ a: 100 }) }) + const res = resolveTemplateString({ string: "${a}", context: new GenericContext({ a: 100 }) }) expect(res).to.equal(100) }) it("should return a resolved boolean true directly", () => { - const res = resolveTemplateString({ string: "${a}", context: new TestContext({ a: true }) }) + const res = resolveTemplateString({ string: "${a}", context: new GenericContext({ a: true }) }) expect(res).to.equal(true) }) it("should return a resolved boolean false directly", () => { - const res = resolveTemplateString({ string: "${a}", context: new TestContext({ a: false }) }) + const res = resolveTemplateString({ string: "${a}", context: new GenericContext({ a: false }) }) expect(res).to.equal(false) }) it("should return a resolved null directly", () => { - const res = resolveTemplateString({ string: "${a}", context: new TestContext({ a: null }) }) + const res = resolveTemplateString({ string: "${a}", context: new GenericContext({ a: null }) }) expect(res).to.equal(null) }) it("should return a resolved object directly", () => { - const res = resolveTemplateString({ string: "${a}", context: new TestContext({ a: { b: 123 } }) }) + const res = resolveTemplateString({ string: "${a}", context: new GenericContext({ a: { b: 123 } }) }) expect(res).to.eql({ b: 123 }) }) it("should return a resolved array directly", () => { - const res = resolveTemplateString({ string: "${a}", context: new TestContext({ a: [123] }) }) + const res = resolveTemplateString({ string: "${a}", context: new GenericContext({ a: [123] }) }) expect(res).to.eql([123]) }) }) context("when the template string is a part of a string", () => { it("should format a resolved number into the string", () => { - const res = resolveTemplateString({ string: "foo-${a}", context: new TestContext({ a: 100 }) }) + const res = resolveTemplateString({ string: "foo-${a}", context: new GenericContext({ a: 100 }) }) expect(res).to.equal("foo-100") }) it("should format a resolved boolean true into the string", () => { - const res = resolveTemplateString({ string: "foo-${a}", context: new TestContext({ a: true }) }) + const res = resolveTemplateString({ string: "foo-${a}", context: new GenericContext({ a: true }) }) expect(res).to.equal("foo-true") }) it("should format a resolved boolean false into the string", () => { - const res = resolveTemplateString({ string: "foo-${a}", context: new TestContext({ a: false }) }) + const res = resolveTemplateString({ string: "foo-${a}", context: new GenericContext({ a: false }) }) expect(res).to.equal("foo-false") }) it("should format a resolved null into the string", () => { - const res = resolveTemplateString({ string: "foo-${a}", context: new TestContext({ a: null }) }) + const res = resolveTemplateString({ string: "foo-${a}", context: new GenericContext({ a: null }) }) expect(res).to.equal("foo-null") }) @@ -992,7 +997,7 @@ describe("resolveTemplateString", () => { it("does not resolve template expressions when 'b' is missing in the context", () => { const res = resolveTemplateString({ string: "${a}-${b}", - context: new TestContext({ a: "foo" }), + context: new GenericContext({ a: "foo" }), contextOpts: { allowPartial: true }, }) expect(res).to.equal("${a}-${b}") @@ -1001,7 +1006,7 @@ describe("resolveTemplateString", () => { it("does not resolve template expressions when 'a' is missing in the context", () => { const res = resolveTemplateString({ string: "${a}-${b}", - context: new TestContext({ b: "foo" }), + context: new GenericContext({ b: "foo" }), contextOpts: { allowPartial: true }, }) expect(res).to.equal("${a}-${b}") @@ -1010,7 +1015,7 @@ describe("resolveTemplateString", () => { it("does not resolve template expressions when 'a' is missing in the context when evaluating a conditional expression", () => { const res = resolveTemplateString({ string: "${a || b}-${c}", - context: new TestContext({ b: 123, c: "foo" }), + context: new GenericContext({ b: 123, c: "foo" }), contextOpts: { allowPartial: true, }, @@ -1021,7 +1026,7 @@ describe("resolveTemplateString", () => { it("resolves template expressions when the context is fully available", () => { const res = resolveTemplateString({ string: "${a}-${b}", - context: new TestContext({ a: "foo", b: "bar" }), + context: new GenericContext({ a: "foo", b: "bar" }), contextOpts: { allowPartial: true }, }) expect(res).to.equal("foo-bar") @@ -1031,7 +1036,7 @@ describe("resolveTemplateString", () => { context("contains operator", () => { it("should throw when right-hand side is not a primitive", () => { - const c = new TestContext({ a: [1, 2], b: [3, 4] }) + const c = new GenericContext({ a: [1, 2], b: [3, 4] }) void expectError(() => resolveTemplateString({ string: "${a contains b}", context: c }), { contains: @@ -1040,7 +1045,7 @@ describe("resolveTemplateString", () => { }) it("should throw when left-hand side is not a string, array or object", () => { - const c = new TestContext({ a: "foo", b: null }) + const c = new GenericContext({ a: "foo", b: null }) void expectError(() => resolveTemplateString({ string: "${b contains a}", context: c }), { contains: @@ -1049,34 +1054,37 @@ describe("resolveTemplateString", () => { }) it("positive string literal contains string literal", () => { - const res = resolveTemplateString({ string: "${'foobar' contains 'foo'}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${'foobar' contains 'foo'}", context: new GenericContext({}) }) expect(res).to.equal(true) }) it("string literal contains string literal (negative)", () => { - const res = resolveTemplateString({ string: "${'blorg' contains 'blarg'}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${'blorg' contains 'blarg'}", context: new GenericContext({}) }) expect(res).to.equal(false) }) it("string literal contains string reference", () => { - const res = resolveTemplateString({ string: "${a contains 'foo'}", context: new TestContext({ a: "foobar" }) }) + const res = resolveTemplateString({ string: "${a contains 'foo'}", context: new GenericContext({ a: "foobar" }) }) expect(res).to.equal(true) }) it("string reference contains string literal (negative)", () => { - const res = resolveTemplateString({ string: "${a contains 'blarg'}", context: new TestContext({ a: "foobar" }) }) + const res = resolveTemplateString({ + string: "${a contains 'blarg'}", + context: new GenericContext({ a: "foobar" }), + }) expect(res).to.equal(false) }) it("string contains number", () => { - const res = resolveTemplateString({ string: "${a contains 0}", context: new TestContext({ a: "hmm-0" }) }) + const res = resolveTemplateString({ string: "${a contains 0}", context: new GenericContext({ a: "hmm-0" }) }) expect(res).to.equal(true) }) it("object contains string literal", () => { const res = resolveTemplateString({ string: "${a contains 'foo'}", - context: new TestContext({ a: { foo: 123 } }), + context: new GenericContext({ a: { foo: 123 } }), }) expect(res).to.equal(true) }) @@ -1084,7 +1092,7 @@ describe("resolveTemplateString", () => { it("object contains string literal (negative)", () => { const res = resolveTemplateString({ string: "${a contains 'bar'}", - context: new TestContext({ a: { foo: 123 } }), + context: new GenericContext({ a: { foo: 123 } }), }) expect(res).to.equal(false) }) @@ -1092,7 +1100,7 @@ describe("resolveTemplateString", () => { it("object contains string reference", () => { const res = resolveTemplateString({ string: "${a contains b}", - context: new TestContext({ a: { foo: 123 }, b: "foo" }), + context: new GenericContext({ a: { foo: 123 }, b: "foo" }), }) expect(res).to.equal(true) }) @@ -1100,20 +1108,23 @@ describe("resolveTemplateString", () => { it("object contains number reference", () => { const res = resolveTemplateString({ string: "${a contains b}", - context: new TestContext({ a: { 123: 456 }, b: 123 }), + context: new GenericContext({ a: { 123: 456 }, b: 123 }), }) expect(res).to.equal(true) }) it("object contains number literal", () => { - const res = resolveTemplateString({ string: "${a contains 123}", context: new TestContext({ a: { 123: 456 } }) }) + const res = resolveTemplateString({ + string: "${a contains 123}", + context: new GenericContext({ a: { 123: 456 } }), + }) expect(res).to.equal(true) }) it("array contains string reference", () => { const res = resolveTemplateString({ string: "${a contains b}", - context: new TestContext({ a: ["foo"], b: "foo" }), + context: new GenericContext({ a: ["foo"], b: "foo" }), }) expect(res).to.equal(true) }) @@ -1121,23 +1132,23 @@ describe("resolveTemplateString", () => { it("array contains string reference (negative)", () => { const res = resolveTemplateString({ string: "${a contains b}", - context: new TestContext({ a: ["foo"], b: "bar" }), + context: new GenericContext({ a: ["foo"], b: "bar" }), }) expect(res).to.equal(false) }) it("array contains string literal", () => { - const res = resolveTemplateString({ string: "${a contains 'foo'}", context: new TestContext({ a: ["foo"] }) }) + const res = resolveTemplateString({ string: "${a contains 'foo'}", context: new GenericContext({ a: ["foo"] }) }) expect(res).to.equal(true) }) it("array contains number", () => { - const res = resolveTemplateString({ string: "${a contains 1}", context: new TestContext({ a: [0, 1] }) }) + const res = resolveTemplateString({ string: "${a contains 1}", context: new GenericContext({ a: [0, 1] }) }) expect(res).to.equal(true) }) it("array contains numeric index (negative)", () => { - const res = resolveTemplateString({ string: "${a contains 1}", context: new TestContext({ a: [0] }) }) + const res = resolveTemplateString({ string: "${a contains 1}", context: new GenericContext({ a: [0] }) }) expect(res).to.equal(false) }) }) @@ -1146,7 +1157,7 @@ describe("resolveTemplateString", () => { it("single-line if block (positive)", () => { const res = resolveTemplateString({ string: "prefix ${if a}content ${endif}suffix", - context: new TestContext({ a: true }), + context: new GenericContext({ a: true }), }) expect(res).to.equal("prefix content suffix") }) @@ -1154,7 +1165,7 @@ describe("resolveTemplateString", () => { it("single-line if block (negative)", () => { const res = resolveTemplateString({ string: "prefix ${if a}content ${endif}suffix", - context: new TestContext({ a: false }), + context: new GenericContext({ a: false }), }) expect(res).to.equal("prefix suffix") }) @@ -1162,7 +1173,7 @@ describe("resolveTemplateString", () => { it("single-line if/else statement (positive)", () => { const res = resolveTemplateString({ string: "prefix ${if a == 123}content ${else}other ${endif}suffix", - context: new TestContext({ a: 123 }), + context: new GenericContext({ a: 123 }), }) expect(res).to.equal("prefix content suffix") }) @@ -1170,7 +1181,7 @@ describe("resolveTemplateString", () => { it("single-line if/else statement (negative)", () => { const res = resolveTemplateString({ string: "prefix ${if a}content ${else}other ${endif}suffix", - context: new TestContext({ a: false }), + context: new GenericContext({ a: false }), }) expect(res).to.equal("prefix other suffix") }) @@ -1178,7 +1189,7 @@ describe("resolveTemplateString", () => { it("multi-line if block (positive)", () => { const res = resolveTemplateString({ string: "prefix\n${if a}content\n${endif}suffix", - context: new TestContext({ a: true }), + context: new GenericContext({ a: true }), }) expect(res).to.equal(dedent` prefix @@ -1190,7 +1201,7 @@ describe("resolveTemplateString", () => { it("template string within if block", () => { const res = resolveTemplateString({ string: "prefix\n${if a}templated: ${b}\n${endif}suffix", - context: new TestContext({ a: true, b: "content" }), + context: new GenericContext({ a: true, b: "content" }), }) expect(res).to.equal(dedent` prefix @@ -1202,7 +1213,7 @@ describe("resolveTemplateString", () => { it("nested if block (both positive)", () => { const res = resolveTemplateString({ string: "prefix\n${if a}some ${if b}content\n${endif}${endif}suffix", - context: new TestContext({ a: true, b: true }), + context: new GenericContext({ a: true, b: true }), }) expect(res).to.equal(dedent` prefix @@ -1214,7 +1225,7 @@ describe("resolveTemplateString", () => { it("nested if block (outer negative)", () => { const res = resolveTemplateString({ string: "prefix\n${if a}some ${if b}content\n${endif}${endif}suffix", - context: new TestContext({ a: false, b: true }), + context: new GenericContext({ a: false, b: true }), }) expect(res).to.equal(dedent` prefix @@ -1225,7 +1236,7 @@ describe("resolveTemplateString", () => { it("nested if block (inner negative)", () => { const res = resolveTemplateString({ string: "prefix\n${if a}some\n${if b}content\n${endif}${endif}suffix", - context: new TestContext({ a: true, b: false }), + context: new GenericContext({ a: true, b: false }), }) expect(res).to.equal(dedent` prefix @@ -1237,7 +1248,7 @@ describe("resolveTemplateString", () => { it("if/else statement inside if block", () => { const res = resolveTemplateString({ string: "prefix\n${if a}some\n${if b}nope${else}content\n${endif}${endif}suffix", - context: new TestContext({ a: true, b: false }), + context: new GenericContext({ a: true, b: false }), }) expect(res).to.equal(dedent` prefix @@ -1250,7 +1261,7 @@ describe("resolveTemplateString", () => { it("if block inside if/else statement", () => { const res = resolveTemplateString({ string: "prefix\n${if a}some\n${if b}content\n${endif}${else}nope ${endif}suffix", - context: new TestContext({ a: true, b: false }), + context: new GenericContext({ a: true, b: false }), }) expect(res).to.equal(dedent` prefix @@ -1262,7 +1273,10 @@ describe("resolveTemplateString", () => { it("throws if an if block has an optional suffix", () => { void expectError( () => - resolveTemplateString({ string: "prefix ${if a}?content ${endif}", context: new TestContext({ a: true }) }), + resolveTemplateString({ + string: "prefix ${if a}?content ${endif}", + context: new GenericContext({ a: true }), + }), { contains: "Invalid template string (prefix ${if a}?content ${endif}): Cannot specify optional suffix in if-block.", @@ -1272,7 +1286,7 @@ describe("resolveTemplateString", () => { it("throws if an if block doesn't have a matching endif", () => { void expectError( - () => resolveTemplateString({ string: "prefix ${if a}content", context: new TestContext({ a: true }) }), + () => resolveTemplateString({ string: "prefix ${if a}content", context: new GenericContext({ a: true }) }), { contains: "Invalid template string (prefix ${if a}content): Missing ${endif} after ${if ...} block.", } @@ -1281,7 +1295,7 @@ describe("resolveTemplateString", () => { it("throws if an endif block doesn't have a matching if", () => { void expectError( - () => resolveTemplateString({ string: "prefix content ${endif}", context: new TestContext({ a: true }) }), + () => resolveTemplateString({ string: "prefix content ${endif}", context: new GenericContext({ a: true }) }), { contains: "Invalid template string (prefix content ${endif}): Found ${endif} block without a preceding ${if...} block.", @@ -1292,32 +1306,38 @@ describe("resolveTemplateString", () => { context("helper functions", () => { it("resolves a helper function with a string literal", () => { - const res = resolveTemplateString({ string: "${base64Encode('foo')}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${base64Encode('foo')}", context: new GenericContext({}) }) expect(res).to.equal("Zm9v") }) it("resolves a template string in a helper argument", () => { - const res = resolveTemplateString({ string: "${base64Encode('${a}')}", context: new TestContext({ a: "foo" }) }) + const res = resolveTemplateString({ + string: "${base64Encode('${a}')}", + context: new GenericContext({ a: "foo" }), + }) expect(res).to.equal("Zm9v") }) it("resolves a helper function with multiple arguments", () => { - const res = resolveTemplateString({ string: "${split('a,b,c', ',')}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${split('a,b,c', ',')}", context: new GenericContext({}) }) expect(res).to.eql(["a", "b", "c"]) }) it("resolves a helper function with a template key reference", () => { - const res = resolveTemplateString({ string: "${base64Encode(a)}", context: new TestContext({ a: "foo" }) }) + const res = resolveTemplateString({ string: "${base64Encode(a)}", context: new GenericContext({ a: "foo" }) }) expect(res).to.equal("Zm9v") }) it("generates a correct hash with a string literal from the sha256 helper function", () => { - const res = resolveTemplateString({ string: "${sha256('This Is A Test String')}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${sha256('This Is A Test String')}", + context: new GenericContext({}), + }) expect(res).to.equal("9a058284378d1cc6b4348aacb6ba847918376054b094bbe06eb5302defc52685") }) it("throws if an argument is missing", () => { - void expectError(() => resolveTemplateString({ string: "${base64Decode()}", context: new TestContext({}) }), { + void expectError(() => resolveTemplateString({ string: "${base64Decode()}", context: new GenericContext({}) }), { contains: "Invalid template string (${base64Decode()}): Missing argument 'string' (at index 0) for base64Decode helper function.", }) @@ -1325,7 +1345,7 @@ describe("resolveTemplateString", () => { it("throws if a wrong argument type is passed", () => { void expectError( - () => resolveTemplateString({ string: "${base64Decode(a)}", context: new TestContext({ a: 1234 }) }), + () => resolveTemplateString({ string: "${base64Decode(a)}", context: new GenericContext({ a: 1234 }) }), { contains: "Invalid template string (${base64Decode(a)}): Error validating argument 'string' for base64Decode helper function:\n\nvalue must be a string", @@ -1334,22 +1354,26 @@ describe("resolveTemplateString", () => { }) it("throws if the function can't be found", () => { - void expectError(() => resolveTemplateString({ string: "${floop('blop')}", context: new TestContext({}) }), { + void expectError(() => resolveTemplateString({ string: "${floop('blop')}", context: new GenericContext({}) }), { contains: "Invalid template string (${floop('blop')}): Could not find helper function 'floop'. Available helper functions:", }) }) it("throws if the function fails", () => { - void expectError(() => resolveTemplateString({ string: "${jsonDecode('{]}')}", context: new TestContext({}) }), { - contains: "Invalid template string (${jsonDecode('{]}')}): Error from helper function jsonDecode: SyntaxError", - }) + void expectError( + () => resolveTemplateString({ string: "${jsonDecode('{]}')}", context: new GenericContext({}) }), + { + contains: + "Invalid template string (${jsonDecode('{]}')}): Error from helper function jsonDecode: SyntaxError", + } + ) }) it("does not apply helper function on unresolved template string and returns string as-is, when allowPartial=true", () => { const res = resolveTemplateString({ string: "${base64Encode('${environment.namespace}')}", - context: new TestContext({}), + context: new GenericContext({}), contextOpts: { allowPartial: true, }, @@ -1360,7 +1384,7 @@ describe("resolveTemplateString", () => { it("does not apply helper function on unresolved template object and returns string as-is, when allowPartial=true", () => { const res = resolveTemplateString({ string: "${base64Encode(var.foo)}", - context: new TestContext({ foo: { $forEach: ["a", "b"], $return: "${item.value}" } }), + context: new GenericContext({ foo: { $forEach: ["a", "b"], $return: "${item.value}" } }), contextOpts: { allowPartial: true, }, @@ -1370,22 +1394,22 @@ describe("resolveTemplateString", () => { context("concat", () => { it("allows empty strings", () => { - const res = resolveTemplateString({ string: "${concat('', '')}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${concat('', '')}", context: new GenericContext({}) }) expect(res).to.equal("") }) context("throws when", () => { function expectArgTypeError({ template, - testContextVars = {}, + GenericContextVars = {}, errorMessage, }: { template: string - testContextVars?: object + GenericContextVars?: object errorMessage: string }) { void expectError( - () => resolveTemplateString({ string: template, context: new TestContext(testContextVars) }), + () => resolveTemplateString({ string: template, context: new GenericContext(GenericContextVars) }), { contains: `Invalid template string (\${concat(a, b)}): ${errorMessage}`, } @@ -1395,7 +1419,7 @@ describe("resolveTemplateString", () => { it("using an incompatible argument types (string and object)", () => { return expectArgTypeError({ template: "${concat(a, b)}", - testContextVars: { + GenericContextVars: { a: "123", b: ["a"], }, @@ -1407,7 +1431,7 @@ describe("resolveTemplateString", () => { it("using an unsupported argument types (number and object)", () => { return expectArgTypeError({ template: "${concat(a, b)}", - testContextVars: { + GenericContextVars: { a: 123, b: ["a"], }, @@ -1421,24 +1445,24 @@ describe("resolveTemplateString", () => { context("isEmpty", () => { context("allows nulls", () => { it("resolves null as 'true'", () => { - const res = resolveTemplateString({ string: "${isEmpty(null)}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${isEmpty(null)}", context: new GenericContext({}) }) expect(res).to.be.true }) it("resolves references to null as 'true'", () => { - const res = resolveTemplateString({ string: "${isEmpty(a)}", context: new TestContext({ a: null }) }) + const res = resolveTemplateString({ string: "${isEmpty(a)}", context: new GenericContext({ a: null }) }) expect(res).to.be.true }) }) context("allows empty strings", () => { it("resolves an empty string as 'true'", () => { - const res = resolveTemplateString({ string: "${isEmpty('')}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${isEmpty('')}", context: new GenericContext({}) }) expect(res).to.be.true }) it("resolves a reference to an empty string as 'true'", () => { - const res = resolveTemplateString({ string: "${isEmpty(a)}", context: new TestContext({ a: "" }) }) + const res = resolveTemplateString({ string: "${isEmpty(a)}", context: new GenericContext({ a: "" }) }) expect(res).to.be.true }) }) @@ -1448,7 +1472,7 @@ describe("resolveTemplateString", () => { it("allows numeric indices", () => { const res = resolveTemplateString({ string: "${slice(foo, 0, 3)}", - context: new TestContext({ foo: "abcdef" }), + context: new GenericContext({ foo: "abcdef" }), }) expect(res).to.equal("abc") }) @@ -1456,14 +1480,15 @@ describe("resolveTemplateString", () => { it("allows numeric strings as indices", () => { const res = resolveTemplateString({ string: "${slice(foo, '0', '3')}", - context: new TestContext({ foo: "abcdef" }), + context: new GenericContext({ foo: "abcdef" }), }) expect(res).to.equal("abc") }) it("throws on invalid string in the start index", () => { void expectError( - () => resolveTemplateString({ string: "${slice(foo, 'a', 3)}", context: new TestContext({ foo: "abcdef" }) }), + () => + resolveTemplateString({ string: "${slice(foo, 'a', 3)}", context: new GenericContext({ foo: "abcdef" }) }), { contains: `Invalid template string (\${slice(foo, 'a', 3)}): Error from helper function slice: Error: start index must be a number or a numeric string (got "a")`, } @@ -1472,7 +1497,8 @@ describe("resolveTemplateString", () => { it("throws on invalid string in the end index", () => { void expectError( - () => resolveTemplateString({ string: "${slice(foo, 0, 'b')}", context: new TestContext({ foo: "abcdef" }) }), + () => + resolveTemplateString({ string: "${slice(foo, 0, 'b')}", context: new GenericContext({ foo: "abcdef" }) }), { contains: `Invalid template string (\${slice(foo, 0, 'b')}): Error from helper function slice: Error: end index must be a number or a numeric string (got "b")`, } @@ -1483,43 +1509,46 @@ describe("resolveTemplateString", () => { context("array literals", () => { it("returns an empty array literal back", () => { - const res = resolveTemplateString({ string: "${[]}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${[]}", context: new GenericContext({}) }) expect(res).to.eql([]) }) it("returns an array literal of literals back", () => { const res = resolveTemplateString({ string: "${['foo', \"bar\", 123, true, false]}", - context: new TestContext({}), + context: new GenericContext({}), }) expect(res).to.eql(["foo", "bar", 123, true, false]) }) it("resolves a key in an array literal", () => { - const res = resolveTemplateString({ string: "${[foo]}", context: new TestContext({ foo: "bar" }) }) + const res = resolveTemplateString({ string: "${[foo]}", context: new GenericContext({ foo: "bar" }) }) expect(res).to.eql(["bar"]) }) it("resolves a nested key in an array literal", () => { - const res = resolveTemplateString({ string: "${[foo.bar]}", context: new TestContext({ foo: { bar: "baz" } }) }) + const res = resolveTemplateString({ + string: "${[foo.bar]}", + context: new GenericContext({ foo: { bar: "baz" } }), + }) expect(res).to.eql(["baz"]) }) it("calls a helper in an array literal", () => { const res = resolveTemplateString({ string: "${[foo, base64Encode('foo')]}", - context: new TestContext({ foo: "bar" }), + context: new GenericContext({ foo: "bar" }), }) expect(res).to.eql(["bar", "Zm9v"]) }) it("calls a helper with an array literal argument", () => { - const res = resolveTemplateString({ string: "${join(['foo', 'bar'], ',')}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${join(['foo', 'bar'], ',')}", context: new GenericContext({}) }) expect(res).to.eql("foo,bar") }) it("allows empty string separator in join helper function", () => { - const res = resolveTemplateString({ string: "${join(['foo', 'bar'], '')}", context: new TestContext({}) }) + const res = resolveTemplateString({ string: "${join(['foo', 'bar'], '')}", context: new GenericContext({}) }) expect(res).to.eql("foobar") }) }) @@ -1534,7 +1563,7 @@ describe("resolveTemplateStrings", () => { noTemplate: "at-all", }, } - const templateContext = new TestContext({ + const templateContext = new GenericContext({ key: "value", something: "else", }) @@ -1555,7 +1584,7 @@ describe("resolveTemplateStrings", () => { some: "${key}?", other: "${missing}?", } - const templateContext = new TestContext({ + const templateContext = new GenericContext({ key: "value", }) @@ -1573,7 +1602,7 @@ describe("resolveTemplateStrings", () => { b: "B", c: "c", } - const templateContext = new TestContext({}) + const templateContext = new GenericContext({}) const result = resolveTemplateStrings({ source: undefined, value: obj, context: templateContext }) @@ -1590,7 +1619,7 @@ describe("resolveTemplateStrings", () => { c: "c", $merge: { a: "a", b: "b" }, } - const templateContext = new TestContext({}) + const templateContext = new GenericContext({}) const result = resolveTemplateStrings({ source: undefined, value: obj, context: templateContext }) @@ -1607,7 +1636,7 @@ describe("resolveTemplateStrings", () => { b: "B", c: "c", } - const templateContext = new TestContext({ obj: { a: "a", b: "b" } }) + const templateContext = new GenericContext({ obj: { a: "a", b: "b" } }) const result = resolveTemplateStrings({ source: undefined, value: obj, context: templateContext }) @@ -1627,7 +1656,7 @@ describe("resolveTemplateStrings", () => { a: "a", }, } - const templateContext = new TestContext({ obj: { b: "b" } }) + const templateContext = new GenericContext({ obj: { b: "b" } }) const result = resolveTemplateStrings({ source: undefined, value: obj, context: templateContext }) @@ -1643,7 +1672,7 @@ describe("resolveTemplateStrings", () => { $merge: "${var.doesnotexist || var.obj}", c: "c", } - const templateContext = new TestContext({ var: { obj: { a: "a", b: "b" } } }) + const templateContext = new GenericContext({ var: { obj: { a: "a", b: "b" } } }) const result = resolveTemplateStrings({ value: obj, context: templateContext, source: undefined }) @@ -1664,7 +1693,7 @@ describe("resolveTemplateStrings", () => { }, }, } - const templateContext = new TestContext({ + const templateContext = new GenericContext({ inputs: { "merged-object": { $merge: "${var.empty || var.input-object}", @@ -1706,7 +1735,7 @@ describe("resolveTemplateStrings", () => { }, }, } - const templateContext = new TestContext({ + const templateContext = new GenericContext({ inputs: { "merged-object": { $merge: "${var.empty || var.input-object}", @@ -1735,7 +1764,7 @@ describe("resolveTemplateStrings", () => { $merge: "${var.doesnotexist}", c: "c", } - const templateContext = new TestContext({ var: { obj: { a: "a", b: "b" } } }) + const templateContext = new GenericContext({ var: { obj: { a: "a", b: "b" } } }) expect(() => resolveTemplateStrings({ value: obj, context: templateContext, source: undefined })).to.throw( "Invalid template string" @@ -1747,7 +1776,7 @@ describe("resolveTemplateStrings", () => { const obj = { foo: ["a", { $concat: ["b", "c"] }, "d"], } - const res = resolveTemplateStrings({ source: undefined, value: obj, context: new TestContext({}) }) + const res = resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({}) }) expect(res).to.eql({ foo: ["a", "b", "c", "d"], }) @@ -1760,7 +1789,7 @@ describe("resolveTemplateStrings", () => { const res = resolveTemplateStrings({ source: undefined, value: obj, - context: new TestContext({ foo: ["b", "c"] }), + context: new GenericContext({ foo: ["b", "c"] }), }) expect(res).to.eql({ foo: ["a", "b", "c", "d"], @@ -1774,7 +1803,7 @@ describe("resolveTemplateStrings", () => { const res = resolveTemplateStrings({ source: undefined, value: obj, - context: new TestContext({ foo: ["b", "c"] }), + context: new GenericContext({ foo: ["b", "c"] }), }) expect(res).to.eql({ foo: ["a", "b", "c", "d"], @@ -1786,9 +1815,12 @@ describe("resolveTemplateStrings", () => { foo: ["a", { $concat: "b" }, "d"], } - void expectError(() => resolveTemplateStrings({ source: undefined, value: obj, context: new TestContext({}) }), { - contains: "Value of $concat key must be (or resolve to) an array (got string)", - }) + void expectError( + () => resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({}) }), + { + contains: "Value of $concat key must be (or resolve to) an array (got string)", + } + ) }) it("throws if object with $concat key contains other keys as well", () => { @@ -1796,9 +1828,12 @@ describe("resolveTemplateStrings", () => { foo: ["a", { $concat: "b", nope: "nay", oops: "derp" }, "d"], } - void expectError(() => resolveTemplateStrings({ source: undefined, value: obj, context: new TestContext({}) }), { - contains: 'A list item with a $concat key cannot have any other keys (found "nope" and "oops")', - }) + void expectError( + () => resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({}) }), + { + contains: 'A list item with a $concat key cannot have any other keys (found "nope" and "oops")', + } + ) }) it("ignores if $concat value is not an array and allowPartial=true", () => { @@ -1808,7 +1843,7 @@ describe("resolveTemplateStrings", () => { const res = resolveTemplateStrings({ source: undefined, value: obj, - context: new TestContext({}), + context: new GenericContext({}), contextOpts: { allowPartial: true }, }) expect(res).to.eql({ @@ -1826,7 +1861,7 @@ describe("resolveTemplateStrings", () => { $else: 456, }, } - const res = resolveTemplateStrings({ source: undefined, value: obj, context: new TestContext({ foo: 1 }) }) + const res = resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({ foo: 1 }) }) expect(res).to.eql({ bar: 123 }) }) @@ -1838,7 +1873,7 @@ describe("resolveTemplateStrings", () => { $else: 456, }, } - const res = resolveTemplateStrings({ source: undefined, value: obj, context: new TestContext({ foo: 2 }) }) + const res = resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({ foo: 2 }) }) expect(res).to.eql({ bar: 456 }) }) @@ -1849,7 +1884,7 @@ describe("resolveTemplateStrings", () => { $then: 123, }, } - const res = resolveTemplateStrings({ source: undefined, value: obj, context: new TestContext({ foo: 2 }) }) + const res = resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({ foo: 2 }) }) expect(res).to.eql({ bar: undefined }) }) @@ -1864,7 +1899,7 @@ describe("resolveTemplateStrings", () => { const res = resolveTemplateStrings({ source: undefined, value: obj, - context: new TestContext({ foo: 2 }), + context: new GenericContext({ foo: 2 }), contextOpts: { allowPartial: true }, }) expect(res).to.eql(obj) @@ -1879,7 +1914,7 @@ describe("resolveTemplateStrings", () => { } void expectError( - () => resolveTemplateStrings({ source: undefined, value: obj, context: new TestContext({ foo: "bla" }) }), + () => resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({ foo: "bla" }) }), { contains: "Value of $if key must be (or resolve to) a boolean (got string)", } @@ -1894,7 +1929,7 @@ describe("resolveTemplateStrings", () => { } void expectError( - () => resolveTemplateStrings({ source: undefined, value: obj, context: new TestContext({ foo: 1 }) }), + () => resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({ foo: 1 }) }), { contains: "Missing $then field next to $if field", } @@ -1911,7 +1946,7 @@ describe("resolveTemplateStrings", () => { } void expectError( - () => resolveTemplateStrings({ source: undefined, value: obj, context: new TestContext({ foo: 1 }) }), + () => resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({ foo: 1 }) }), { contains: 'Found one or more unexpected keys on $if object: "foo"', } @@ -1927,7 +1962,7 @@ describe("resolveTemplateStrings", () => { $return: "foo", }, } - const res = resolveTemplateStrings({ source: undefined, value: obj, context: new TestContext({}) }) + const res = resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({}) }) expect(res).to.eql({ foo: ["foo", "foo", "foo"], }) @@ -1944,7 +1979,7 @@ describe("resolveTemplateStrings", () => { $return: "${item.key}: ${item.value}", }, } - const res = resolveTemplateStrings({ source: undefined, value: obj, context: new TestContext({}) }) + const res = resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({}) }) expect(res).to.eql({ foo: ["a: 1", "b: 2", "c: 3"], }) @@ -1958,9 +1993,12 @@ describe("resolveTemplateStrings", () => { }, } - void expectError(() => resolveTemplateStrings({ source: undefined, value: obj, context: new TestContext({}) }), { - contains: "Value of $forEach key must be (or resolve to) an array or mapping object (got string)", - }) + void expectError( + () => resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({}) }), + { + contains: "Value of $forEach key must be (or resolve to) an array or mapping object (got string)", + } + ) }) it("ignores the loop if the input isn't a list or object and allowPartial=true", () => { @@ -1973,7 +2011,7 @@ describe("resolveTemplateStrings", () => { const res = resolveTemplateStrings({ source: undefined, value: obj, - context: new TestContext({}), + context: new GenericContext({}), contextOpts: { allowPartial: true }, }) expect(res).to.eql(obj) @@ -1986,9 +2024,12 @@ describe("resolveTemplateStrings", () => { }, } - void expectError(() => resolveTemplateStrings({ source: undefined, value: obj, context: new TestContext({}) }), { - contains: "Missing $return field next to $forEach field.", - }) + void expectError( + () => resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({}) }), + { + contains: "Missing $return field next to $forEach field.", + } + ) }) it("throws if there are superfluous keys on the object", () => { @@ -2001,9 +2042,12 @@ describe("resolveTemplateStrings", () => { }, } - void expectError(() => resolveTemplateStrings({ source: undefined, value: obj, context: new TestContext({}) }), { - contains: 'Found one or more unexpected keys on $forEach object: "$concat" and "foo"', - }) + void expectError( + () => resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({}) }), + { + contains: 'Found one or more unexpected keys on $forEach object: "$concat" and "foo"', + } + ) }) it("exposes item.value and item.key when resolving the $return clause", () => { @@ -2016,7 +2060,7 @@ describe("resolveTemplateStrings", () => { const res = resolveTemplateStrings({ source: undefined, value: obj, - context: new TestContext({ foo: ["a", "b", "c"] }), + context: new GenericContext({ foo: ["a", "b", "c"] }), }) expect(res).to.eql({ foo: ["0: a", "1: b", "2: c"], @@ -2033,7 +2077,7 @@ describe("resolveTemplateStrings", () => { const res = resolveTemplateStrings({ source: undefined, value: obj, - context: new TestContext({ foo: ["a", "b", "c"] }), + context: new GenericContext({ foo: ["a", "b", "c"] }), }) expect(res).to.eql({ foo: ["a", "b", "c"], @@ -2051,7 +2095,7 @@ describe("resolveTemplateStrings", () => { const res = resolveTemplateStrings({ source: undefined, value: obj, - context: new TestContext({ foo: ["a", "b", "c"] }), + context: new GenericContext({ foo: ["a", "b", "c"] }), }) expect(res).to.eql({ foo: ["a", "c"], @@ -2067,9 +2111,12 @@ describe("resolveTemplateStrings", () => { }, } - void expectError(() => resolveTemplateStrings({ source: undefined, value: obj, context: new TestContext({}) }), { - contains: "$filter clause in $forEach loop must resolve to a boolean value (got object)", - }) + void expectError( + () => resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({}) }), + { + contains: "$filter clause in $forEach loop must resolve to a boolean value (got object)", + } + ) }) it("handles $concat clauses in $return", () => { @@ -2081,7 +2128,7 @@ describe("resolveTemplateStrings", () => { }, }, } - const res = resolveTemplateStrings({ source: undefined, value: obj, context: new TestContext({}) }) + const res = resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({}) }) expect(res).to.eql({ foo: ["a-1", "a-2", "b-1", "b-2", "c-1", "c-2"], }) @@ -2100,7 +2147,7 @@ describe("resolveTemplateStrings", () => { }, }, } - const res = resolveTemplateStrings({ source: undefined, value: obj, context: new TestContext({}) }) + const res = resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({}) }) expect(res).to.eql({ foo: [ ["A1", "A2"], @@ -2116,7 +2163,7 @@ describe("resolveTemplateStrings", () => { $return: "foo", }, } - const res = resolveTemplateStrings({ source: undefined, value: obj, context: new TestContext({}) }) + const res = resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({}) }) expect(res).to.eql({ foo: [], }) @@ -2154,7 +2201,7 @@ describe("resolveTemplateStrings", () => { }, } - const res = resolveTemplateStrings({ source: undefined, value: obj, context: new TestContext({ services }) }) + const res = resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({ services }) }) expect(res).to.eql({ services: [ { @@ -2198,7 +2245,7 @@ describe("getContextLookupReferences", () => { path: [], }, }), - new NoOpContext() + new GenericContext({}) ) ) const expected: ContextLookupReferenceFinding[] = [ @@ -2241,7 +2288,7 @@ describe("getContextLookupReferences", () => { expect(result).to.eql(expected) }) - it("should handle keys with dots correctly", () => { + it("should handle keys with dots correctly", async () => { const obj = { a: "some ${templated['key.with.dots']}", b: "${more.stuff}", @@ -2256,13 +2303,17 @@ describe("getContextLookupReferences", () => { path: [], }, }), - new NoOpContext() + new GenericContext({}) ) ) - const unresolvable = foundKeys[3].keyPath[1] + const unresolvable = foundKeys[3].keyPath[1] as UnresolvableValue expect(unresolvable).to.be.instanceOf(UnresolvableValue) + expectFuzzyMatch( + unresolvable.getError().message, + "invalid template string (${keythatis[unresolvable]}) at path c: could not find key unresolvable. available keys: (none)." + ) const expected: ContextLookupReferenceFinding[] = [ { @@ -2311,7 +2362,7 @@ describe("getActionTemplateReferences", () => { test: '${actions["test"].test-a}', testFoo: '${actions["test"].test-a.outputs.foo}', } - const actionTemplateReferences = Array.from(getActionTemplateReferences(config as any, new TestContext({}))) + const actionTemplateReferences = Array.from(getActionTemplateReferences(config as any, new GenericContext({}))) expect(actionTemplateReferences).to.eql([ { kind: "Build", @@ -2360,7 +2411,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: "${actions}", } - void expectError(() => Array.from(getActionTemplateReferences(config as any, new TestContext({}))), { + void expectError(() => Array.from(getActionTemplateReferences(config as any, new GenericContext({}))), { contains: "Found invalid action reference (missing kind)", }) }) @@ -2369,7 +2420,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: '${actions["badkind"].some-name}', } - void expectError(() => Array.from(getActionTemplateReferences(config as any, new TestContext({}))), { + void expectError(() => Array.from(getActionTemplateReferences(config as any, new GenericContext({}))), { contains: "Found invalid action reference (invalid kind 'badkind')", }) }) @@ -2378,7 +2429,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: "${actions[123]}", } - void expectError(() => Array.from(getActionTemplateReferences(config as any, new TestContext({}))), { + void expectError(() => Array.from(getActionTemplateReferences(config as any, new GenericContext({}))), { contains: "Found invalid action reference (kind is not a string)", }) }) @@ -2387,7 +2438,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: "${actions[foo.bar].some-name}", } - void expectError(() => Array.from(getActionTemplateReferences(config as any, new TestContext({}))), { + void expectError(() => Array.from(getActionTemplateReferences(config as any, new GenericContext({}))), { contains: "found invalid action reference: invalid template string (${actions[foo.bar].some-name}) at path foo: could not find key foo. available keys: (none)", }) @@ -2397,7 +2448,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: '${actions[foo.bar || "hello"].some-name}', } - void expectError(() => Array.from(getActionTemplateReferences(config as any, new TestContext({}))), { + void expectError(() => Array.from(getActionTemplateReferences(config as any, new GenericContext({}))), { contains: "found invalid action reference (invalid kind 'hello')", }) }) @@ -2406,7 +2457,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: '${actions["build"]}', } - void expectError(() => Array.from(getActionTemplateReferences(config as any, new TestContext({}))), { + void expectError(() => Array.from(getActionTemplateReferences(config as any, new GenericContext({}))), { contains: "Found invalid action reference (missing name)", }) }) @@ -2415,7 +2466,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: '${actions["build"].123}', } - void expectError(() => Array.from(getActionTemplateReferences(config as any, new TestContext({}))), { + void expectError(() => Array.from(getActionTemplateReferences(config as any, new GenericContext({}))), { contains: "Found invalid action reference (name is not a string)", }) }) @@ -2424,7 +2475,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: '${actions["build"][123]}', } - void expectError(() => Array.from(getActionTemplateReferences(config as any, new TestContext({}))), { + void expectError(() => Array.from(getActionTemplateReferences(config as any, new GenericContext({}))), { contains: "Found invalid action reference (name is not a string)", }) }) @@ -2438,7 +2489,7 @@ describe("getActionTemplateReferences", () => { tasks: '${runtime["tasks"].task-a}', tasksFoo: '${runtime["tasks"].task-a.outputs.foo}', } - const actionTemplateReferences = Array.from(getActionTemplateReferences(config as any, new TestContext({}))) + const actionTemplateReferences = Array.from(getActionTemplateReferences(config as any, new GenericContext({}))) expect(actionTemplateReferences).to.eql([ { kind: "Deploy", @@ -2467,7 +2518,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: "${runtime}", } - void expectError(() => Array.from(getActionTemplateReferences(config as any, new TestContext({}))), { + void expectError(() => Array.from(getActionTemplateReferences(config as any, new GenericContext({}))), { contains: "Found invalid runtime reference (missing kind)", }) }) @@ -2476,7 +2527,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: '${runtime["badkind"].some-name}', } - void expectError(() => Array.from(getActionTemplateReferences(config as any, new TestContext({}))), { + void expectError(() => Array.from(getActionTemplateReferences(config as any, new GenericContext({}))), { contains: "Found invalid runtime reference (invalid kind 'badkind')", }) }) @@ -2485,7 +2536,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: "${runtime[123]}", } - void expectError(() => Array.from(getActionTemplateReferences(config as any, new TestContext({}))), { + void expectError(() => Array.from(getActionTemplateReferences(config as any, new GenericContext({}))), { contains: "Found invalid runtime reference (kind is not a string)", }) }) @@ -2494,7 +2545,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: "${runtime[foo.bar].some-name}", } - void expectError(() => Array.from(getActionTemplateReferences(config as any, new TestContext({}))), { + void expectError(() => Array.from(getActionTemplateReferences(config as any, new GenericContext({}))), { contains: "found invalid runtime reference: invalid template string (${runtime[foo.bar].some-name}) at path foo: could not find key foo. available keys: (none).", }) @@ -2504,7 +2555,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: '${runtime["tasks"]}', } - void expectError(() => Array.from(getActionTemplateReferences(config as any, new TestContext({}))), { + void expectError(() => Array.from(getActionTemplateReferences(config as any, new GenericContext({}))), { contains: "Found invalid runtime reference (missing name)", }) }) @@ -2513,7 +2564,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: '${runtime["tasks"].123}', } - void expectError(() => Array.from(getActionTemplateReferences(config as any, new TestContext({}))), { + void expectError(() => Array.from(getActionTemplateReferences(config as any, new GenericContext({}))), { contains: "Found invalid runtime reference (name is not a string)", }) }) @@ -2530,8 +2581,8 @@ describe.skip("throwOnMissingSecretKeys", () => { }, ] - throwOnMissingSecretKeys(configs, {}, "Module") - throwOnMissingSecretKeys(configs, { someSecret: "123" }, "Module") + throwOnMissingSecretKeys(configs, new GenericContext({}), {}, "Module") + throwOnMissingSecretKeys(configs, new GenericContext({}), { someSecret: "123" }, "Module") }) it("should throw an error if one or more secrets is missing", () => { @@ -2550,7 +2601,7 @@ describe.skip("throwOnMissingSecretKeys", () => { ] void expectError( - () => throwOnMissingSecretKeys(configs, { b: "123" }, "Module"), + () => throwOnMissingSecretKeys(configs, new GenericContext({}), { b: "123" }, "Module"), (err) => { expect(err.message).to.match(/Module moduleA: a/) expect(err.message).to.match(/Module moduleB: a, c/) @@ -2559,7 +2610,7 @@ describe.skip("throwOnMissingSecretKeys", () => { ) void expectError( - () => throwOnMissingSecretKeys(configs, {}, "Module"), + () => throwOnMissingSecretKeys(configs, new GenericContext({}), {}, "Module"), (err) => { expect(err.message).to.match(/Module moduleA: a, b/) expect(err.message).to.match(/Module moduleB: a, b, c/) From 3a71187ae65a08783decbd00ef7852af9dda51df Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Fri, 6 Dec 2024 08:30:26 +0100 Subject: [PATCH 40/43] fix: handle further edge cases in `getContextLookupReferences` --- core/src/config/template-contexts/base.ts | 2 - core/src/resolve-module.ts | 2 +- core/src/template-string/ast.ts | 128 ++++++-------------- core/src/template-string/static-analysis.ts | 2 +- core/src/template-string/template-string.ts | 18 +-- core/test/unit/src/template-string.ts | 99 +++++++++++++-- 6 files changed, 138 insertions(+), 113 deletions(-) diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index 557bb48cc2..5675c9c247 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -219,8 +219,6 @@ export abstract class ConfigContext { return { resolved: value, getUnavailableReason } } - // If we're allowing partial strings, we throw the error immediately to end the resolution flow. The error - // is caught in the surrounding template resolution code. if (this._alwaysAllowPartial || opts.allowPartial) { return { resolved: CONTEXT_RESOLVE_KEY_AVAILABLE_LATER, diff --git a/core/src/resolve-module.ts b/core/src/resolve-module.ts index a57870d32b..caad7a6e16 100644 --- a/core/src/resolve-module.ts +++ b/core/src/resolve-module.ts @@ -661,7 +661,7 @@ export class ModuleResolver { value: { ...config, inputs: {}, variables: {} }, context: configContext, contextOpts: { - allowPartial: false, + allowPartial: true, // Modules will be converted to actions later, and the actions will be properly unescaped. // We avoid premature un-escaping here, // because otherwise it will strip the escaped value in the module config diff --git a/core/src/template-string/ast.ts b/core/src/template-string/ast.ts index 9af51dc1e6..db9c41947a 100644 --- a/core/src/template-string/ast.ts +++ b/core/src/template-string/ast.ts @@ -54,8 +54,8 @@ export type Location = { source: TemplateStringSource } -export type TemplateEvaluationResult = - | TemplatePrimitive +export type TemplateEvaluationResult = + | T | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER @@ -92,12 +92,7 @@ export abstract class TemplateExpression { yield* astVisitAll(this, source) } - abstract evaluate( - args: EvaluateArgs - ): - | CollectionOrValue - | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND - | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER + abstract evaluate(args: EvaluateArgs): TemplateEvaluationResult> } export class IdentifierExpression extends TemplateExpression { @@ -142,12 +137,7 @@ export class ArrayLiteralExpression extends TemplateExpression { super(loc) } - override evaluate( - args: EvaluateArgs - ): - | CollectionOrValue - | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND - | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { + override evaluate(args: EvaluateArgs): TemplateEvaluationResult> { const result: CollectionOrValue = [] for (const e of this.literal) { const res = e.evaluate(args) @@ -169,9 +159,7 @@ export abstract class UnaryExpression extends TemplateExpression { super(loc) } - override evaluate( - args: EvaluateArgs - ): TemplatePrimitive | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { + override evaluate(args: EvaluateArgs): TemplateEvaluationResult { const inner = this.innerExpression.evaluate(args) if (typeof inner === "symbol") { @@ -207,24 +195,27 @@ export abstract class LogicalExpression extends TemplateExpression { } } -// you need to call with unwrap: isTruthy(unwrap(value)) +export function isNotFound( + v: + | CollectionOrValue + // CONTEXT_RESOLVE_KEY_AVAILABLE_LATER is not included here on purpose, because it must always be handled separately by returning early. + | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND +): v is typeof CONTEXT_RESOLVE_KEY_NOT_FOUND { + return v === CONTEXT_RESOLVE_KEY_NOT_FOUND +} + export function isTruthy(v: CollectionOrValue): boolean { if (isTemplatePrimitive(v)) { return !!v - } else { - // collections are truthy, regardless wether they are empty or not. - v satisfies Collection - return true } + + // collections are truthy, regardless wether they are empty or not. + v satisfies Collection + return true } export class LogicalOrExpression extends LogicalExpression { - override evaluate( - args: EvaluateArgs - ): - | CollectionOrValue - | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND - | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { + override evaluate(args: EvaluateArgs): TemplateEvaluationResult> { const left = this.left.evaluate({ ...args, optional: true, @@ -235,7 +226,7 @@ export class LogicalOrExpression extends LogicalExpression { return left } - if (left !== CONTEXT_RESOLVE_KEY_NOT_FOUND && isTruthy(left)) { + if (!isNotFound(left) && isTruthy(left)) { return left } @@ -244,12 +235,7 @@ export class LogicalOrExpression extends LogicalExpression { } export class LogicalAndExpression extends LogicalExpression { - override evaluate( - args: EvaluateArgs - ): - | CollectionOrValue - | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND - | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { + override evaluate(args: EvaluateArgs): TemplateEvaluationResult> { const left = this.left.evaluate({ ...args, optional: true, @@ -261,7 +247,7 @@ export class LogicalAndExpression extends LogicalExpression { } // We return false in case the variable could not be resolved. This is a quirk of Garden's template language that we want to keep for backwards compatibility. - if (left === CONTEXT_RESOLVE_KEY_NOT_FOUND) { + if (isNotFound(left)) { return false } @@ -279,7 +265,7 @@ export class LogicalAndExpression extends LogicalExpression { return right } - if (right === CONTEXT_RESOLVE_KEY_NOT_FOUND) { + if (isNotFound(right)) { return false } @@ -297,28 +283,19 @@ export abstract class BinaryExpression extends TemplateExpression { super(loc) } - override evaluate( - args: EvaluateArgs - ): - | CollectionOrValue - | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND - | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { + override evaluate(args: EvaluateArgs): TemplateEvaluationResult> { const left = this.left.evaluate(args) - if (left === CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) { + if (typeof left === "symbol") { return left } const right = this.right.evaluate(args) - if (right === CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) { + if (typeof right === "symbol") { return right } - if (left === CONTEXT_RESOLVE_KEY_NOT_FOUND || right === CONTEXT_RESOLVE_KEY_NOT_FOUND) { - return CONTEXT_RESOLVE_KEY_NOT_FOUND - } - return this.transform(left, right, args) } @@ -479,12 +456,8 @@ export class FormatStringExpression extends TemplateExpression { super(loc) } - override evaluate( - args: EvaluateArgs - ): - | CollectionOrValue - | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND - | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { + override evaluate(args: EvaluateArgs): TemplateEvaluationResult> { + // make sure we do not override outer optional const optional = args.optional !== undefined ? args.optional : this.isOptional const result = this.innerExpression.evaluate({ @@ -492,7 +465,9 @@ export class FormatStringExpression extends TemplateExpression { optional, }) - if (optional && result === CONTEXT_RESOLVE_KEY_NOT_FOUND) { + // Only if this expression is optional we return undefined instead of symbol. + // If merely optional is true in EvaluateArgs, we must return symbol. + if (this.isOptional && result === CONTEXT_RESOLVE_KEY_NOT_FOUND) { return undefined } @@ -528,12 +503,7 @@ export class IfBlockExpression extends TemplateExpression { super(loc) } - override evaluate( - args: EvaluateArgs - ): - | CollectionOrValue - | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND - | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { + override evaluate(args: EvaluateArgs): TemplateEvaluationResult> { const condition = this.condition.evaluate(args) if (typeof condition === "symbol") { @@ -554,9 +524,7 @@ export class StringConcatExpression extends TemplateExpression { this.expressions = expressions } - override evaluate( - args: EvaluateArgs - ): string | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { + override evaluate(args: EvaluateArgs): TemplateEvaluationResult { let result: string = "" for (const expr of this.expressions) { @@ -586,9 +554,7 @@ export class MemberExpression extends TemplateExpression { super(loc) } - override evaluate( - args: EvaluateArgs - ): string | number | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { + override evaluate(args: EvaluateArgs): TemplateEvaluationResult { const inner = this.innerExpression.evaluate(args) if (typeof inner === "symbol") { @@ -620,10 +586,7 @@ export class ContextLookupExpression extends TemplateExpression { opts, optional, yamlSource, - }: EvaluateArgs): - | CollectionOrValue - | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND - | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { + }: EvaluateArgs): TemplateEvaluationResult> { const keyPath: (string | number)[] = [] for (const k of this.keyPath) { const evaluated = k.evaluate({ context, opts, optional, yamlSource }) @@ -635,14 +598,13 @@ export class ContextLookupExpression extends TemplateExpression { const { resolved, getUnavailableReason } = this.resolveContext(context, keyPath, opts, yamlSource) - // if context returns key available later, then we do not need to throw, because partial mode is enabled. - if (resolved === CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) { + if (opts.allowPartial && resolved === CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) { return resolved } // if we encounter a key not found symbol, it's an error unless the optional flag is true, which is used by // logical operators and expressions, as well as the optional suffix in FormatStringExpression. - if (resolved === CONTEXT_RESOLVE_KEY_NOT_FOUND) { + if (typeof resolved === "symbol") { if (optional) { return CONTEXT_RESOLVE_KEY_NOT_FOUND } @@ -693,12 +655,7 @@ export class FunctionCallExpression extends TemplateExpression { super(loc) } - override evaluate( - args: EvaluateArgs - ): - | CollectionOrValue - | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND - | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { + override evaluate(args: EvaluateArgs): TemplateEvaluationResult> { const functionArgs: CollectionOrValue[] = [] for (const functionArg of this.args) { const result = functionArg.evaluate(args) @@ -795,12 +752,7 @@ export class TernaryExpression extends TemplateExpression { super(loc) } - override evaluate( - args: EvaluateArgs - ): - | CollectionOrValue - | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND - | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER { + override evaluate(args: EvaluateArgs): TemplateEvaluationResult> { const conditionResult = this.condition.evaluate({ ...args, optional: true, @@ -812,7 +764,7 @@ export class TernaryExpression extends TemplateExpression { // evaluate ternary expression const evaluationResult = - conditionResult !== CONTEXT_RESOLVE_KEY_NOT_FOUND && isTruthy(conditionResult) + !isNotFound(conditionResult) && isTruthy(conditionResult) ? this.ifTrue.evaluate(args) : this.ifFalse.evaluate(args) diff --git a/core/src/template-string/static-analysis.ts b/core/src/template-string/static-analysis.ts index 6f62e8da88..691ea075a9 100644 --- a/core/src/template-string/static-analysis.ts +++ b/core/src/template-string/static-analysis.ts @@ -11,7 +11,7 @@ import { isArray, isPlainObject } from "../util/objects.js" import { ContextLookupExpression, TemplateExpression } from "./ast.js" import type { TemplatePrimitive } from "./types.js" import { parseTemplateString } from "./template-string.js" -import type { ConfigContext } from "../config/template-contexts/base.js" +import { type ConfigContext } from "../config/template-contexts/base.js" import { GardenError, InternalError } from "../exceptions.js" import { type ConfigSource } from "../config/validation.js" diff --git a/core/src/template-string/template-string.ts b/core/src/template-string/template-string.ts index 3b41fc5a59..08d65171e5 100644 --- a/core/src/template-string/template-string.ts +++ b/core/src/template-string/template-string.ts @@ -179,16 +179,18 @@ export function resolveTemplateString({ yamlSource: source, }) - if (!contextOpts.allowPartial && result === CONTEXT_RESOLVE_KEY_NOT_FOUND) { + if (!contextOpts.allowPartial && typeof result === "symbol") { throw new InternalError({ - message: "allowPartial is false, but template expression evaluated to symbol.", + message: `allowPartial is false, but template expression returned symbol ${String(result)}. ast.ContextLookupExpression should have thrown an error.`, }) - // TODO: think about if it's really ok to partially resolve if allowPartial is false. This can happen if a context with _alwaysPartial is used together with allowPartial false. - } else if (result === CONTEXT_RESOLVE_KEY_NOT_FOUND || result === CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) { - // The template expression cannot be evaluated yet, we may be able to do it later. - // TODO: return ast.TemplateExpression here, instead of string; Otherwise we'll inevitably have a bug - // where garden will resolve template expressions that might be contained in expression evaluation results - // e.g. if an environment variable contains template string, we don't want to evaluate the template string in there. + } + + // Requested partial evaluation and the template expression cannot be evaluated yet. We may be able to do it later. + if (result === CONTEXT_RESOLVE_KEY_NOT_FOUND || result === CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) { + // TODO: Parse all template expressions after reading the YAML config and only re-evaluate ast.TemplateExpression instances in + // resolveTemplateStrings; Otherwise we'll inevitably have a bug where garden will resolve template expressions that might be + // contained in expression evaluation results e.g. if an environment variable contains template string, we don't want to + // evaluate the template string in there. // See also https://github.com/garden-io/garden/issues/5825 return string } diff --git a/core/test/unit/src/template-string.ts b/core/test/unit/src/template-string.ts index 90fc82408d..7499c8cfef 100644 --- a/core/test/unit/src/template-string.ts +++ b/core/test/unit/src/template-string.ts @@ -13,7 +13,7 @@ import { throwOnMissingSecretKeys, getActionTemplateReferences, } from "../../../src/template-string/template-string.js" -import { GenericContext } from "../../../src/config/template-contexts/base.js" +import { CONTEXT_RESOLVE_KEY_AVAILABLE_LATER, GenericContext } from "../../../src/config/template-contexts/base.js" import type { TestGarden } from "../../helpers.js" import { expectError, expectFuzzyMatch, getDataDir, makeTestGarden } from "../../helpers.js" import { dedent } from "../../../src/util/string.js" @@ -2288,11 +2288,15 @@ describe("getContextLookupReferences", () => { expect(result).to.eql(expected) }) - it("should handle keys with dots correctly", async () => { + it("should handle keys with dots and unresolvable member expressions correctly", async () => { const obj = { a: "some ${templated['key.with.dots']}", b: "${more.stuff}", c: "${keyThatIs[unresolvable]}", + d: '${keyThatIs["${unresolvable}"]}', + e: "${optionalAndUnresolvable}?", + f: "${keyThatIs[availableLater]}", + g: '${keyThatIs["${availableLater}"]}', } const foundKeys = Array.from( getContextLookupReferences( @@ -2303,18 +2307,12 @@ describe("getContextLookupReferences", () => { path: [], }, }), - new GenericContext({}) + new GenericContext({ + availableLater: CONTEXT_RESOLVE_KEY_AVAILABLE_LATER, + }) ) ) - const unresolvable = foundKeys[3].keyPath[1] as UnresolvableValue - - expect(unresolvable).to.be.instanceOf(UnresolvableValue) - expectFuzzyMatch( - unresolvable.getError().message, - "invalid template string (${keythatis[unresolvable]}) at path c: could not find key unresolvable. available keys: (none)." - ) - const expected: ContextLookupReferenceFinding[] = [ { type: "resolvable", @@ -2339,13 +2337,88 @@ describe("getContextLookupReferences", () => { }, { type: "unresolvable", - keyPath: ["keyThatIs", unresolvable], + keyPath: ["keyThatIs", foundKeys[3].keyPath[1]], yamlSource: { path: ["c"], }, }, + { + type: "resolvable", + keyPath: ["unresolvable"], + yamlSource: { + path: ["d"], + }, + }, + { + type: "unresolvable", + keyPath: ["keyThatIs", foundKeys[5].keyPath[1]], + yamlSource: { + path: ["d"], + }, + }, + { + type: "resolvable", + keyPath: ["optionalAndUnresolvable"], + yamlSource: { + path: ["e"], + }, + }, + { + type: "resolvable", + keyPath: ["availableLater"], + yamlSource: { + path: ["f"], + }, + }, + { + type: "unresolvable", + keyPath: ["keyThatIs", foundKeys[8].keyPath[1]], + yamlSource: { + path: ["f"], + }, + }, + { + type: "resolvable", + keyPath: ["availableLater"], + yamlSource: { + path: ["g"], + }, + }, + { + type: "unresolvable", + keyPath: ["keyThatIs", foundKeys[10].keyPath[1]], + yamlSource: { path: ["g"] }, + }, ] - expect(foundKeys).to.deep.equals(expected) + expect(foundKeys, `Unexpected found keys. JSON: ${JSON.stringify(foundKeys)}`).to.deep.equals(expected) + + const unresolvable1 = foundKeys[3].keyPath[1] as UnresolvableValue + expect(unresolvable1).to.be.instanceOf(UnresolvableValue) + expectFuzzyMatch( + unresolvable1.getError().message, + "invalid template string (${keythatis[unresolvable]}) at path c: could not find key unresolvable." + ) + + const unresolvable2 = foundKeys[5].keyPath[1] as UnresolvableValue + expect(unresolvable2).to.be.instanceOf(UnresolvableValue) + expectFuzzyMatch( + unresolvable2.getError().message, + "invalid template string (${unresolvable}) at path d: could not find key unresolvable." + ) + + const availableLater1 = foundKeys[8].keyPath[1] as UnresolvableValue + expect(availableLater1).to.be.instanceOf(UnresolvableValue) + expectFuzzyMatch( + availableLater1.getError().message, + "invalid template string (${keythatis[availablelater]}) at path f: could not find key availableLater." + ) + + const availableLater2 = foundKeys[10].keyPath[1] as UnresolvableValue + expect(availableLater2).to.be.instanceOf(UnresolvableValue) + expectFuzzyMatch( + availableLater2.getError().message, + "invalid template string (${availablelater}) at path g: could not find key availableLater." + ) }) }) From f9f9e43bcf8701bf4071abd077e98e1c49c2ca98 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Fri, 6 Dec 2024 08:58:21 +0100 Subject: [PATCH 41/43] fix: handle optional expressions in brackets correctly --- core/src/template-string/ast.ts | 5 +-- core/test/unit/src/template-string.ts | 49 +++++++++++++++++++++------ 2 files changed, 39 insertions(+), 15 deletions(-) diff --git a/core/src/template-string/ast.ts b/core/src/template-string/ast.ts index db9c41947a..392119b84a 100644 --- a/core/src/template-string/ast.ts +++ b/core/src/template-string/ast.ts @@ -457,12 +457,9 @@ export class FormatStringExpression extends TemplateExpression { } override evaluate(args: EvaluateArgs): TemplateEvaluationResult> { - // make sure we do not override outer optional - const optional = args.optional !== undefined ? args.optional : this.isOptional - const result = this.innerExpression.evaluate({ ...args, - optional, + optional: args.optional || this.isOptional, }) // Only if this expression is optional we return undefined instead of symbol. diff --git a/core/test/unit/src/template-string.ts b/core/test/unit/src/template-string.ts index 7499c8cfef..01ece3bbcf 100644 --- a/core/test/unit/src/template-string.ts +++ b/core/test/unit/src/template-string.ts @@ -65,17 +65,44 @@ describe("resolveTemplateString", () => { expect(res).to.equal("${foo}?") }) - it("should not crash when variable in a member expression cannot be resolved", () => { - const res = resolveTemplateString({ - string: '${actions.run["${inputs.deployableTarget}-dummy"].var}', - context: new GenericContext({ - actions: { - run: {}, - }, - }), - contextOpts: { allowPartial: true }, - }) - expect(res).to.equal('${actions.run["${inputs.deployableTarget}-dummy"].var}') + it("should not crash when variable in a member expression cannot be resolved with allowPartial=true", () => { + const inputs = [ + '${actions.run["${inputs.deployableTarget}-dummy"].var}', + '${actions.build["${parent.name}"].outputs.deployment-image-id}', + '${actions.build["${parent.name}?"]}', + ] + for (const input of inputs) { + const res = resolveTemplateString({ + string: input, + context: new GenericContext({ + actions: { + run: {}, + build: {}, + }, + }), + contextOpts: { allowPartial: true }, + }) + expect(res).to.equal(input) + } + }) + + it("should fail if optional expression in member expression cannot be resolved with allowPartial=false", async () => { + await expectError( + () => + resolveTemplateString({ + string: '${actions.build["${parent.name}?"]}', + context: new GenericContext({ + actions: { + build: {}, + }, + }), + contextOpts: { allowPartial: false }, + }), + { + contains: + 'Invalid template string (${actions.build["${parent.name}?"]}): Expression in brackets must resolve to a string or number (got undefined).', + } + ) }) it("should support a string literal in a template string as a means to escape it", () => { From 2caced0996f0979965bb28853af8ad93ab1fdb62 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Fri, 6 Dec 2024 09:36:36 +0100 Subject: [PATCH 42/43] fix: special edge case in module resolution flow --- core/src/config/template-contexts/base.ts | 3 +++ core/src/resolve-module.ts | 3 ++- core/src/template-string/ast.ts | 2 +- core/src/template-string/template-string.ts | 21 +++++++++++---------- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index 5675c9c247..7ef0eb1213 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -25,6 +25,9 @@ export type ContextKey = ContextKeySegment[] export interface ContextResolveOpts { // Allow templates to be partially resolved (used to defer runtime template resolution, for example) allowPartial?: boolean + // Allow partial resolution for values that originate from a special context that always returns CONTEXT_RESOLVE_KEY_AVAILABLE_LATER. + // This is used for module resolution and can be removed whenever we remove support for modules. + allowPartialContext?: boolean // a list of previously resolved paths, used to detect circular references stack?: Set // Unescape escaped template strings diff --git a/core/src/resolve-module.ts b/core/src/resolve-module.ts index caad7a6e16..422af69cba 100644 --- a/core/src/resolve-module.ts +++ b/core/src/resolve-module.ts @@ -661,7 +661,8 @@ export class ModuleResolver { value: { ...config, inputs: {}, variables: {} }, context: configContext, contextOpts: { - allowPartial: true, + allowPartial: false, + allowPartialContext: true, // Modules will be converted to actions later, and the actions will be properly unescaped. // We avoid premature un-escaping here, // because otherwise it will strip the escaped value in the module config diff --git a/core/src/template-string/ast.ts b/core/src/template-string/ast.ts index 392119b84a..99afd5ff80 100644 --- a/core/src/template-string/ast.ts +++ b/core/src/template-string/ast.ts @@ -595,7 +595,7 @@ export class ContextLookupExpression extends TemplateExpression { const { resolved, getUnavailableReason } = this.resolveContext(context, keyPath, opts, yamlSource) - if (opts.allowPartial && resolved === CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) { + if ((opts.allowPartial || opts.allowPartialContext) && resolved === CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) { return resolved } diff --git a/core/src/template-string/template-string.ts b/core/src/template-string/template-string.ts index 08d65171e5..c1f230fe20 100644 --- a/core/src/template-string/template-string.ts +++ b/core/src/template-string/template-string.ts @@ -179,23 +179,24 @@ export function resolveTemplateString({ yamlSource: source, }) - if (!contextOpts.allowPartial && typeof result === "symbol") { + if (typeof result !== "symbol") { + return result + } + + if (!contextOpts.allowPartial && !contextOpts.allowPartialContext) { throw new InternalError({ message: `allowPartial is false, but template expression returned symbol ${String(result)}. ast.ContextLookupExpression should have thrown an error.`, }) } // Requested partial evaluation and the template expression cannot be evaluated yet. We may be able to do it later. - if (result === CONTEXT_RESOLVE_KEY_NOT_FOUND || result === CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) { - // TODO: Parse all template expressions after reading the YAML config and only re-evaluate ast.TemplateExpression instances in - // resolveTemplateStrings; Otherwise we'll inevitably have a bug where garden will resolve template expressions that might be - // contained in expression evaluation results e.g. if an environment variable contains template string, we don't want to - // evaluate the template string in there. - // See also https://github.com/garden-io/garden/issues/5825 - return string - } - return result + // TODO: Parse all template expressions after reading the YAML config and only re-evaluate ast.TemplateExpression instances in + // resolveTemplateStrings; Otherwise we'll inevitably have a bug where garden will resolve template expressions that might be + // contained in expression evaluation results e.g. if an environment variable contains template string, we don't want to + // evaluate the template string in there. + // See also https://github.com/garden-io/garden/issues/5825 + return string } /** From d25be9f4ce45a00bce2a3ebfa2beb678d1f5d434 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Fri, 6 Dec 2024 10:25:22 +0100 Subject: [PATCH 43/43] chore: fix lint --- core/src/template-string/template-string.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/core/src/template-string/template-string.ts b/core/src/template-string/template-string.ts index c1f230fe20..13c1d53a8a 100644 --- a/core/src/template-string/template-string.ts +++ b/core/src/template-string/template-string.ts @@ -9,11 +9,7 @@ import type { GardenErrorParams } from "../exceptions.js" import { ConfigurationError, GardenError, InternalError, TemplateStringError } from "../exceptions.js" import type { ConfigContext, ContextKeySegment, ContextResolveOpts } from "../config/template-contexts/base.js" -import { - CONTEXT_RESOLVE_KEY_AVAILABLE_LATER, - CONTEXT_RESOLVE_KEY_NOT_FOUND, - GenericContext, -} from "../config/template-contexts/base.js" +import { GenericContext } from "../config/template-contexts/base.js" import cloneDeep from "fast-copy" import { difference, isPlainObject, isString } from "lodash-es" import type { ActionReference, Primitive, StringMap } from "../config/common.js"