diff --git a/core/src/actions/base.ts b/core/src/actions/base.ts index 0803a73524..64d4c33612 100644 --- a/core/src/actions/base.ts +++ b/core/src/actions/base.ts @@ -47,6 +47,7 @@ import type { ActionStatus, ActionWrapperParams, BaseActionConfig, + BaseActionConfigMetadata, ExecutedAction, ExecutedActionWrapperParams, GetOutputValueType, @@ -65,6 +66,8 @@ import { joinWithPosix } from "../util/fs.js" import type { LinkedSource } from "../config-store/local.js" import type { BaseActionTaskParams, ExecuteTask } from "../tasks/base.js" import { styles } from "../logger/styles.js" +import { GardenConfig } from "../template-string/validation.js" +import { ConfigContext } from "../config/template-contexts/base.js" // TODO: split this file @@ -242,8 +245,11 @@ export const baseActionConfigSchema = createSchema({ }), }) -export interface BaseRuntimeActionConfig - extends BaseActionConfig { +export type BaseRuntimeActionConfig = BaseActionConfig< + K, + N, + S +> & { build?: string } @@ -330,7 +336,7 @@ export abstract class BaseAction< // Note: These need to be public because we need to reference the types (a current TS limitation) // We do however also use `_staticOutputs` so that one isn't just for types - _config: C + _config: GardenConfig abstract _staticOutputs: StaticOutputs // This property is only used for types. // In theory it would be easy to replace it with a type that uses `infer` on BaseAction to grab the correct type @@ -351,12 +357,12 @@ export abstract class BaseAction< protected readonly projectRoot: string protected readonly _supportedModes: ActionModes protected readonly _treeVersion: TreeVersion - protected readonly variables: DeepPrimitiveMap + protected readonly variables: ConfigContext constructor(protected readonly params: ActionWrapperParams) { - this.kind = params.config.kind - this.type = params.config.type - this.name = params.config.name + this.kind = params.config.config.kind + this.type = params.config.config.type + this.name = params.config.config.name this.uid = params.uid this.baseBuildDirectory = params.baseBuildDirectory this.compatibleTypes = params.compatibleTypes @@ -410,7 +416,7 @@ export abstract class BaseAction< isDisabled(): boolean { // TODO: return true if group is disabled // TODO: implement environments field on action config - return actionIsDisabled(this._config, "TODO") + return actionIsDisabled(this._config.config, "TODO") } /** @@ -435,17 +441,16 @@ export abstract class BaseAction< } hasRemoteSource() { - return !!this._config.source?.repository?.url + return !!this._config.config.source?.repository?.url } groupName() { - const internal = this.getConfig("internal") - return internal?.groupName + return this.getInternal().groupName } sourcePath(): string { - const basePath = this.remoteSourcePath || this._config.internal.basePath - const sourceRelPath = this._config.source?.path + const basePath = this.remoteSourcePath || this._config.configFileDirname || this.projectRoot + const sourceRelPath = this._config.config.source?.path if (sourceRelPath) { // TODO: validate that this is a directory here? @@ -456,7 +461,7 @@ export abstract class BaseAction< } configPath() { - return this._config.internal.configFilePath + return this._config.configFilePath } moduleName(): string | null { @@ -568,7 +573,7 @@ export abstract class BaseAction< } } - getVariables(): DeepPrimitiveMap { + getVariables(): ConfigContext { return this.variables } @@ -576,14 +581,14 @@ export abstract class BaseAction< return this.getFullVersion().versionString } - getInternal(): BaseActionConfig["internal"] { - return { ...this.getConfig("internal") } + getInternal(): BaseActionConfigMetadata { + return { ...this._config.metadata } } getConfig(): C getConfig(key: K): C[K] getConfig(key?: keyof C["spec"]) { - return cloneDeep(key ? this._config[key] : this._config) + return key ? this._config.config[key] : this._config.config } /** @@ -694,7 +699,7 @@ export interface ResolvedActionExtension< getOutputs(): StaticOutputs - getVariables(): DeepPrimitiveMap + getVariables(): ConfigContext } // TODO: see if we can avoid the duplication here with ResolvedBuildAction @@ -725,8 +730,6 @@ export abstract class ResolvedRuntimeAction< this.executedDependencies = params.executedDependencies this.resolvedDependencies = params.resolvedDependencies this._staticOutputs = params.staticOutputs - this._config.spec = params.spec - this._config.internal.inputs = params.inputs } /** @@ -765,7 +768,7 @@ export abstract class ResolvedRuntimeAction< getSpec(): Config["spec"] getSpec(key: K): Config["spec"][K] getSpec(key?: keyof Config["spec"]) { - return cloneDeep(key ? this._config.spec[key] : this._config.spec) + return key ? this._config.config.spec[key] : this._config.config.spec } getOutput(key: K): GetOutputValueType { @@ -856,20 +859,20 @@ export function getSourceAbsPath(basePath: string, sourceRelPath: string) { return joinWithPosix(basePath, sourceRelPath) } -export function describeActionConfig(config: ActionConfig) { - const d = `${config.type} ${config.kind} ${config.name}` - if (config.internal?.moduleName) { - return d + ` (from module ${config.internal?.moduleName})` - } else if (config.internal?.groupName) { - return d + ` (from group ${config.internal?.groupName})` +export function describeActionConfig(c: GardenConfig) { + const d = `${c.config.type} ${c.config.kind} ${c.config.name}` + if (c.metadata.moduleName) { + return d + ` (from module ${c.metadata.moduleName})` + } else if (c.metadata.groupName) { + return d + ` (from group ${c.metadata.groupName})` } else { return d } } -export function describeActionConfigWithPath(config: ActionConfig, rootPath: string) { - const path = relative(rootPath, config.internal.configFilePath || config.internal.basePath) - return `${describeActionConfig(config)} in ${path}` +export function describeActionConfigWithPath(c: GardenConfig, rootPath: string) { + const path = relative(rootPath, c.metadata.configFilePath || c.metadata.basePath) + return `${describeActionConfig(c)} in ${path}` } /** diff --git a/core/src/actions/build.ts b/core/src/actions/build.ts index 185fee8fce..f803d7f33c 100644 --- a/core/src/actions/build.ts +++ b/core/src/actions/build.ts @@ -32,14 +32,13 @@ import { createBuildTask } from "../tasks/build.js" import type { BaseActionTaskParams, ExecuteTask } from "../tasks/base.js" import { ResolveActionTask } from "../tasks/resolve-action.js" -export interface BuildCopyFrom { +export type BuildCopyFrom = { build: string sourcePath: string targetPath: string } -export interface BuildActionConfig - extends BaseActionConfig<"Build", T, S> { +export type BuildActionConfig = BaseActionConfig<"Build", T, S> & { type: T allowPublish?: boolean buildAtSource?: boolean @@ -141,7 +140,7 @@ export const buildActionConfigSchema = createSchema({ }) export class BuildAction< - C extends BuildActionConfig = BuildActionConfig, + C extends BuildActionConfig = BuildActionConfig, StaticOutputs extends {} = any, RuntimeOutputs extends {} = any, > extends BaseAction { @@ -175,7 +174,7 @@ export class BuildAction< * If `buildAtSource: true` is set on the config, the path is the base path of the action. */ getBuildPath() { - if (this._config.buildAtSource) { + if (this._config.metadata.buildAtSource) { return this.sourcePath() } else { return join(this.baseBuildDirectory, this.name) @@ -217,8 +216,7 @@ export class ResolvedBuildAction< this.resolvedDependencies = params.resolvedDependencies this.resolved = true this._staticOutputs = params.staticOutputs - this._config.spec = params.spec - this._config.internal.inputs = params.inputs + this._config = params.config } getExecutedDependencies() { @@ -237,7 +235,7 @@ export class ResolvedBuildAction< getSpec(): C["spec"] getSpec(key: K): C["spec"][K] getSpec(key?: keyof C["spec"]) { - return key ? this._config.spec[key] : this._config.spec + return key ? this._config.config.spec[key] : this._config.config.spec } getOutput(key: K): GetOutputValueType { @@ -282,6 +280,10 @@ export function isBuildAction(action: Action): action is BuildAction { return action.kind === "Build" } +export function isResolvedBuildAction(action: ResolvedAction): action is ResolvedBuildAction { + return action.kind === "Build" +} + export function isBuildActionConfig(config: BaseActionConfig): config is BuildActionConfig { return config.kind === "Build" } diff --git a/core/src/actions/helpers.ts b/core/src/actions/helpers.ts index 1e054c7a48..a5a8a66923 100644 --- a/core/src/actions/helpers.ts +++ b/core/src/actions/helpers.ts @@ -13,9 +13,21 @@ import type { Log } from "../logger/log-entry.js" import { createActionLog } from "../logger/log-entry.js" import { renderDivider } from "../logger/util.js" import { getLinkedSources } from "../util/ext-source-util.js" -import { buildActionConfigSchema, ExecutedBuildAction, isBuildAction, ResolvedBuildAction } from "./build.js" -import { deployActionConfigSchema, ExecutedDeployAction, isDeployAction, ResolvedDeployAction } from "./deploy.js" -import { ExecutedRunAction, isRunAction, ResolvedRunAction, runActionConfigSchema } from "./run.js" +import { + buildActionConfigSchema, + ExecutedBuildAction, + isBuildAction, + isResolvedBuildAction, + ResolvedBuildAction, +} from "./build.js" +import { + deployActionConfigSchema, + ExecutedDeployAction, + isDeployAction, + isResolvedDeployAction, + ResolvedDeployAction, +} from "./deploy.js" +import { ExecutedRunAction, isResolvedRunAction, isRunAction, ResolvedRunAction, runActionConfigSchema } from "./run.js" import { ExecutedTestAction, isTestAction, ResolvedTestAction, testActionConfigSchema } from "./test.js" import type { Action, @@ -38,7 +50,7 @@ import { styles } from "../logger/styles.js" /** * Creates a corresponding Resolved version of the given Action, given the additional parameters needed. */ -export function actionToResolved(action: T, params: ResolveActionParams) { +export function actionToResolved(action: T, params: ResolveActionParams) { if (isBuildAction(action)) { return new ResolvedBuildAction({ ...action["params"], ...params }) } else if (isDeployAction(action)) { @@ -57,13 +69,13 @@ export function actionToResolved(action: T, params: ResolveAct */ export function resolvedActionToExecuted( action: T, - params: ExecuteActionParams + params: ExecuteActionParams ): Executed { - if (isBuildAction(action)) { + if (isResolvedBuildAction(action)) { return new ExecutedBuildAction({ ...action["params"], ...params }) as Executed - } else if (isDeployAction(action)) { + } else if (isResolvedDeployAction(action)) { return new ExecutedDeployAction({ ...action["params"], ...params }) as Executed - } else if (isRunAction(action)) { + } else if (isResolvedRunAction(action)) { return new ExecutedRunAction({ ...action["params"], ...params }) as Executed } else if (isTestAction(action)) { return new ExecutedTestAction({ ...action["params"], ...params }) as Executed diff --git a/core/src/actions/run.ts b/core/src/actions/run.ts index 90d3427cae..43b346ebbc 100644 --- a/core/src/actions/run.ts +++ b/core/src/actions/run.ts @@ -10,7 +10,7 @@ import { memoize } from "lodash-es" import { joi } from "../config/common.js" import type { BaseRuntimeActionConfig } from "./base.js" import { baseRuntimeActionConfigSchema, ExecutedRuntimeAction, ResolvedRuntimeAction, RuntimeAction } from "./base.js" -import type { Action, BaseActionConfig } from "./types.js" +import type { Action, BaseActionConfig, ResolvedAction } from "./types.js" import { DEFAULT_RUN_TIMEOUT_SEC } from "../constants.js" import { createRunTask } from "../tasks/run.js" import type { BaseActionTaskParams, ExecuteTask } from "../tasks/base.js" @@ -83,6 +83,10 @@ export function isRunAction(action: Action): action is RunAction { return action.kind === "Run" } +export function isResolvedRunAction(action: ResolvedAction): action is ResolvedRunAction { + return action.kind === "Run" +} + export function isRunActionConfig(config: BaseActionConfig): config is RunActionConfig { return config.kind === "Run" } diff --git a/core/src/actions/test.ts b/core/src/actions/test.ts index 6ee23ea5bb..311e5e837b 100644 --- a/core/src/actions/test.ts +++ b/core/src/actions/test.ts @@ -10,7 +10,7 @@ import { memoize } from "lodash-es" import { joi } from "../config/common.js" import type { BaseRuntimeActionConfig } from "./base.js" import { baseRuntimeActionConfigSchema, ExecutedRuntimeAction, ResolvedRuntimeAction, RuntimeAction } from "./base.js" -import type { Action, BaseActionConfig } from "./types.js" +import type { Action, BaseActionConfig, ResolvedAction } from "./types.js" import { DEFAULT_TEST_TIMEOUT_SEC } from "../constants.js" import type { BaseActionTaskParams, ExecuteTask } from "../tasks/base.js" import { createTestTask } from "../tasks/test.js" @@ -48,7 +48,7 @@ export class TestAction< } export class ResolvedTestAction< - C extends TestActionConfig = any, + C extends TestActionConfig = TestActionConfig, StaticOutputs extends {} = any, RuntimeOutputs extends {} = any, > extends ResolvedRuntimeAction { @@ -83,6 +83,10 @@ export function isTestAction(action: Action): action is TestAction { return action.kind === "Test" } +export function isResolvedTestAction(action: ResolvedAction): action is ResolvedTestAction { + return action.kind === "Test" +} + export function isTestActionConfig(config: BaseActionConfig): config is TestActionConfig { return config.kind === "Test" } diff --git a/core/src/actions/types.ts b/core/src/actions/types.ts index 2e7f6aadcd..6309f0c077 100644 --- a/core/src/actions/types.ts +++ b/core/src/actions/types.ts @@ -18,10 +18,11 @@ import type { ActionKind } from "../plugin/action-types.js" import type { GraphResults } from "../graph/results.js" import type { BaseAction } from "./base.js" import type { ValidResultType } from "../tasks/base.js" -import type { BaseGardenResource, GardenResourceInternalFields } from "../config/base.js" +import type { BaseGardenResource, BaseGardenResourceMetadata } from "../config/base.js" import type { LinkedSource } from "../config-store/local.js" import type { GardenApiVersion } from "../constants.js" - +import { GardenConfig } from "../template-string/validation.js" +import { ConfigContext } from "../config/template-contexts/base.js" // TODO: split this file export type { ActionKind } from "../plugin/action-types.js" @@ -29,24 +30,33 @@ export type { ActionKind } from "../plugin/action-types.js" export const actionKinds: ActionKind[] = ["Build", "Deploy", "Run", "Test"] export const actionKindsLower = actionKinds.map((k) => k.toLowerCase()) -interface SourceRepositorySpec { +type SourceRepositorySpec = { url: string // TODO: subPath?: string // TODO: commitHash?: string } -export interface ActionSourceSpec { +export type ActionSourceSpec = { path?: string repository?: SourceRepositorySpec } +export type BaseActionConfigMetadata = BaseGardenResourceMetadata & { + groupName?: string + resolved?: boolean // Set to true if no resolution is required, e.g. set for actions converted from modules + treeVersion?: TreeVersion // Set during module resolution to avoid duplicate scanning for Build actions + // For forwards-compatibility, applied on actions returned from module conversion handlers + remoteClonePath?: string + moduleName?: string + moduleVersion?: ModuleVersion +} + /** * These are the built-in fields in all action configs. * * See inline comments below for information on what templating is allowed on different fields. */ -export interface BaseActionConfig - extends BaseGardenResource { +export type BaseActionConfig = BaseGardenResource & { // Basics // -> No templating is allowed on these. apiVersion?: GardenApiVersion @@ -61,15 +71,7 @@ export interface BaseActionConfig No templating is allowed on these. - internal: GardenResourceInternalFields & { - groupName?: string - resolved?: boolean // Set to true if no resolution is required, e.g. set for actions converted from modules - treeVersion?: TreeVersion // Set during module resolution to avoid duplicate scanning for Build actions - // For forwards-compatibility, applied on actions returned from module conversion handlers - remoteClonePath?: string - moduleName?: string - moduleVersion?: ModuleVersion - } + // internal: // Flow/execution control // -> Templating with ActionConfigContext allowed @@ -93,7 +95,7 @@ export interface BaseActionConfig Deploy: DeployActionConfig Run: RunActionConfig @@ -110,21 +112,21 @@ export interface ActionConfigTypes { export const actionStateTypes = ["ready", "not-ready", "processing", "failed", "unknown"] as const export type ActionState = (typeof actionStateTypes)[number] -export interface ActionStatus< +export type ActionStatus< T extends BaseAction = BaseAction, D extends {} = any, O extends {} = GetActionOutputType, -> extends ValidResultType { +> = ValidResultType & { state: ActionState detail: D | null outputs: O } -export interface ActionStatusMap { +export type ActionStatusMap = { [key: string]: ActionStatus } -export interface ActionDependencyAttributes { +export type ActionDependencyAttributes = { explicit: boolean // Set to true if action config explicitly states the dependency needsStaticOutputs: boolean // Set to true if action cannot be resolved without resolving the dependency needsExecutedOutputs: boolean // Set to true if action cannot be resolved without the dependency executed @@ -132,7 +134,7 @@ export interface ActionDependencyAttributes { export type ActionDependency = ActionReference & ActionDependencyAttributes -export interface ActionModes { +export type ActionModes = { sync?: boolean local?: boolean } @@ -143,10 +145,10 @@ export type ActionModeMap = { [mode in ActionMode]?: string[] } -export interface ActionWrapperParams { +export type ActionWrapperParams = { baseBuildDirectory: string // /.garden/build by default compatibleTypes: string[] - config: C + config: GardenConfig // It's not ideal that we're passing this here, but since we reuse the params of the base action in // `actionToResolved` and `resolvedActionToExecuted`, it's probably clearest and least magical to pass it in // explicitly at action creation time (which only happens in a very few places in the code base anyway). @@ -161,18 +163,17 @@ export interface ActionWrapperParams { remoteSourcePath: string | null supportedModes: ActionModes treeVersion: TreeVersion - variables: DeepPrimitiveMap + variables: ConfigContext } -export interface ResolveActionParams { +export type ResolveActionParams = { resolvedGraph: ResolvedConfigGraph dependencyResults: GraphResults executedDependencies: ExecutedAction[] resolvedDependencies: ResolvedAction[] - spec: C["spec"] staticOutputs: StaticOutputs - inputs: DeepPrimitiveMap - variables: DeepPrimitiveMap + config: GardenConfig + variables: ConfigContext } export type ResolvedActionWrapperParams< @@ -180,11 +181,11 @@ export type ResolvedActionWrapperParams< StaticOutputs extends {} = any, > = ActionWrapperParams & ResolveActionParams -export interface ExecuteActionParams< +export type ExecuteActionParams< C extends BaseActionConfig = BaseActionConfig, StaticOutputs extends {} = any, RuntimeOutputs extends {} = any, -> { +> = { status: ActionStatus, any, RuntimeOutputs> } @@ -201,24 +202,26 @@ export type Action = BuildAction | DeployAction | RunAction | TestAction export type ResolvedAction = ResolvedBuildAction | ResolvedDeployAction | ResolvedRunAction | ResolvedTestAction export type ExecutedAction = ExecutedBuildAction | ExecutedDeployAction | ExecutedRunAction | ExecutedTestAction -export type Resolved = T extends BuildAction - ? ResolvedBuildAction - : T extends DeployAction - ? ResolvedDeployAction - : T extends RunAction - ? ResolvedRunAction - : T extends TestAction - ? ResolvedTestAction +// TODO: use `infer` for StaticOutputs and RuntimeOutputs, as we do for Config +export type Resolved = T extends BuildAction + ? ResolvedBuildAction + : T extends DeployAction + ? ResolvedDeployAction + : T extends RunAction + ? ResolvedRunAction + : T extends TestAction + ? ResolvedTestAction : T -export type Executed = T extends BuildAction - ? ExecutedBuildAction - : T extends DeployAction - ? ExecutedDeployAction - : T extends RunAction - ? ExecutedRunAction - : T extends TestAction - ? ExecutedTestAction +// TODO: use `infer` for StaticOutputs and RuntimeOutputs, as we do for Config +export type Executed = T extends BuildAction + ? ExecutedBuildAction + : T extends DeployAction + ? ExecutedDeployAction + : T extends RunAction + ? ExecutedRunAction + : T extends TestAction + ? ExecutedTestAction : T export type ActionReferenceMap = { @@ -231,7 +234,7 @@ export type ActionConfigMap = { } } -export interface ActionConfigsByKey { +export type ActionConfigsByKey = { [key: string]: ActionConfig } diff --git a/core/src/cli/cli.ts b/core/src/cli/cli.ts index 36731b851d..005db275e9 100644 --- a/core/src/cli/cli.ts +++ b/core/src/cli/cli.ts @@ -37,14 +37,13 @@ import { } from "./helpers.js" import type { ParameterObject, GlobalOptions, ParameterValues } from "./params.js" import { globalOptions, OUTPUT_RENDERERS } from "./params.js" -import type { ProjectConfig } from "../config/project.js" import { ERROR_LOG_FILENAME, DEFAULT_GARDEN_DIR_NAME, LOGS_DIR_NAME, gardenEnv } from "../constants.js" import { generateBasicDebugInfoReport } from "../commands/get/get-debug-info.js" import type { AnalyticsHandler } from "../analytics/analytics.js" import type { GardenPluginReference } from "../plugin/plugin.js" import type { CloudApiFactory } from "../cloud/api.js" import { CloudApi, CloudApiTokenRefreshError, getGardenCloudDomain } from "../cloud/api.js" -import { findProjectConfig } from "../config/base.js" +import { type UnrefinedProjectConfig, findProjectConfig } from "../config/base.js" import { pMemoizeDecorator } from "../lib/p-memoize.js" import { getCustomCommands } from "../commands/custom.js" import { Profile } from "../util/profiling.js" @@ -242,8 +241,8 @@ ${renderCommands(commands)} let cloudApi: CloudApi | undefined if (!command.noProject) { - const config = await this.getProjectConfig(log, workingDir) - const cloudDomain = getGardenCloudDomain(config?.domain) + const projectConfig = await this.getProjectConfig(log, workingDir) + const cloudDomain = getGardenCloudDomain(projectConfig?.config.domain) const distroName = getCloudDistributionName(cloudDomain) try { @@ -258,7 +257,7 @@ ${renderCommands(commands)} `) // Project is configured for cloud usage => fail early to force re-auth - if (config && config.id) { + if (projectConfig && projectConfig.config.id) { throw err } } else { @@ -446,7 +445,7 @@ ${renderCommands(commands)} return done(1, styles.error(`Could not find specified root path (${argv.root})`)) } - let projectConfig: ProjectConfig | undefined + let projectConfig: UnrefinedProjectConfig | undefined // First look for native Garden commands const picked = pickCommand(Object.values(this.commands), argv._) @@ -648,14 +647,14 @@ ${renderCommands(commands)} } @pMemoizeDecorator() - async getProjectConfig(log: Log, workingDir: string): Promise { + async getProjectConfig(log: Log, workingDir: string): Promise { return findProjectConfig({ log, path: workingDir }) } @pMemoizeDecorator() private async getCustomCommands(log: Log, workingDir: string): Promise { const projectConfig = await this.getProjectConfig(log, workingDir) - const projectRoot = projectConfig?.path + const projectRoot = projectConfig?.configFileDirname if (!projectRoot) { return [] diff --git a/core/src/commands/create/create-project.ts b/core/src/commands/create/create-project.ts index 0cdae3290f..aac15e93cc 100644 --- a/core/src/commands/create/create-project.ts +++ b/core/src/commands/create/create-project.ts @@ -117,7 +117,7 @@ export class CreateProjectCommand extends Command c.kind === "Project").length > 0) { + if (configs.filter((c) => c.config.kind === "Project").length > 0) { throw new CreateError({ message: `A Garden project already exists in ${configPath}`, }) diff --git a/core/src/commands/custom.ts b/core/src/commands/custom.ts index 5dfcd73531..533a804308 100644 --- a/core/src/commands/custom.ts +++ b/core/src/commands/custom.ts @@ -15,8 +15,8 @@ import type { Parameter, ParameterObject } from "../cli/params.js" import { BooleanParameter, globalOptions, IntegerParameter, StringParameter } from "../cli/params.js" import { loadConfigResources } from "../config/base.js" import type { CommandResource, CustomCommandOption } from "../config/command.js" -import { customCommandExecSchema, customCommandGardenCommandSchema, customCommandSchema } from "../config/command.js" -import { joi } from "../config/common.js" +import { customCommandSchema } from "../config/command.js" +import { omitFromSchema } from "../config/common.js" import { CustomCommandContext } from "../config/template-contexts/custom-command.js" import { validateWithPath } from "../config/validation.js" import type { GardenError } from "../exceptions.js" @@ -32,6 +32,8 @@ import { getBuiltinCommands } from "./commands.js" import type { Log } from "../logger/log-entry.js" import { getTracePropagationEnvVars } from "../util/open-telemetry/propagation.js" import { styles } from "../logger/styles.js" +import { GardenConfig } from "../template-string/validation.js" +import { GenericContext, LayeredContext } from "../config/template-contexts/base.js" function convertArgSpec(spec: CustomCommandOption) { const params = { @@ -77,15 +79,15 @@ export class CustomCommandWrapper extends Command { override allowUndefinedArguments = true - constructor(public spec: CommandResource) { - super(spec) - this.name = spec.name - this.help = spec.description?.short - this.description = spec.description?.long + constructor(public config: GardenConfig, BaseGardenResourceMetadata>) { + super(config) + this.name = config.config.name + this.help = config.config.description?.short + this.description = config.config.description?.long // Convert argument specs, so they'll be validated - this.arguments = spec.args ? mapValues(keyBy(spec.args, "name"), convertArgSpec) : {} - this.options = mapValues(keyBy(spec.opts, "name"), convertArgSpec) + this.arguments = config.config.args ? mapValues(keyBy(config.config.args, "name"), convertArgSpec) : {} + this.options = mapValues(keyBy(config.config.opts, "name"), convertArgSpec) } override printHeader({ log }: PrintHeaderParams) { @@ -112,41 +114,25 @@ export class CustomCommandWrapper extends Command { // Strip the command name and any specified arguments off the $rest variable const rest = removeSlice(parsed._unknown, this.getPath()).slice(Object.keys(this.arguments || {}).length) - const yamlDoc = this.spec.internal.yamlDoc + // command variables are lazy and have precendence over garden variables + const commandVariables = new GenericContext(this.config.config.variables) + const variables = new LayeredContext(commandVariables, garden.variables) - // Render the command variables - const variablesContext = new CustomCommandContext({ ...garden, args, opts, rest }) - const commandVariables = resolveTemplateStrings({ - value: this.spec.variables, - context: variablesContext, - source: { yamlDoc, basePath: ["variables"] }, - }) - const variables: any = jsonMerge(cloneDeep(garden.variables), commandVariables) - - // Make a new template context with the resolved variables + // Make a new template context with the lazy variables const commandContext = new CustomCommandContext({ ...garden, args, opts, variables, rest }) const result: CustomCommandResult = {} const errors: GardenError[] = [] + const refined = this.config + .withContext(commandContext) + .refineWithJoi(customCommandSchema()) + // Run exec command - if (this.spec.exec) { + if (refined.config.exec) { const startedAt = new Date() - const exec = validateWithPath({ - config: resolveTemplateStrings({ - value: this.spec.exec, - context: commandContext, - source: { yamlDoc, basePath: ["exec"] }, - }), - schema: customCommandExecSchema(), - path: this.spec.internal.basePath, - projectRoot: garden.projectRoot, - configType: `exec field in custom Command '${this.name}'`, - source: undefined, - }) - - const command = exec.command + const command = refined.config.exec.command log.debug(`Running exec command: ${command.join(" ")}`) const res = await execa(command[0], command.slice(1), { @@ -154,7 +140,7 @@ export class CustomCommandWrapper extends Command { buffer: true, env: { ...process.env, - ...(exec.env || {}), + ...(refined.config.exec.env || {}), ...getTracePropagationEnvVars(), }, cwd: garden.projectRoot, @@ -182,23 +168,10 @@ export class CustomCommandWrapper extends Command { } // Run Garden command - if (this.spec.gardenCommand) { + if (refined.config.gardenCommand) { const startedAt = new Date() - let gardenCommand = validateWithPath({ - config: resolveTemplateStrings({ - value: this.spec.gardenCommand, - context: commandContext, - source: { yamlDoc, basePath: ["gardenCommand"] }, - }), - schema: customCommandGardenCommandSchema(), - path: this.spec.internal.basePath, - projectRoot: garden.projectRoot, - configType: `gardenCommand field in custom Command '${this.name}'`, - source: undefined, - }) - - log.debug(`Running Garden command: ${gardenCommand.join(" ")}`) + log.debug(`Running Garden command: ${refined.config.gardenCommand.join(" ")}`) // Doing runtime check to avoid updating hundreds of test invocations with a new required param, sorry. - JE if (!cli) { @@ -206,7 +179,7 @@ export class CustomCommandWrapper extends Command { } // Pass explicitly set global opts with the command, if they're not set in the command itself. - const parsedCommand = parseCliArgs({ stringArgs: gardenCommand, command: this, cli: false }) + const parsedCommand = parseCliArgs({ stringArgs: refined.config.gardenCommand, command: this, cli: false }) const globalFlags = Object.entries(opts) .filter(([flag, value]) => { @@ -220,7 +193,7 @@ export class CustomCommandWrapper extends Command { }) .flatMap(([flag, value]) => ["--" + flag, value + ""]) - gardenCommand = [...globalFlags, ...gardenCommand] + const gardenCommand = [...globalFlags, ...refined.config.gardenCommand] const res = await cli.run({ args: gardenCommand, @@ -263,17 +236,16 @@ export async function getCustomCommands(log: Log, projectRoot: string) { const builtinNames = getBuiltinCommands().flatMap((c) => c.getPaths().map((p) => p.join(" "))) // Filter and validate the resources - const commandResources = resources - .filter((r) => { - if (r.kind !== "Command") { + const commandResources = resources.filter((r) => { + if (r.config.kind !== "Command") { return false } - if (builtinNames.includes(r.name)) { + if (builtinNames.includes(r.config.name)) { // eslint-disable-next-line no-console console.log( styles.warning( - `Ignoring custom command ${r.name} because it conflicts with a built-in command with the same name` + `Ignoring custom command ${r.config.name} because it conflicts with a built-in command with the same name` ) ) return false @@ -281,20 +253,11 @@ export async function getCustomCommands(log: Log, projectRoot: string) { return true }) - .map((config) => - validateWithPath({ - config, - schema: customCommandSchema().keys({ - // Allowing any values here because they're fully resolved later - exec: joi.any(), - gardenCommand: joi.any(), - }), - path: (config).internal.basePath, - projectRoot, - configType: `custom Command '${config.name}'`, - source: { yamlDoc: (config).internal.yamlDoc }, - }) - ) + .map((config) => { + return config.refineWithJoi>( + omitFromSchema(customCommandSchema(), "exec", "gardenCommand") + ) + }) return commandResources.map((spec) => new CustomCommandWrapper(spec)) } diff --git a/core/src/commands/get/get-modules.ts b/core/src/commands/get/get-modules.ts index 48f98073ec..3989f36976 100644 --- a/core/src/commands/get/get-modules.ts +++ b/core/src/commands/get/get-modules.ts @@ -11,7 +11,7 @@ import { Command } from "../base.js" import { StringsParameter, BooleanParameter } from "../../cli/params.js" import type { GardenModule } from "../../types/module.js" import { moduleSchema } from "../../types/module.js" -import { keyBy, omit, sortBy } from "lodash-es" +import { isString, keyBy, omit, sortBy } from "lodash-es" import type { StringMap } from "../../config/common.js" import { joiIdentifierMap, createSchema } from "../../config/common.js" import { printEmoji, printHeader, renderDivider } from "../../logger/util.js" @@ -21,8 +21,9 @@ import { relative, sep } from "path" import type { Garden } from "../../index.js" import type { Log } from "../../logger/log-entry.js" import { highlightYaml, safeDumpYaml } from "../../util/serialization.js" -import { deepMap } from "../../util/objects.js" +import { CollectionOrValue, deepMap } from "../../util/objects.js" import { styles } from "../../logger/styles.js" +import { TemplatePrimitive } from "../../template-string/inputs.js" const getModulesArgs = { modules: new StringsParameter({ @@ -158,11 +159,11 @@ function getRelativeModulePath(projectRoot: string, modulePath: string): string * * Used for sanitizing output that may contain secret values. */ -function filterSecrets(object: T, secrets: StringMap): T { +function filterSecrets>(object: T, secrets: StringMap): T { const secretValues = new Set(Object.values(secrets)) const secretNames = Object.keys(secrets) const sanitized = deepMap(object, (value) => { - if (secretValues.has(value)) { + if (isString(value) && secretValues.has(value)) { const name = secretNames.find((n) => secrets[n] === value)! return `[filtered secret: ${name}]` } else { diff --git a/core/src/commands/plugins.ts b/core/src/commands/plugins.ts index 0f9c3f5c7a..3047b3f8e7 100644 --- a/core/src/commands/plugins.ts +++ b/core/src/commands/plugins.ts @@ -65,7 +65,7 @@ export class PluginsCommand extends Command { } async action({ garden, log, args }: CommandParams): Promise { - const providerConfigs = garden.getRawProviderConfigs() + const providerConfigs = garden.getConfiguredProviders() const configuredPlugins = providerConfigs.map((p) => p.name) if (!args.command || !args.plugin) { diff --git a/core/src/commands/serve.ts b/core/src/commands/serve.ts index f65f75e38e..72964658c9 100644 --- a/core/src/commands/serve.ts +++ b/core/src/commands/serve.ts @@ -102,7 +102,7 @@ export class ServeCommand< const manager = this.getManager(log, undefined) - manager.defaultProjectRoot = projectConfig?.path || process.cwd() + manager.defaultProjectRoot = projectConfig?.configFilePath || process.cwd() manager.defaultEnv = opts.env if (projectConfig) { @@ -121,11 +121,11 @@ export class ServeCommand< this.commandLine.cwd = defaultGarden.projectRoot } } catch (error) { - log.warn(`Unable to load Garden project found at ${projectConfig.path}: ${error}`) + log.warn(`Unable to load Garden project found at ${projectConfig.configFileDirname}: ${error}`) } } - const cloudDomain = getGardenCloudDomain(projectConfig?.domain) + const cloudDomain = getGardenCloudDomain(projectConfig?.config.domain) try { this.server = await startServer({ @@ -169,10 +169,10 @@ export class ServeCommand< } if (projectConfig && cloudApi && defaultGarden) { - let projectId = projectConfig?.id + let projectId = projectConfig?.config.id if (!projectId) { - const cloudProject = await cloudApi.getProjectByName(projectConfig.name) + const cloudProject = await cloudApi.getProjectByName(projectConfig.config.name) projectId = cloudProject?.id } diff --git a/core/src/commands/util/fetch-tools.ts b/core/src/commands/util/fetch-tools.ts index 80c7666bcb..4af83c8b22 100644 --- a/core/src/commands/util/fetch-tools.ts +++ b/core/src/commands/util/fetch-tools.ts @@ -93,7 +93,7 @@ export class FetchToolsCommand extends Command<{}, FetchToolsOpts> { // No need to fetch the same tools multiple times, if they're used in multiple providers const deduplicated = uniqBy(tools, ({ tool }) => tool["versionPath"]) - const versionedConfigs = garden.getRawProviderConfigs({ names: ["pulumi", "terraform"], allowMissing: true }) + const versionedConfigs = garden.getConfiguredProviders({ names: ["pulumi", "terraform"], allowMissing: true }) // If the version of the tool is configured on the provider, // download only that version of the tool. diff --git a/core/src/config/base.ts b/core/src/config/base.ts index fdb4477733..b7aaf1af2e 100644 --- a/core/src/config/base.ts +++ b/core/src/config/base.ts @@ -23,20 +23,33 @@ import { defaultDotIgnoreFile, listDirectory } from "../util/fs.js" import { isConfigFilename } from "../util/fs.js" import type { ConfigTemplateKind } from "./config-template.js" import { isNotNull, isTruthy } from "../util/util.js" -import type { DeepPrimitiveMap, PrimitiveMap } from "./common.js" +import type { PrimitiveMap } from "./common.js" import { createSchema, joi } from "./common.js" import { emitNonRepeatableWarning } from "../warnings.js" import type { ActionKind } from "../actions/types.js" import { actionKinds } from "../actions/types.js" -import { mayContainTemplateString } from "../template-string/template-string.js" +import { + TemplateProvenance, + mayContainTemplateString, +} from "../template-string/template-string.js" import type { Log } from "../logger/log-entry.js" import type { Document, DocumentOptions } from "yaml" import { parseAllDocuments } from "yaml" import { dedent, deline } from "../util/string.js" +import { GardenConfig } from "../template-string/validation.js" +import { ConfigContext, GenericContext } from "./template-contexts/base.js" +import { Collection, CollectionOrValue } from "../util/objects.js" +import { TemplatePrimitive } from "../template-string/inputs.js" +import { inferType, s } from "./zod.js" export const configTemplateKind = "ConfigTemplate" export const renderTemplateKind = "RenderTemplate" export const noTemplateFields = ["apiVersion", "kind", "type", "name", "internal"] +const untemplatableKeys: Record = { + "Workflow": ["triggers"], + "Command": ["args", "opts"], +} + export const varfileDescription = ` The format of the files is determined by the configured file's extension: @@ -52,37 +65,26 @@ export interface YamlDocumentWithSource extends Document { source: string } -export interface GardenResourceInternalFields { - /** - * The path/working directory where commands and operations relating to the config should be executed. This is - * most commonly the directory containing the config file. - * - * Note: WHen possible, use `action.getSourcePath()` instead, since it factors in remote source paths and source - * overrides (i.e. `BaseActionConfig.source.path`). This is a lower-level field that doesn't contain template strings, - * and can thus be used early in the resolution flow. - */ - basePath: string - /** - * The path to the resource's config file, if any. - * - * Configs that are read from a file should always have this set, but generated configs (e.g. from templates - * or `augmentGraph` handlers) don't necessarily have a path on disk. - */ - configFilePath?: string +export type BaseGardenResourceMetadata = { // -> set by templates - inputs?: DeepPrimitiveMap + inputs?: ConfigContext parentName?: string templateName?: string - // Used to map fields to specific doc and location - yamlDoc?: YamlDocumentWithSource } -export interface BaseGardenResource { - apiVersion?: GardenApiVersion - kind: string - name: string - internal: GardenResourceInternalFields -} +export const baseGardenResourceSchema = s.object({ + apiVersion: s.string().optional(), + kind: s.string(), + name: s.string(), + // internal: s + // .object({ + // inputs: s.map(s.string(), s.union([s.string(), s.number()]).nullable()).optional(), + // parentName: s.string().optional(), + // templateName: s.string().optional(), + // }) + // .default({}), +}) +export type BaseGardenResource = inferType export const baseInternalFieldsSchema = createSchema({ name: "base-internal-fields", @@ -160,7 +162,7 @@ export async function loadConfigResources( projectRoot: string, configPath: string, allowInvalid = false -): Promise { +): Promise[]> { const fileData = await readConfigFile(configPath, projectRoot) const resources = await validateRawConfig({ @@ -186,7 +188,7 @@ export async function validateRawConfig({ configPath: string projectRoot: string allowInvalid?: boolean -}) { +}): Promise[]> { let rawSpecs = await loadAndValidateYaml(rawConfig, `${basename(configPath)} in directory ${dirname(configPath)}`) // Ignore empty resources @@ -226,10 +228,10 @@ export function prepareResource({ projectRoot: string description: string allowInvalid?: boolean -}): GardenResource | ModuleConfig | null { +}): GardenConfig | null { const relPath = relative(projectRoot, configFilePath) - const spec = doc.toJS() + let spec = doc.toJS() if (spec === null) { return null @@ -243,8 +245,6 @@ export function prepareResource({ let kind = spec.kind - const basePath = dirname(configFilePath) - if (!allowInvalid) { for (const field of noTemplateFields) { if (spec[field] && mayContainTemplateString(spec[field])) { @@ -266,13 +266,7 @@ export function prepareResource({ } if (kind === "Project") { - spec.path = basePath - spec.configPath = configFilePath - spec.internal = { - basePath, - yamlDoc: doc, - } - return prepareProjectResource(log, spec) + spec = prepareProjectResource(log, spec) } else if ( actionKinds.includes(kind) || kind === "Command" || @@ -280,41 +274,50 @@ export function prepareResource({ kind === configTemplateKind || kind === renderTemplateKind ) { - spec.internal = { - basePath, - configFilePath, - yamlDoc: doc, - } - return spec + // these are allowed } else if (kind === "Module") { - spec.path = basePath spec.configPath = configFilePath delete spec.internal - return prepareModuleResource(spec, configFilePath, projectRoot) + spec = prepareModuleResource(spec, configFilePath, projectRoot) } else if (allowInvalid) { - return spec + // we shouldn't throw } else if (!kind) { throw new ConfigurationError({ message: `Missing \`kind\` field in ${description}`, }) } else { throw new ConfigurationError({ - message: `Unknown kind ${kind} in ${description}`, + message: `Unknown \`kind\` ${kind} in ${description}`, }) } + + return new GardenConfig({ + unparsedConfig: spec, + untemplatableKeys: noTemplateFields.concat(untemplatableKeys[kind] || []), + configFilePath, + context: new GenericContext({}), + metadata: { + // TODO + }, + opts: { + // TODO reconsider the interface to enable / disable partial + allowPartial: true, + }, + source: { yamlDoc: doc, basePath: [] }, + }).refineWithZod(baseGardenResourceSchema) } // TODO-0.14: remove these deprecation handlers in 0.14 -type DeprecatedConfigHandler = (log: Log, spec: ProjectConfig) => ProjectConfig +type DeprecatedConfigHandler = (log: Log, spec: Collection) => Collection -function handleDotIgnoreFiles(log: Log, projectSpec: ProjectConfig) { +function handleDotIgnoreFiles(log: Log, projectSpec: Collection) { // If the project config has an explicitly defined `dotIgnoreFile` field, // it means the config has already been updated to 0.13 format. - if (!!projectSpec.dotIgnoreFile) { + if (!!projectSpec["dotIgnoreFile"]) { return projectSpec } - const dotIgnoreFiles = projectSpec.dotIgnoreFiles + const dotIgnoreFiles = projectSpec["dotIgnoreFiles"] // If the project config has neither new `dotIgnoreFile` nor old `dotIgnoreFiles` fields // then there is nothing to do. if (!dotIgnoreFiles) { @@ -340,8 +343,8 @@ function handleDotIgnoreFiles(log: Log, projectSpec: ProjectConfig) { }) } -function handleProjectModules(log: Log, projectSpec: ProjectConfig): ProjectConfig { - // Field 'modules' was intentionally removed from the internal interface `ProjectConfig`, +function handleProjectModules(log: Log, projectSpec: Collection): Collection { + // Field 'modules' was intentionally removed from the internal interface `Collection`, // but it still can be presented in the runtime if the old config format is used. if (projectSpec["modules"]) { emitNonRepeatableWarning( @@ -353,7 +356,7 @@ function handleProjectModules(log: Log, projectSpec: ProjectConfig): ProjectConf return projectSpec } -function handleMissingApiVersion(log: Log, projectSpec: ProjectConfig): ProjectConfig { +function handleMissingApiVersion(log: Log, projectSpec: Collection): Collection { // We conservatively set the apiVersion to be compatible with 0.12. if (projectSpec["apiVersion"] === undefined) { emitNonRepeatableWarning( @@ -384,8 +387,8 @@ const bonsaiDeprecatedConfigHandlers: DeprecatedConfigHandler[] = [ handleProjectModules, ] -export function prepareProjectResource(log: Log, spec: any): ProjectConfig { - let projectSpec = spec +export function prepareProjectResource(log: Log, spec: any): CollectionOrValue { + let projectSpec = spec for (const handler of bonsaiDeprecatedConfigHandlers) { projectSpec = handler(log, projectSpec) } @@ -475,6 +478,8 @@ export function prepareBuildDependencies(buildDependencies: any[]): BuildDepende .filter(isTruthy) } +export type UnrefinedProjectConfig = GardenConfig> + export async function findProjectConfig({ log, path, @@ -485,10 +490,10 @@ export async function findProjectConfig({ path: string allowInvalid?: boolean scan?: boolean -}): Promise { +}): Promise { const sepCount = path.split(sep).length - 1 - let allProjectSpecs: GardenResource[] = [] + let allProjectSpecs: GardenConfig[] = [] for (let i = 0; i < sepCount; i++) { const configFiles = (await listDirectory(path, { recursive: false })).filter(isConfigFilename) @@ -496,7 +501,7 @@ export async function findProjectConfig({ for (const configFile of configFiles) { const resources = await loadConfigResources(log, path, join(path, configFile), allowInvalid) - const projectSpecs = resources.filter((s) => s.kind === "Project") + const projectSpecs = resources.filter((s) => s.config.kind === "Project") if (projectSpecs.length > 1 && !allowInvalid) { throw new ConfigurationError({ @@ -508,12 +513,18 @@ export async function findProjectConfig({ } if (allProjectSpecs.length > 1 && !allowInvalid) { - const configPaths = allProjectSpecs.map((c) => `- ${(c as ProjectConfig).configPath}`) + const configPaths = allProjectSpecs.map((c) => `- ${c.configFilePath}`) throw new ConfigurationError({ message: `Multiple project declarations found at paths:\n${configPaths.join("\n")}`, }) } else if (allProjectSpecs.length === 1) { - return allProjectSpecs[0] + const source: TemplateProvenance = { + yamlPath: [], + source: {}, + } + return allProjectSpecs[0].refineWithZod( + s.object({ kind: s.literal("Project"), domain: s.string().optional(), id: s.string().optional() }) + ) } if (!scan) { @@ -535,7 +546,7 @@ export async function loadVarfile({ configRoot: string path: string | undefined defaultPath: string | undefined -}): Promise { +}): Promise { if (!path && !defaultPath) { throw new ParameterError({ message: `Neither a path nor a defaultPath was provided. Config root: ${configRoot}`, @@ -551,38 +562,39 @@ export async function loadVarfile({ } if (!exists) { - return {} + return new GenericContext({}) } + let parsed: PrimitiveMap try { const data = await readFile(resolvedPath) const relPath = relative(configRoot, resolvedPath) const filename = basename(resolvedPath.toLowerCase()) if (filename.endsWith(".json")) { - const parsed = JSON.parse(data.toString()) + parsed = JSON.parse(data.toString()) if (!isPlainObject(parsed)) { throw new ConfigurationError({ message: `Configured variable file ${relPath} must be a valid plain JSON object. Got: ${typeof parsed}`, }) } - return parsed } else if (filename.endsWith(".yml") || filename.endsWith(".yaml")) { - const parsed = load(data.toString()) + parsed = load(data.toString()) as PrimitiveMap if (!isPlainObject(parsed)) { throw new ConfigurationError({ message: `Configured variable file ${relPath} must be a single plain YAML mapping. Got: ${typeof parsed}`, }) } - return parsed as PrimitiveMap } else { // Note: For backwards-compatibility we fall back on using .env as a default format, and don't specifically // validate the extension for that. - return dotenv.parse(await readFile(resolvedPath)) + parsed = dotenv.parse(await readFile(resolvedPath)) } } catch (error) { throw new ConfigurationError({ message: `Unable to load varfile at '${path}': ${error}`, }) } + + return new GenericContext(parsed) } diff --git a/core/src/config/command.ts b/core/src/config/command.ts index a09187346f..79d60dda5e 100644 --- a/core/src/config/command.ts +++ b/core/src/config/command.ts @@ -21,21 +21,21 @@ import { unusedApiVersionSchema, } from "./common.js" -interface BaseParameter { +type BaseParameter = { name: string description: string required?: boolean } -export interface CustomCommandArgument extends BaseParameter { +export type CustomCommandArgument = BaseParameter & { type: "string" | "integer" } -export interface CustomCommandOption extends BaseParameter { +export type CustomCommandOption = BaseParameter & { type: CustomCommandArgument["type"] | "boolean" } -export interface CommandResource extends BaseGardenResource { +export type CommandResource = BaseGardenResource & { description: { short: string long?: string diff --git a/core/src/config/common.ts b/core/src/config/common.ts index e240f517cd..3f3ef9a13f 100644 --- a/core/src/config/common.ts +++ b/core/src/config/common.ts @@ -44,15 +44,16 @@ addFormats(ajv) export type Primitive = string | number | boolean | null -export interface StringMap { +export type StringMap = { [key: string]: string } -export interface PrimitiveMap { +export type PrimitiveMap = { [key: string]: Primitive } -export interface DeepPrimitiveMap { +// TODO: Replace with the new `Collection` type. +export type DeepPrimitiveMap = { [key: string]: Primitive | DeepPrimitiveMap | Primitive[] | DeepPrimitiveMap[] } @@ -520,7 +521,7 @@ joi = joi.extend({ }, }) -export interface ActionReference { +export type ActionReference = { kind: K name: string } @@ -656,6 +657,10 @@ export function removeSchema(name: string) { } } +export function omitFromSchema(s: Joi.ObjectSchema, ...keys: string[]) { + return s.extract(Object.keys(s.describe().keys).filter((k) => !keys.includes(k))) +} + /** * Parse, validate and normalize an action reference. * diff --git a/core/src/config/config-template.ts b/core/src/config/config-template.ts index 83966564e5..3209030292 100644 --- a/core/src/config/config-template.ts +++ b/core/src/config/config-template.ts @@ -11,19 +11,18 @@ import { joi, joiUserIdentifier, createSchema, unusedApiVersionSchema } from "./ import type { BaseModuleSpec } from "./module.js" import { baseModuleSpecSchema } from "./module.js" import { dedent, naturalList } from "../util/string.js" -import type { BaseGardenResource } from "./base.js" +import type { BaseGardenResource, BaseGardenResourceMetadata } from "./base.js" import { configTemplateKind, renderTemplateKind, baseInternalFieldsSchema } from "./base.js" -import { resolveTemplateStrings } from "../template-string/template-string.js" -import { validateConfig } from "./validation.js" import type { Garden } from "../garden.js" import { ConfigurationError } from "../exceptions.js" -import { resolve, posix, dirname } from "path" +import { resolve, posix } from "path" import fsExtra from "fs-extra" const { readFile } = fsExtra import { ProjectConfigContext } from "./template-contexts/project.js" import type { ActionConfig } from "../actions/types.js" import { actionKinds } from "../actions/types.js" import type { WorkflowConfig } from "./workflow.js" +import { GardenConfig } from "../template-string/validation.js" const inputTemplatePattern = "${inputs.*}" const parentNameTemplate = "${parent.name}" @@ -40,24 +39,27 @@ export type TemplatableConfigWithPath = TemplatableConfig & { path?: string } export type ConfigTemplateKind = typeof configTemplateKind -interface TemplatedModuleSpec extends Partial { +type TemplatedModuleSpec = Partial & { type: string } -export interface ConfigTemplateResource extends BaseGardenResource { +export type UnrefinedConfigTemplateResource = GardenConfig, BaseGardenResourceMetadata> +export type ConfigTemplateResource = BaseGardenResource & { + kind: ConfigTemplateKind inputsSchemaPath?: string modules?: TemplatedModuleSpec[] configs?: TemplatableConfigWithPath[] } -export interface ConfigTemplateConfig extends ConfigTemplateResource { +export type ResolveConfigTemplateResult = { + refined: GardenConfig inputsSchema: CustomObjectSchema } export async function resolveConfigTemplate( garden: Garden, - resource: ConfigTemplateResource -): Promise { + resource: UnrefinedConfigTemplateResource +): Promise { // Resolve template strings, minus module templates and files const partial = { ...resource, @@ -67,16 +69,10 @@ export async function resolveConfigTemplate( const loggedIn = garden.isLoggedIn() const enterpriseDomain = garden.cloudApi?.domain const context = new ProjectConfigContext({ ...garden, loggedIn, enterpriseDomain }) - const resolved = resolveTemplateStrings({ value: partial, context, source: { yamlDoc: resource.internal.yamlDoc } }) - const configPath = resource.internal.configFilePath - // Validate the partial config - const validated = validateConfig({ - config: resolved, - schema: configTemplateSchema(), - projectRoot: garden.projectRoot, - yamlDocBasePath: [], - }) + const refined = resource.withContext(context).refineWithJoi( + configTemplateSchema() + ) // Read and validate the JSON schema, if specified // -> default to any object @@ -85,15 +81,15 @@ export async function resolveConfigTemplate( additionalProperties: true, } - const configDir = configPath ? dirname(configPath) : resource.internal.basePath + const configDir = refined.configFileDirname! - if (validated.inputsSchemaPath) { - const path = resolve(configDir, ...validated.inputsSchemaPath.split(posix.sep)) + if (refined.config.inputsSchemaPath) { + const path = resolve(configDir, ...refined.config.inputsSchemaPath.split(posix.sep)) try { inputsJsonSchema = JSON.parse((await readFile(path)).toString()) } catch (error) { throw new ConfigurationError({ - message: `Unable to read inputs schema at '${validated.inputsSchemaPath}' for ${configTemplateKind} ${validated.name}: ${error}`, + message: `Unable to read inputs schema at '${refined.config.inputsSchemaPath}' for ${configTemplateKind} ${refined.config.name}: ${error}`, }) } @@ -101,17 +97,15 @@ export async function resolveConfigTemplate( if (type !== "object") { throw new ConfigurationError({ - message: `Inputs schema at '${validated.inputsSchemaPath}' for ${configTemplateKind} ${validated.name} has type ${type}, but should be "object".`, + message: `Inputs schema at '${refined.config.inputsSchemaPath}' for ${configTemplateKind} ${refined.config.name} has type ${type}, but should be "object".`, }) } } // Add the module templates back and return return { - ...validated, + refined, inputsSchema: joi.object().jsonSchema(inputsJsonSchema), - modules: resource.modules, - configs: resource.configs, } } diff --git a/core/src/config/module.ts b/core/src/config/module.ts index a30b004287..65cda795a2 100644 --- a/core/src/config/module.ts +++ b/core/src/config/module.ts @@ -27,11 +27,12 @@ import { testConfigSchema } from "./test.js" import type { TaskConfig } from "./task.js" import { taskConfigSchema } from "./task.js" import { dedent, stableStringify } from "../util/string.js" -import { configTemplateKind, varfileDescription } from "./base.js" +import { BaseGardenResource, configTemplateKind, varfileDescription } from "./base.js" import type { GardenApiVersion } from "../constants.js" import { DEFAULT_BUILD_TIMEOUT_SEC } from "../constants.js" +import { GardenConfig } from "../template-string/validation.js" -interface BuildCopySpec { +type BuildCopySpec = { source: string target: string } @@ -58,7 +59,7 @@ const copySchema = createSchema({ }), }) -export interface BuildDependencyConfig { +export type BuildDependencyConfig = { name: string copy: BuildCopySpec[] } @@ -74,12 +75,12 @@ export const buildDependencySchema = createSchema({ }), }) -export interface BaseBuildSpec { +export type BaseBuildSpec = { dependencies: BuildDependencyConfig[] timeout: number } -export interface GenerateFileSpec { +export type GenerateFileSpec = { sourcePath?: string targetPath: string resolveTemplates: boolean @@ -88,7 +89,7 @@ export interface GenerateFileSpec { export type ModuleSpec = object -interface ModuleSpecCommon { +type ModuleSpecCommon = { apiVersion?: string allowPublish?: boolean build?: BaseBuildSpec @@ -105,7 +106,7 @@ interface ModuleSpecCommon { varfile?: string } -export interface BaseModuleSpec extends ModuleSpecCommon { +export type BaseModuleSpec = ModuleSpecCommon & { /** * the apiVersion field is unused in all Modules at the moment and hidden in the reference docs. */ @@ -264,8 +265,13 @@ export const baseModuleSpecSchema = createSchema({ keys: baseModuleSpecKeys, }) -export interface ModuleConfig - extends BaseModuleSpec { +export type UnrefinedModuleConfig = GardenConfig> +export type ModuleConfig< + M extends {} = any, + S extends {} = any, + T extends {} = any, + W extends {} = any, +> = BaseModuleSpec & { path: string configPath?: string basePath?: string // The directory of the config. Disambiguates `path` when the module has a remote source. diff --git a/core/src/config/project.ts b/core/src/config/project.ts index 6a86db36eb..ba5420a10d 100644 --- a/core/src/config/project.ts +++ b/core/src/config/project.ts @@ -20,16 +20,14 @@ import { joiUserIdentifier, joiVariables, joiVariablesDescription, + omitFromSchema, } from "./common.js" -import { validateConfig, validateWithPath } from "./validation.js" -import { resolveTemplateStrings } from "../template-string/template-string.js" import { EnvironmentConfigContext, ProjectConfigContext } from "./template-contexts/project.js" import { findByName, getNames } from "../util/util.js" -import { ConfigurationError, ParameterError, ValidationError } from "../exceptions.js" -import cloneDeep from "fast-copy" +import { ConfigurationError, InternalError, ParameterError, ValidationError } from "../exceptions.js" import { memoize } from "lodash-es" import type { GenericProviderConfig } from "./provider.js" -import { providerConfigBaseSchema } from "./provider.js" +import { baseProviderConfigSchemaZod, providerConfigBaseSchema } from "./provider.js" import type { GitScanMode } from "../constants.js" import { DOCS_BASE_URL, GardenApiVersion, gitScanModes } from "../constants.js" import { defaultDotIgnoreFile } from "../util/fs.js" @@ -39,8 +37,10 @@ import { profileAsync } from "../util/profiling.js" import type { BaseGardenResource } from "./base.js" import { baseInternalFieldsSchema, loadVarfile, varfileDescription } from "./base.js" import type { Log } from "../logger/log-entry.js" -import { renderDivider } from "../logger/util.js" import { styles } from "../logger/styles.js" +import { GardenConfig } from "../template-string/validation.js" +import { s } from "./zod.js" +import { ConfigContext, GenericContext, LayeredContext } from "./template-contexts/base.js" export const defaultVarfilePath = "garden.env" export const defaultEnvVarfilePath = (environmentName: string) => `garden.${environmentName}.env` @@ -55,7 +55,7 @@ export interface ParsedEnvironment { namespace?: string } -export interface EnvironmentConfig { +export type EnvironmentConfig = { name: string defaultNamespace: string | null varfile?: string @@ -125,7 +125,7 @@ export const environmentsSchema = memoize(() => joiSparseArray(environmentSchema()).unique("name").description("A list of environments to configure for the project.") ) -export interface SourceConfig { +export type SourceConfig = { name: string repositoryUrl: string } @@ -182,27 +182,25 @@ export const linkedModuleSchema = createSchema({ }), }) -export interface OutputSpec { +export type OutputSpec = { name: string value: Primitive } -export interface ProxyConfig { +export type ProxyConfig = { hostname: string } -interface GitConfig { +type GitConfig = { mode: GitScanMode } -export interface ProjectConfig extends BaseGardenResource { +export type ProjectConfig = Readonly export const projectNameSchema = memoize(() => joiIdentifier().required().description("The name of the project.").example("my-sweet-project") @@ -307,10 +305,8 @@ export const projectSchema = createSchema({ Note that the value ${GardenApiVersion.v1} will break compatibility of your project with Garden Acorn (0.12). - `), + `).required(), // We set the default before validation in `handleMissingApiVersion` kind: joi.string().default("Project").valid("Project").description("Indicate what kind of config this is."), - path: projectRootSchema().meta({ internal: true }), - configPath: joi.string().meta({ internal: true }).description("The path to the project config file."), internal: baseInternalFieldsSchema(), name: projectNameSchema(), // TODO: Refer to enterprise documentation for more details. @@ -335,8 +331,7 @@ export const projectSchema = createSchema({ ), defaultEnvironment: joi .environment() - .allow("") - .default("") + .default((parent) => parent.environments[0].name) .description( deline` The default environment to use when calling commands without the \`--env\` parameter. @@ -427,7 +422,8 @@ export const projectSchema = createSchema({ rename: [["modules", "scan"]], }) -export function getDefaultEnvironmentName(defaultName: string, config: ProjectConfig): string { +type ProjectConfigWithEnvironments = { environments: { name: string }[]; defaultEnvironment: string } +export function getDefaultEnvironmentName(defaultName: string, config: ProjectConfigWithEnvironments): string { const environments = config.environments // the default environment is the first specified environment in the config, unless specified @@ -446,6 +442,9 @@ export function getDefaultEnvironmentName(defaultName: string, config: ProjectCo } } +type ProjectConfigWithoutEnvironmentsSourcesAndProviders = GardenConfig< + Omit +> /** * Resolves and validates the given raw project configuration, and returns it in a canonical form. * @@ -454,8 +453,6 @@ export function getDefaultEnvironmentName(defaultName: string, config: ProjectCo * @param config raw project configuration */ export function resolveProjectConfig({ - log, - defaultEnvironmentName, config, artifactsPath, vcsInfo, @@ -467,7 +464,7 @@ export function resolveProjectConfig({ }: { log: Log defaultEnvironmentName: string - config: ProjectConfig + config: GardenConfig> artifactsPath: string vcsInfo: VcsInfo username: string @@ -475,72 +472,33 @@ export function resolveProjectConfig({ enterpriseDomain: string | undefined secrets: PrimitiveMap commandInfo: CommandInfo -}): ProjectConfig { - // Resolve template strings for non-environment-specific fields (apart from `sources`). - const { environments = [], name, sources = [] } = config - - let globalConfig: any - try { - globalConfig = resolveTemplateStrings({ - value: { - apiVersion: config.apiVersion, - varfile: config.varfile, - variables: config.variables, - environments: [], - sources: [], - }, - context: new ProjectConfigContext({ - projectName: name, - projectRoot: config.path, - artifactsPath, - vcsInfo, - username, - loggedIn, - enterpriseDomain, - secrets, - commandInfo, - }), - source: { yamlDoc: config.internal.yamlDoc, basePath: [] }, +}): GardenConfig> { + if (!config.configFileDirname) { + throw new InternalError({ + message: "Could not determine project root", }) - } catch (err) { - log.error("Failed to resolve project configuration.") - log.error(styles.bold(renderDivider())) - throw err } - // Validate after resolving global fields - config = validateConfig({ - config: { - ...config, - ...globalConfig, - name, - defaultEnvironment: defaultEnvironmentName, - // environments are validated later - environments: [{ defaultNamespace: null, name: "fake-env-only-here-for-inital-load", variables: {} }], - sources: [], - }, - schema: projectSchema(), - projectRoot: config.path, - yamlDocBasePath: [], + const context = new ProjectConfigContext({ + projectName: config.config.name, + projectRoot: config.configFileDirname, + artifactsPath, + vcsInfo, + username, + loggedIn, + enterpriseDomain, + secrets, + commandInfo, }) - const providers = config.providers - - // This will be validated separately, after resolving templates - config.environments = environments - - config = { - ...config, - environments: config.environments, - providers, - sources, - } - - config.defaultEnvironment = getDefaultEnvironmentName(defaultEnvironmentName, config) + const partialSchema = omitFromSchema(projectSchema(), "environments", "sources") + type PartialProjectConfig = Omit - return config + return config.withContext(context).refineWithJoi(partialSchema) } +export type ProjectConfigWithoutSources = GardenConfig> + /** * Given an environment name, pulls the relevant environment-specific configuration from the specified project * config, and merges values appropriately. Also resolves template strings in the picked environment. @@ -578,7 +536,7 @@ export const pickEnvironment = profileAsync(async function _pickEnvironment({ secrets, commandInfo, }: { - projectConfig: ProjectConfig + projectConfig: ProjectConfigWithoutEnvironmentsSourcesAndProviders envString: string artifactsPath: string vcsInfo: VcsInfo @@ -587,17 +545,67 @@ export const pickEnvironment = profileAsync(async function _pickEnvironment({ enterpriseDomain: string | undefined secrets: PrimitiveMap commandInfo: CommandInfo -}) { - const { environments, name: projectName, path: projectRoot } = projectConfig +}): Promise<{ + environmentName: string + namespace: string + defaultNamespace: string | null + production: boolean + variables: ConfigContext + refinedConfig: ProjectConfigWithoutSources, +}> { + const projectRoot = projectConfig.configFileDirname + + if (!projectRoot) { + throw new InternalError({ + message: "Could not determine project root", + }) + } + + const { name: projectName, variables } = projectConfig.config + + const projectVarfileVars = await loadVarfile({ + configRoot: projectRoot, + path: projectConfig.config.varfile, + defaultPath: defaultVarfilePath, + }) + + const environmentConfigContextVariables = new LayeredContext( + // projectVarfileVars has precendence over the variables declared in project.garden.yml + projectVarfileVars, + new GenericContext(variables) + ) + + const context = new EnvironmentConfigContext({ + projectName, + projectRoot, + artifactsPath, + vcsInfo, + username, + variables: environmentConfigContextVariables, + loggedIn, + enterpriseDomain, + secrets, + commandInfo, + }) + + const partialSchema = omitFromSchema(projectSchema(), "sources") + type PartialProjectConfig = Omit + const refined = projectConfig + .withContext(context) + .refineWithJoi(partialSchema) + .refineWithZod( + s.object({ + providers: s.array(baseProviderConfigSchemaZod).optional(), + }) + ) + const parsed = parseEnvironment(envString) const { environment } = parsed let { namespace } = parsed let environmentConfig: EnvironmentConfig | undefined - let index = -1 - for (const env of environments) { - index++ + for (const env of refined.config.environments) { if (env.name === environment) { environmentConfig = env break @@ -605,7 +613,7 @@ export const pickEnvironment = profileAsync(async function _pickEnvironment({ } if (!environmentConfig) { - const definedEnvironments = getNames(environments) + const definedEnvironments = getNames(refined.config.environments) throw new ParameterError({ message: `Project ${projectName} does not specify environment ${environment} (Available environments: ${naturalList( @@ -614,76 +622,25 @@ export const pickEnvironment = profileAsync(async function _pickEnvironment({ }) } - const projectVarfileVars = await loadVarfile({ - configRoot: projectConfig.path, - path: projectConfig.varfile, - defaultPath: defaultVarfilePath, - }) - const projectVariables: DeepPrimitiveMap = merge(projectConfig.variables, projectVarfileVars) - - const source = { yamlDoc: projectConfig.internal.yamlDoc, basePath: ["environments", index] } - - // Resolve template strings in the environment config, except providers - environmentConfig = resolveTemplateStrings({ - value: { ...environmentConfig }, - context: new EnvironmentConfigContext({ - projectName, - projectRoot, - artifactsPath, - vcsInfo, - username, - variables: projectVariables, - loggedIn, - enterpriseDomain, - secrets, - commandInfo, - }), - source, - }) - - environmentConfig = validateWithPath({ - config: environmentConfig, - schema: environmentSchema(), - configType: `environment ${environment}`, - path: projectConfig.path, - projectRoot: projectConfig.path, - source, - }) - namespace = getNamespace(environmentConfig, namespace) - const fixedProviders = fixedPlugins.map((name) => ({ name })) - const allProviders = [ - ...fixedProviders, - ...projectConfig.providers.filter((p) => !p.environments || p.environments.includes(environment)), - ] - - const mergedProviders: { [name: string]: GenericProviderConfig } = {} - - for (const provider of allProviders) { - if (!!mergedProviders[provider.name]) { - // Merge using a JSON Merge Patch (see https://tools.ietf.org/html/rfc7396) - apply(mergedProviders[provider.name], provider) - } else { - mergedProviders[provider.name] = cloneDeep(provider) - } - } - const envVarfileVars = await loadVarfile({ - configRoot: projectConfig.path, + configRoot: projectConfig.configFileDirname, path: environmentConfig.varfile, defaultPath: defaultEnvVarfilePath(environment), }) - const variables: DeepPrimitiveMap = merge(projectVariables, merge(environmentConfig.variables, envVarfileVars)) - return { environmentName: environment, namespace, defaultNamespace: environmentConfig.defaultNamespace, production: !!environmentConfig.production, - providers: Object.values(mergedProviders), - variables, + variables: new LayeredContext( + envVarfileVars, + new GenericContext(environmentConfig.variables), + environmentConfigContextVariables + ), + refinedConfig: refined, } }) diff --git a/core/src/config/provider.ts b/core/src/config/provider.ts index 6a14d94a42..a288a820d2 100644 --- a/core/src/config/provider.ts +++ b/core/src/config/provider.ts @@ -29,7 +29,7 @@ import { dashboardPagesSchema } from "../plugin/handlers/Provider/getDashboardPa import type { ActionState } from "../actions/types.js" import type { ValidResultType } from "../tasks/base.js" import { uuidv4 } from "../util/random.js" -import { s } from "./zod.js" +import { type inferType, s } from "./zod.js" // TODO: dedupe from the joi schema below export const baseProviderConfigSchemaZod = s.object({ @@ -51,12 +51,7 @@ export const baseProviderConfigSchemaZod = s.object({ .example(["dev", "stage"]), }) -export interface BaseProviderConfig { - name: string - dependencies?: string[] - environments?: string[] -} - +export type BaseProviderConfig = inferType export interface GenericProviderConfig extends BaseProviderConfig { [key: string]: any } diff --git a/core/src/config/render-template.ts b/core/src/config/render-template.ts index 23143cd503..377ed936f8 100644 --- a/core/src/config/render-template.ts +++ b/core/src/config/render-template.ts @@ -7,9 +7,9 @@ */ import { Document } from "yaml" -import type { ModuleConfig } from "./module.js" +import { type UnrefinedModuleConfig, coreModuleSpecSchema, BaseModuleSpec } from "./module.js" import { dedent, deline, naturalList } from "../util/string.js" -import type { BaseGardenResource, RenderTemplateKind, YamlDocumentWithSource } from "./base.js" +import type { BaseGardenResource, BaseGardenResourceMetadata, RenderTemplateKind, YamlDocumentWithSource } from "./base.js" import { baseInternalFieldsSchema, configTemplateKind, @@ -20,24 +20,26 @@ import { import { maybeTemplateString, resolveTemplateString, - resolveTemplateStrings, } from "../template-string/template-string.js" -import { validateWithPath } from "./validation.js" import type { Garden } from "../garden.js" -import { ConfigurationError, GardenError } from "../exceptions.js" +import { ConfigurationError, GardenError, InternalError } from "../exceptions.js" import { resolve, posix } from "path" -import fsExtra from "fs-extra" +import fsExtra, { ensureDirSync } from "fs-extra" const { ensureDir } = fsExtra -import type { TemplatedModuleConfig } from "../plugins/templated.js" -import { omit } from "lodash-es" import { EnvironmentConfigContext } from "./template-contexts/project.js" -import type { ConfigTemplateConfig, TemplatableConfig } from "./config-template.js" +import type { ResolveConfigTemplateResult, TemplatableConfig } from "./config-template.js" import { templatableKinds, templateNoTemplateFields } from "./config-template.js" -import { createSchema, joi, joiIdentifier, joiUserIdentifier, unusedApiVersionSchema } from "./common.js" +import { createSchema, joi, joiIdentifier, joiUserIdentifier, omitFromSchema, unusedApiVersionSchema } from "./common.js" import type { DeepPrimitiveMap } from "@garden-io/platform-api-types" import { RenderTemplateConfigContext } from "./template-contexts/render.js" import type { Log } from "../logger/log-entry.js" -import { GardenApiVersion } from "../constants.js" +import { GardenConfig } from "../template-string/validation.js" +import { evaluate } from "../template-string/lazy.js" +import { isPlainObject } from "../util/objects.js" +import { TemplateLeaf } from "../template-string/inputs.js" +import { GenericContext } from "./template-contexts/base.js" +import { BaseActionConfigMetadata } from "../actions/types.js" +import { s } from "./zod.js" export const renderTemplateConfigSchema = createSchema({ name: renderTemplateKind, @@ -60,7 +62,7 @@ export const renderTemplateConfigSchema = createSchema({ }), }) -export interface RenderTemplateConfig extends BaseGardenResource { +export type RenderTemplateConfig = BaseGardenResource & { kind: RenderTemplateKind disabled?: boolean template: string @@ -85,26 +87,41 @@ export const templatedModuleSpecSchema = createSchema({ }), }) -export function convertTemplatedModuleToRender(config: TemplatedModuleConfig): RenderTemplateConfig { - return { - apiVersion: config.apiVersion || GardenApiVersion.v0, - kind: renderTemplateKind, - name: config.name, - disabled: config.disabled, +export function convertTemplatedModuleToRender(config: UnrefinedModuleConfig): GardenConfig { + return config.transformParsedConfig((config, context, opts) => { + const evaluated = evaluate({ value: config, context, opts }) - internal: { - basePath: config.path, - configFilePath: config.configPath, - }, + if (!isPlainObject(evaluated)) { + throw new InternalError({ + message: `Expected a plain object`, + }) + } - template: config.spec.template, - inputs: config.spec.inputs, - } + const spec = evaluate({ value: evaluated.spec, context, opts }) + + if (!isPlainObject(spec)) { + throw new InternalError({ + message: `Expected a plain object`, + }) + } + + return { + apiVersion: evaluated.apiVersion || TemplateLeaf.from(undefined), + kind: TemplateLeaf.from(renderTemplateKind), + name: evaluated.name || TemplateLeaf.from(undefined), + disabled: evaluated.disabled || TemplateLeaf.from(undefined), + + template: spec.template || TemplateLeaf.from(undefined), + inputs: spec.inputs || TemplateLeaf.from(undefined), + } + }).refineWithJoi(renderTemplateConfigSchema()) } -export interface RenderConfigTemplateResult { - resolved: RenderTemplateConfig - modules: ModuleConfig[] +export type RefinedRenderTemplateConfig = GardenConfig, BaseActionConfigMetadata> + +export type RenderConfigTemplateResult = { + refined: RefinedRenderTemplateConfig + modules: GardenConfig[] configs: TemplatableConfig[] } @@ -116,8 +133,8 @@ export async function renderConfigTemplate({ }: { garden: Garden log: Log - config: RenderTemplateConfig - templates: { [name: string]: ConfigTemplateConfig } + config: GardenConfig, BaseActionConfigMetadata> + templates: { [name: string]: ResolveConfigTemplateResult } }): Promise { // Resolve template strings for fields. Note that inputs are partially resolved, and will be fully resolved later // when resolving the resulting modules. Inputs that are used in module names must however be resolvable @@ -126,50 +143,31 @@ export async function renderConfigTemplate({ const enterpriseDomain = garden.cloudApi?.domain const templateContext = new EnvironmentConfigContext({ ...garden, loggedIn, enterpriseDomain }) - const yamlDoc = config.internal.yamlDoc + const withInputs = config.withContext(templateContext).refineWithZod(s.object({ + inputs: s.map(s.string(), s.any()), + })) - const resolvedWithoutInputs = resolveTemplateStrings({ - value: { ...omit(config, "inputs") }, - context: templateContext, - source: { yamlDoc }, - }) - const partiallyResolvedInputs = resolveTemplateStrings({ - value: config.inputs || {}, - context: templateContext, - contextOpts: { - allowPartial: true, - }, - source: { yamlDoc, basePath: ["inputs"] }, - }) - let resolved: RenderTemplateConfig = { - ...resolvedWithoutInputs, - inputs: partiallyResolvedInputs, - } + // inputs are needed to continue to resolve the actions that result from this template config + // in later stages, the inputs need to be added to the respective resolve contexts. + const inputs = new GenericContext(withInputs.config.inputs) + config.metadata.inputs = inputs - const configType = "Render " + resolved.name + const refined = config.withContext(templateContext).refineWithJoi>( + omitFromSchema(renderTemplateConfigSchema(), "inputs") + ) // Return immediately if config is disabled - if (resolved.disabled) { - return { resolved, modules: [], configs: [] } + if (refined.config.disabled) { + return { refined, modules: [], configs: [] } } - // Validate the module spec - resolved = validateWithPath({ - config: resolved, - configType, - path: resolved.internal.configFilePath || resolved.internal.basePath, - schema: renderTemplateConfigSchema(), - projectRoot: garden.projectRoot, - source: undefined, - }) - - const template = templates[resolved.template] + const template = templates[refined.config.template] if (!template) { const availableTemplates = Object.keys(templates) throw new ConfigurationError({ message: deline` - Render ${resolved.name} references template ${resolved.template} which cannot be found. + Render ${refined.config.name} references template ${refined.config.template} which cannot be found. Available templates: ${naturalList(availableTemplates)} `, }) @@ -180,17 +178,17 @@ export async function renderConfigTemplate({ ...garden, loggedIn: garden.isLoggedIn(), enterpriseDomain, - parentName: resolved.name, - templateName: template.name, - inputs: partiallyResolvedInputs, + parentName: refined.config.name, + templateName: template.refined.config.name, + inputs, }) // TODO: remove in 0.14 - const modules = await renderModules({ garden, template, context, renderConfig: resolved }) + const modules = await renderModules({ garden, template, context, renderConfig: refined }) - const configs = await renderConfigs({ garden, log, template, context, renderConfig: resolved }) + const configs = await renderConfigs({ garden, log, template, context, renderConfig: refined }) - return { resolved, modules, configs } + return { refined, modules, configs } } async function renderModules({ @@ -200,61 +198,80 @@ async function renderModules({ renderConfig, }: { garden: Garden - template: ConfigTemplateConfig + template: ResolveConfigTemplateResult context: RenderTemplateConfigContext - renderConfig: RenderTemplateConfig -}): Promise { - const yamlDoc = template.internal.yamlDoc - + renderConfig: RefinedRenderTemplateConfig +}): Promise[]> { return Promise.all( - (template.modules || []).map(async (m, i) => { + (template.refined.config.modules || []).map(async (m, i) => { + // TODO: capture the context instead of partial resolution, if needed. // Run a partial template resolution with the parent+template info - const spec = resolveTemplateStrings({ - value: m, - context, - contextOpts: { allowPartial: true }, - source: { yamlDoc, basePath: ["modules", i] }, - }) - const renderConfigPath = renderConfig.internal.configFilePath || renderConfig.internal.basePath + // const spec = resolveTemplateStrings({ + // value: m, + // context, + // contextOpts: { allowPartial: true }, + // source: { yamlDoc, basePath: ["modules", i] }, + // }) + + // TODO: what about basePath? + const renderConfigPath = renderConfig.configFileDirname! //|| renderConfig.metadata.basePath + + // TODO: Perform transformation on a different level of abstraction for improved safety. + const moduleResource = template.refined.transformUnparsedConfig((config, _context, _opts) => { + if (!isPlainObject(config)) { + throw new InternalError({ + message: `Expected a plain object`, + }) + } - let moduleConfig: ModuleConfig + const spec = config["modules"]?.[i] - try { - moduleConfig = prepareModuleResource(spec, renderConfigPath, garden.projectRoot) - } catch (error) { - if (!(error instanceof GardenError)) { - throw error + if (!isPlainObject(spec)) { + throw new InternalError({ + message: `Expected a plain object`, + }) } - let msg = error.message - if (spec.name && spec.name.includes && spec.name.includes("${")) { - msg += - ". Note that if a template string is used in the name of a module in a template, then the template string must be fully resolvable at the time of module scanning. This means that e.g. references to other modules or runtime outputs cannot be used." + let moduleConfig: BaseModuleSpec + + try { + moduleConfig = prepareModuleResource(spec, renderConfigPath, garden.projectRoot) + } catch (error) { + if (!(error instanceof GardenError)) { + throw error + } + let msg = error.message + + if (spec.name && spec.name.includes && spec.name.includes("${")) { + msg += + ". Note that if a template string is used in the name of a module in a template, then the template string must be fully resolvable at the time of module scanning. This means that e.g. references to other modules or runtime outputs cannot be used." + } + + throw new ConfigurationError({ + message: `${configTemplateKind} ${template.refined.config.name} returned an invalid module (named ${spec.name}) for templated module ${renderConfig.config.name}: ${msg}`, + }) } - throw new ConfigurationError({ - message: `${configTemplateKind} ${template.name} returned an invalid module (named ${spec.name}) for templated module ${renderConfig.name}: ${msg}`, - }) - } + // Resolve the file source path to an absolute path, so that it can be used during module resolution + moduleConfig.generateFiles = (moduleConfig.generateFiles || []).map((f) => ({ + ...f, + sourcePath: f.sourcePath && resolve(template.refined.metadata.basePath, ...f.sourcePath.split(posix.sep)), + })) - // Resolve the file source path to an absolute path, so that it can be used during module resolution - moduleConfig.generateFiles = (moduleConfig.generateFiles || []).map((f) => ({ - ...f, - sourcePath: f.sourcePath && resolve(template.internal.basePath, ...f.sourcePath.split(posix.sep)), - })) + // If a path is set, resolve the path and ensure that directory exists + if (spec.path) { + moduleConfig.path = resolve(template.refined.metadata.basePath, ...spec.path.split(posix.sep)) + ensureDirSync(moduleConfig.path) + } - // If a path is set, resolve the path and ensure that directory exists - if (spec.path) { - moduleConfig.path = resolve(renderConfig.internal.basePath, ...spec.path.split(posix.sep)) - await ensureDir(moduleConfig.path) - } + return moduleConfig + }) - // Attach metadata - moduleConfig.parentName = renderConfig.name - moduleConfig.templateName = template.name - moduleConfig.inputs = renderConfig.inputs + moduleResource.metadata.parentName = renderConfig.config.name + moduleResource.metadata.templateName = template.refined.config.name + moduleResource.metadata.inputs = renderConfig.metadata.inputs - return moduleConfig + return moduleResource.refineWithJoi(coreModuleSpecSchema()) }) ) } @@ -268,9 +285,9 @@ async function renderConfigs({ }: { garden: Garden log: Log - template: ConfigTemplateConfig + template: ResolveConfigTemplateResult context: RenderTemplateConfigContext - renderConfig: RenderTemplateConfig + renderConfig: RefinedRenderTemplateConfig }): Promise { const templateDescription = `${configTemplateKind} '${template.name}'` @@ -307,10 +324,10 @@ async function renderConfigs({ const spec = { ...m, name: resolvedName } const renderConfigPath = renderConfig.internal.configFilePath || renderConfig.internal.basePath - let resource: TemplatableConfig + let resource: GardenConfig try { - resource = prepareResource({ + resource = prepareResource({ log, doc: new Document(spec) as YamlDocumentWithSource, configFilePath: renderConfigPath, diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index 915dcc67b4..695b9d07b6 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -7,24 +7,27 @@ */ import type Joi from "@hapi/joi" -import { isString } from "lodash-es" -import { ConfigurationError } from "../../exceptions.js" -import { - resolveTemplateString, - TemplateStringMissingKeyException, - TemplateStringPassthroughException, -} from "../../template-string/template-string.js" +import { ConfigurationError, InternalError } from "../../exceptions.js" import type { CustomObjectSchema } from "../common.js" import { isPrimitive, joi, joiIdentifier } from "../common.js" import { KeyedSet } from "../../util/keyed-set.js" import { naturalList } from "../../util/string.js" import { styles } from "../../logger/styles.js" +import type { TemplateValue } from "../../template-string/inputs.js" +import { TemplateLeaf, isTemplateLeafValue, isTemplateLeaf } from "../../template-string/inputs.js" +import type { CollectionOrValue } from "../../util/objects.js" +import { deepMap, isPlainObject } from "../../util/objects.js" +import { LazyValue } from "../../template-string/lazy.js" +import { GardenConfig } from "../../template-string/validation.js" export type ContextKeySegment = string | number export type ContextKey = ContextKeySegment[] +export type ObjectPath = (string | number)[] + export interface ContextResolveOpts { // Allow templates to be partially resolved (used to defer runtime template resolution, for example) + // TODO: Remove this from context resolve opts: The context does not care if we resolve template strings partially. allowPartial?: boolean // a list of previously resolved paths, used to detect circular references stack?: string[] @@ -41,7 +44,10 @@ export interface ContextResolveParams { export interface ContextResolveOutput { message?: string partial?: boolean - resolved: any + result: CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND + cached: boolean + // for input tracking + // ResolvedResult: ResolvedResult } export function schema(joiSchema: Joi.Schema) { @@ -55,18 +61,17 @@ export interface ConfigContextType { getSchema(): CustomObjectSchema } +export const CONTEXT_RESOLVE_KEY_NOT_FOUND = Symbol.for("ContextResolveKeyNotFound") + // Note: we're using classes here to be able to use decorators to describe each context node and key +// TODO-steffen&thor: Make all instance variables of all config context classes read-only. export abstract class ConfigContext { private readonly _rootContext: ConfigContext - private readonly _resolvedValues: { [path: string]: any } - - // This is used for special-casing e.g. runtime.* resolution - protected _alwaysAllowPartial: boolean + private readonly _resolvedValues: { [path: string]: CollectionOrValue } constructor(rootContext?: ConfigContext) { this._rootContext = rootContext || this this._resolvedValues = {} - this._alwaysAllowPartial = false } static getSchema() { @@ -87,10 +92,10 @@ export abstract class ConfigContext { const fullPath = renderKeyPath(nodePath.concat(key)) // if the key has previously been resolved, return it directly - const resolved = this._resolvedValues[path] + const cachedResult = this._resolvedValues[path] - if (resolved) { - return { resolved } + if (cachedResult) { + return { cached: true, result: cachedResult } } opts.stack = [...(opts.stack || [])] @@ -150,17 +155,20 @@ export abstract class ConfigContext { if (remainder.length > 0) { opts.stack.push(stackEntry) const res = value.resolve({ key: remainder, nodePath: nestedNodePath, opts }) - value = res.resolved + value = res.result message = res.message partial = !!res.partial + } else { + // TODO: improve error message + throw new ConfigurationError({ + message: `Resolving to a context is not allowed.`, + }) } break } - // handle templated strings in context variables - if (isString(value)) { - opts.stack.push(stackEntry) - value = resolveTemplateString({ string: value, context: this._rootContext, contextOpts: opts }) + if (isTemplateLeaf(value) || value instanceof LazyValue) { + break } if (value === undefined) { @@ -186,31 +194,86 @@ export abstract class ConfigContext { } } - // If we're allowing partial strings, we throw the error immediately to end the resolution flow. The error - // is caught in the surrounding template resolution code. - if (this._alwaysAllowPartial) { - // We use a separate exception type when contexts are specifically indicating that unresolvable keys should - // be passed through. This is caught in the template parser code. - throw new TemplateStringPassthroughException({ - message, - }) - } else if (opts.allowPartial) { - throw new TemplateStringMissingKeyException({ - message, - }) - } else { - // Otherwise we return the undefined value, so that any logical expressions can be evaluated appropriately. - // The template resolver will throw the error later if appropriate. - return { resolved: undefined, message } + return { + message, + cached: false, + result: CONTEXT_RESOLVE_KEY_NOT_FOUND, } } + let result: CollectionOrValue + + if (value instanceof LazyValue) { + result = value + } else if (isTemplateLeaf(value)) { + result = value + } + // Wrap normal data using deepMap + else if (isTemplateLeafValue(value)) { + result = new TemplateLeaf({ + expr: undefined, + value, + inputs: {}, + }) + } else { + // value is a collection + result = deepMap(value, (v) => { + if (isTemplateLeaf(v) || v instanceof LazyValue) { + return v + } + return new TemplateLeaf({ + expr: undefined, + value: v, + inputs: {}, + }) + }) + } + // Cache result, unless it is a partial resolution if (!partial) { - this._resolvedValues[path] = value + this._resolvedValues[path] = result + } + + return { cached: false, result } + } +} + +/** + * LayeredContext takes a list of contexts, and tries to resolve a key in each of them, in order. + * + * It returns the first value that successfully resolved. + */ +export class LayeredContext extends ConfigContext { + private readonly _layers: ConfigContext[] + + constructor(...layers: ConfigContext[]) { + super() + this._layers = layers + } + + override resolve({ key, nodePath, opts }: ContextResolveParams): ContextResolveOutput { + let res: ContextResolveOutput = { cached: false, result: CONTEXT_RESOLVE_KEY_NOT_FOUND } + + for (const [index, layer] of this._layers.entries()) { + const isLastLayer = index === this._layers.length - 1 + + res = layer.resolve({ + key, + nodePath, + opts: { + ...opts, + // Throw an error if we can't find the value in the last layer, unless allowPartial is set + allowPartial: isLastLayer ? opts.allowPartial : false, + }, + }) + + // break if we successfully resolved something + if (res.result !== CONTEXT_RESOLVE_KEY_NOT_FOUND) { + break + } } - return { resolved: value } + return res } } @@ -220,7 +283,20 @@ export abstract class ConfigContext { export class GenericContext extends ConfigContext { constructor(obj: any) { super() - Object.assign(this, obj) + + // If we pass in template variables, we want to store the underlying template value tree + // Otherwise we lose input tracking information + const templateValueTree = GardenConfig.getTemplateValueTree(obj) + + if (templateValueTree) { + // Lazy variables must be in a plain object. + if (!isPlainObject(templateValueTree)) { + throw new InternalError({ message: "Expected template value tree to be a plain object" }) + } + Object.assign(this, templateValueTree) + } else { + Object.assign(this, obj) + } } static override getSchema() { @@ -242,7 +318,11 @@ export class ScanContext extends ConfigContext { override resolve({ key, nodePath }: ContextResolveParams) { const fullKey = nodePath.concat(key) this.foundKeys.add(fullKey) - return { resolved: renderTemplateString(fullKey), partial: true } + return { + partial: true, + cached: false, + result: new TemplateLeaf({ value: renderTemplateString(fullKey), expr: undefined, inputs: {} }), + } } } @@ -319,7 +399,7 @@ function renderTemplateString(key: ContextKeySegment[]) { /** * Given all the segments of a template string, return a string path for the key. */ -function renderKeyPath(key: ContextKeySegment[]): string { +export function renderKeyPath(key: ContextKeySegment[]): string { // Note: We don't support bracket notation for the first part in a template string if (key.length === 0) { return "" diff --git a/core/src/config/template-contexts/custom-command.ts b/core/src/config/template-contexts/custom-command.ts index ddbe3e7b9e..4b4737ae7d 100644 --- a/core/src/config/template-contexts/custom-command.ts +++ b/core/src/config/template-contexts/custom-command.ts @@ -6,12 +6,11 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import type { DeepPrimitiveMap } from "../common.js" import { variableNameRegex, joiPrimitive, joiArray, joiVariables, joiIdentifierMap } from "../common.js" import { joi } from "../common.js" import type { DefaultEnvironmentContextParams } from "./project.js" import { DefaultEnvironmentContext } from "./project.js" -import { schema } from "./base.js" +import { type ConfigContext, schema } from "./base.js" interface ArgsSchema { [name: string]: string | number | string[] @@ -30,10 +29,10 @@ export class CustomCommandContext extends DefaultEnvironmentContext { .description("A map of all variables defined in the command configuration.") .meta({ keyPlaceholder: "" }) ) - public variables: DeepPrimitiveMap + public variables: ConfigContext @schema(joiIdentifierMap(joiPrimitive()).description("Alias for the variables field.")) - public var: DeepPrimitiveMap + public var: ConfigContext @schema( joi @@ -70,7 +69,7 @@ export class CustomCommandContext extends DefaultEnvironmentContext { params: DefaultEnvironmentContextParams & { args: ArgsSchema opts: OptsSchema - variables: DeepPrimitiveMap + variables: ConfigContext rest: string[] } ) { diff --git a/core/src/config/template-contexts/module.ts b/core/src/config/template-contexts/module.ts index f4e0f505b9..55105bf720 100644 --- a/core/src/config/template-contexts/module.ts +++ b/core/src/config/template-contexts/module.ts @@ -187,17 +187,13 @@ class RuntimeConfigContext extends ConfigContext { } } } - - // This ensures that any template string containing runtime.* references is returned unchanged when - // there is no or limited runtime context available. - this._alwaysAllowPartial = allowPartial } } export interface OutputConfigContextParams { garden: Garden resolvedProviders: ProviderMap - variables: DeepPrimitiveMap + variables: ConfigContext modules: GardenModule[] // We only supply this when resolving configuration in dependency order. // Otherwise we pass `${runtime.*} template strings through for later resolution. diff --git a/core/src/config/template-contexts/project.ts b/core/src/config/template-contexts/project.ts index cf53ac5727..11c6be2536 100644 --- a/core/src/config/template-contexts/project.ts +++ b/core/src/config/template-contexts/project.ts @@ -19,6 +19,8 @@ import type { VcsInfo } from "../../vcs/vcs.js" import type { ActionConfig } from "../../actions/types.js" import type { WorkflowConfig } from "../workflow.js" import { styles } from "../../logger/styles.js" +import { CollectionOrValue } from "../../util/objects.js" +import { TemplatePrimitive, TemplateValue } from "../../template-string/inputs.js" class LocalContext extends ConfigContext { @schema( @@ -346,7 +348,7 @@ export class ProjectConfigContext extends DefaultEnvironmentContext { } interface EnvironmentConfigContextParams extends ProjectConfigContextParams { - variables: DeepPrimitiveMap + variables: ConfigContext } /** @@ -358,10 +360,10 @@ export class EnvironmentConfigContext extends ProjectConfigContext { .description("A map of all variables defined in the project configuration.") .meta({ keyPlaceholder: "" }) ) - public variables: DeepPrimitiveMap + public variables: ConfigContext @schema(joiIdentifierMap(joiPrimitive()).description("Alias for the variables field.")) - public var: DeepPrimitiveMap + public var: ConfigContext @schema( joiStringMap(joi.string().description("The secret's value.")) @@ -394,9 +396,9 @@ export class RemoteSourceConfigContext extends EnvironmentConfigContext { ) .meta({ keyPlaceholder: "" }) ) - public override variables: DeepPrimitiveMap + public override variables: ConfigContext - constructor(garden: Garden, variables: DeepPrimitiveMap) { + constructor(garden: Garden, variables: ConfigContext) { super({ projectName: garden.projectName, projectRoot: garden.projectRoot, diff --git a/core/src/config/template-contexts/provider.ts b/core/src/config/template-contexts/provider.ts index 5749baa3bc..d5b423cf21 100644 --- a/core/src/config/template-contexts/provider.ts +++ b/core/src/config/template-contexts/provider.ts @@ -7,7 +7,7 @@ */ import { mapValues } from "lodash-es" -import type { DeepPrimitiveMap, PrimitiveMap } from "../common.js" +import type { PrimitiveMap } from "../common.js" import { joiIdentifierMap, joiPrimitive } from "../common.js" import type { Provider, ProviderMap } from "../provider.js" import type { GenericProviderConfig } from "../provider.js" @@ -65,7 +65,7 @@ export class ProviderConfigContext extends WorkflowConfigContext { ) public providers: Map - constructor(garden: Garden, resolvedProviders: ProviderMap, variables: DeepPrimitiveMap) { + constructor(garden: Garden, resolvedProviders: ProviderMap, variables: ConfigContext) { super(garden, variables) this.providers = new Map(Object.entries(mapValues(resolvedProviders, (p) => new ProviderContext(this, p)))) diff --git a/core/src/config/template-contexts/render.ts b/core/src/config/template-contexts/render.ts index 0301326253..29e5f85d9e 100644 --- a/core/src/config/template-contexts/render.ts +++ b/core/src/config/template-contexts/render.ts @@ -6,9 +6,8 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import type { DeepPrimitiveMap } from "../common.js" import { joiVariables } from "../common.js" -import { ParentContext, schema, TemplateContext } from "./base.js" +import { type ConfigContext, ParentContext, schema, TemplateContext } from "./base.js" import type { ProjectConfigContextParams } from "./project.js" import { ProjectConfigContext } from "./project.js" @@ -24,10 +23,10 @@ export class RenderTemplateConfigContext extends ProjectConfigContext { keyPlaceholder: "", }) ) - public inputs: DeepPrimitiveMap + public inputs: ConfigContext constructor( - params: { parentName: string; templateName: string; inputs: DeepPrimitiveMap } & ProjectConfigContextParams + params: { parentName: string; templateName: string; inputs: ConfigContext } & ProjectConfigContextParams ) { super(params) this.parent = new ParentContext(this, params.parentName) diff --git a/core/src/config/workflow.ts b/core/src/config/workflow.ts index 2a9a674367..746c6b40e4 100644 --- a/core/src/config/workflow.ts +++ b/core/src/config/workflow.ts @@ -17,6 +17,7 @@ import { joiSparseArray, createSchema, unusedApiVersionSchema, + omitFromSchema, } from "./common.js" import { deline, dedent } from "../util/string.js" import type { ServiceLimitSpec } from "../plugins/container/moduleConfig.js" @@ -31,6 +32,7 @@ import { omitUndefined } from "../util/objects.js" import type { BaseGardenResource, GardenResource } from "./base.js" import type { GardenApiVersion } from "../constants.js" import { DOCS_BASE_URL } from "../constants.js" +import { GardenConfig } from "../template-string/validation.js" export const minimumWorkflowRequests = { cpu: 50, // 50 millicpu @@ -54,11 +56,12 @@ export const defaultWorkflowResources = { limits: defaultWorkflowLimits, } -export interface WorkflowConfig extends BaseGardenResource { +export type UnrefinedWorkflowConfig = GardenConfig> +export type WorkflowConfig = BaseGardenResource & { + kind: "Workflow" apiVersion: GardenApiVersion description?: string envVars: PrimitiveMap - kind: "Workflow" resources: { requests: ServiceLimitSpec limits: ServiceLimitSpec @@ -137,10 +140,14 @@ export const workflowConfigSchema = createSchema({ .object() .keys({ requests: workflowResourceRequestsSchema().default(defaultWorkflowRequests), - limits: workflowResourceLimitsSchema().default(defaultWorkflowLimits), + limits: workflowResourceLimitsSchema().default((parent) => { + return parent.parent.limits || defaultWorkflowLimits + }), }) // .default(() => ({})) .meta({ enterprise: true }), + + // TODO: Remove this in a future release with breaking changes limits: workflowResourceLimitsSchema().meta({ enterprise: true, deprecated: "Please use the `resources.limits` field instead.", @@ -160,7 +167,7 @@ export const workflowConfigSchema = createSchema({ allowUnknown: true, }) -export interface WorkflowFileSpec { +export type WorkflowFileSpec = { path: string data?: string secretName?: string @@ -192,7 +199,7 @@ export const workflowFileSchema = createSchema({ xor: [["data", "secretName"]], }) -export interface WorkflowStepSpec { +export type WorkflowStepSpec = { name?: string command?: string[] description?: string @@ -281,7 +288,7 @@ export const triggerEvents = [ "pull-request-merged", ] -export interface TriggerSpec { +export type TriggerSpec = { environment: string namespace?: string events?: string[] @@ -341,92 +348,35 @@ export const triggerSchema = memoize(() => { }) export interface WorkflowConfigMap { - [key: string]: WorkflowConfig + [key: string]: UnrefinedWorkflowConfig } -export function resolveWorkflowConfig(garden: Garden, config: WorkflowConfig) { +export type RefinedWorkflow = Omit + +export function resolveWorkflowConfig(garden: Garden, config: UnrefinedWorkflowConfig): GardenConfig { const log = garden.log const context = new WorkflowConfigContext(garden, garden.variables) - log.silly(() => `Resolving template strings for workflow ${config.name}`) - - const partialConfig = { - // Don't allow templating in names and triggers - ...omit(config, "name", "triggers"), - // Inputs can be partially resolved - internal: omit(config.internal, "inputs"), - // Defer resolution of step commands and scripts (the dummy script will be overwritten again below) - steps: config.steps.map((s) => ({ ...s, command: undefined, script: "echo" })), - } - - let resolvedPartialConfig: WorkflowConfig = { - ...resolveTemplateStrings({ value: partialConfig, context, source: { yamlDoc: config.internal.yamlDoc } }), - name: config.name, - } - - if (config.triggers) { - resolvedPartialConfig.triggers = config.triggers - } - - if (config.internal.inputs) { - resolvedPartialConfig.internal.inputs = resolveTemplateStrings({ - value: config.internal.inputs, - context, - contextOpts: { - allowPartial: true, - }, - // TODO: Map inputs to their original YAML sources - source: undefined, - }) - } - - log.silly(() => `Validating config for workflow ${config.name}`) + log.silly(() => `Validating config for workflow ${refined.config.name}`) - resolvedPartialConfig = validateConfig({ - config: resolvedPartialConfig, - schema: workflowConfigSchema(), - projectRoot: garden.projectRoot, - yamlDocBasePath: [], - }) - - // Re-add the deferred step commands and scripts - const resolvedConfig = { - ...resolvedPartialConfig, - steps: resolvedPartialConfig.steps.map((s, i) => - omitUndefined({ - ...omit(s, "command", "script"), - command: config.steps[i].command, - script: config.steps[i].script, - }) - ), - } - - /** - * TODO: Remove support for workflow.limits the next time we make a release with breaking changes. - * - * workflow.limits is deprecated, so we copy its values into workflow.resources.limits if workflow.limits - * is specified. - */ - - if (resolvedConfig.limits) { - resolvedConfig.resources.limits = resolvedConfig.limits - } + const partialSchema = omitFromSchema(workflowConfigSchema(), "internal", "steps") + const refined = config.withContext(context).refineWithJoi(partialSchema) - const environmentConfigs = garden.getProjectConfig().environments + const environmentConfigs = garden.getProjectConfig().config.environments - validateTriggers(resolvedConfig, environmentConfigs) - populateNamespaceForTriggers(resolvedConfig, environmentConfigs) + validateTriggers(refined, environmentConfigs) + populateNamespaceForTriggers(refined, environmentConfigs) - return resolvedConfig + return refined } /** * Throws if one or more triggers uses an environment that isn't defined in the project's config. */ -function validateTriggers(config: WorkflowConfig, environmentConfigs: EnvironmentConfig[]) { +function validateTriggers(workflow: GardenConfig, environmentConfigs: EnvironmentConfig[]) { const invalidTriggers: TriggerSpec[] = [] const environmentNames = environmentConfigs.map((c) => c.name) - for (const trigger of config.triggers || []) { + for (const trigger of workflow.config.triggers || []) { if (!environmentNames.includes(trigger.environment)) { invalidTriggers.push(trigger) } @@ -435,8 +385,8 @@ function validateTriggers(config: WorkflowConfig, environmentConfigs: Environmen if (invalidTriggers.length > 0) { const msgPrefix = invalidTriggers.length === 1 - ? `Invalid environment in trigger for workflow ${config.name}:` - : `Invalid environments in triggers for workflow ${config.name}:` + ? `Invalid environment in trigger for workflow ${workflow.config.name}:` + : `Invalid environments in triggers for workflow ${workflow.config.name}:` const msg = dedent` ${msgPrefix} @@ -452,9 +402,9 @@ function validateTriggers(config: WorkflowConfig, environmentConfigs: Environmen } } -export function populateNamespaceForTriggers(config: WorkflowConfig, environmentConfigs: EnvironmentConfig[]) { +export function populateNamespaceForTriggers(workflow: GardenConfig, environmentConfigs: EnvironmentConfig[]) { try { - for (const trigger of config.triggers || []) { + for (const trigger of workflow.config.triggers || []) { const environmentConfigForTrigger = environmentConfigs.find((c) => c.name === trigger.environment) trigger.namespace = getNamespace(environmentConfigForTrigger!, trigger.namespace) } @@ -464,7 +414,7 @@ export function populateNamespaceForTriggers(config: WorkflowConfig, environment } throw new ConfigurationError({ - message: `Invalid namespace in trigger for workflow ${config.name}: ${err.message}`, + message: `Invalid namespace in trigger for workflow ${workflow.config.name}: ${err.message}`, wrappedErrors: [err], }) } diff --git a/core/src/exceptions.ts b/core/src/exceptions.ts index 489860ee68..09a339bed3 100644 --- a/core/src/exceptions.ts +++ b/core/src/exceptions.ts @@ -16,6 +16,9 @@ import indentString from "indent-string" import { constants } from "os" import dns from "node:dns" import { styles } from "./logger/styles.js" +import { Location } from "./template-string/ast.js" +import { ObjectPath } from "./config/template-contexts/base.js" +import { ConfigSource } from "./config/validation.js" // Unfortunately, NodeJS does not provide a list of all error codes, so we have to maintain this list manually. // See https://nodejs.org/docs/latest-v18.x/api/dns.html#error-codes @@ -300,14 +303,28 @@ export class CloudApiError extends GardenError { } } +export class TemplateFunctionCallError extends GardenError { + type = "template-function-call" + + constructor({ message, yamlPath, source }: { message: string; yamlPath?: ObjectPath; source?: ConfigSource }) { + super({ message }) + } +} + +type TemplateStringErrorParams = { + message: string + rawTemplateString: string + loc: Location +} export class TemplateStringError extends GardenError { type = "template-string" - path?: (string | number)[] - - constructor(params: GardenErrorParams & { path?: (string | number)[] }) { - super(params) - this.path = params.path + // TODO: improve error message, using position, yamlPath and source + constructor({ message, rawTemplateString, loc }: TemplateStringErrorParams) { + const prefix = `Invalid template string (${styles.accent(truncate(rawTemplateString, { length: 35 }).replace(/\n/g, "\\n"))}): ` + super({ + message: message.startsWith(prefix) ? message : `${prefix}${message}`, + }) } } diff --git a/core/src/garden.ts b/core/src/garden.ts index 618e9e4593..51d5e373be 100644 --- a/core/src/garden.ts +++ b/core/src/garden.ts @@ -11,14 +11,14 @@ const { ensureDir } = fsExtra import { platform, arch } from "os" import { relative, resolve } from "path" import cloneDeep from "fast-copy" -import { flatten, sortBy, keyBy, mapValues, groupBy, set } from "lodash-es" +import { flatten, sortBy, keyBy, mapValues, groupBy, set, uniq } from "lodash-es" import AsyncLock from "async-lock" import { TreeCache } from "./cache.js" import { getBuiltinPlugins } from "./plugins/plugins.js" import type { GardenModule, ModuleConfigMap, ModuleTypeMap } from "./types/module.js" import { getModuleCacheContext } from "./types/module.js" -import type { SourceConfig, ProjectConfig, OutputSpec, ProxyConfig } from "./config/project.js" +import type { SourceConfig, ProjectConfig, ProxyConfig, ProjectConfigWithoutSources } from "./config/project.js" import { resolveProjectConfig, pickEnvironment, @@ -27,6 +27,7 @@ import { projectSourcesSchema, defaultNamespace, defaultEnvironment, + fixedPlugins, } from "./config/project.js" import { findByName, @@ -34,7 +35,7 @@ import { getPackageVersion, getNames, findByNames, - duplicatesByKey, + duplicatesBy, getCloudDistributionName, getCloudLogSectionName, } from "./util/util.js" @@ -56,16 +57,16 @@ import type { ConfigGraph } from "./graph/config-graph.js" import { ResolvedConfigGraph } from "./graph/config-graph.js" import { getRootLogger } from "./logger/logger.js" import type { GardenPluginSpec } from "./plugin/plugin.js" -import type { GardenResource } from "./config/base.js" +import type { BaseGardenResource, UnrefinedProjectConfig } from "./config/base.js" import { loadConfigResources, findProjectConfig, configTemplateKind, renderTemplateKind } from "./config/base.js" -import type { DeepPrimitiveMap, StringMap, PrimitiveMap } from "./config/common.js" -import { treeVersionSchema, joi, allowUnknown } from "./config/common.js" +import type { DeepPrimitiveMap, StringMap } from "./config/common.js" +import { treeVersionSchema, allowUnknown } from "./config/common.js" import { GlobalConfigStore } from "./config-store/global.js" import type { LinkedSource } from "./config-store/local.js" import { LocalConfigStore } from "./config-store/local.js" import type { ExternalSourceType } from "./util/ext-source-util.js" import { getLinkedSources } from "./util/ext-source-util.js" -import type { ModuleConfig } from "./config/module.js" +import type { ModuleConfig, UnrefinedModuleConfig } from "./config/module.js" import { convertModules, ModuleResolver } from "./resolve-module.js" import type { CommandInfo, PluginEventBroker } from "./plugin-context.js" import { createPluginContext } from "./plugin-context.js" @@ -88,7 +89,7 @@ import { defaultConfigFilename, defaultDotIgnoreFile, } from "./util/fs.js" -import type { Provider, GenericProviderConfig, ProviderMap } from "./config/provider.js" +import type { Provider, ProviderMap, BaseProviderConfig } from "./config/provider.js" import { getAllProviderDependencyNames, defaultProvider } from "./config/provider.js" import { ResolveProviderTask } from "./tasks/resolve-provider.js" import { ActionRouter } from "./router/router.js" @@ -105,18 +106,13 @@ import { dedent, deline, naturalList, wordWrap } from "./util/string.js" import { DependencyGraph } from "./graph/common.js" import { Profile, profileAsync } from "./util/profiling.js" import { username } from "username" -import { - throwOnMissingSecretKeys, - resolveTemplateString, - resolveTemplateStrings, -} from "./template-string/template-string.js" -import type { WorkflowConfig, WorkflowConfigMap } from "./config/workflow.js" +import { resolveTemplateStrings } from "./template-string/template-string.js" +import type { RefinedWorkflow, UnrefinedWorkflowConfig, WorkflowConfig, WorkflowConfigMap } from "./config/workflow.js" import { resolveWorkflowConfig, isWorkflowConfig } from "./config/workflow.js" import type { PluginTools } from "./util/ext-tools.js" import { PluginTool } from "./util/ext-tools.js" -import type { ConfigTemplateResource, ConfigTemplateConfig } from "./config/config-template.js" +import type { ResolveConfigTemplateResult, UnrefinedConfigTemplateResource } from "./config/config-template.js" import { resolveConfigTemplate } from "./config/config-template.js" -import type { TemplatedModuleConfig } from "./plugins/templated.js" import { BuildStagingRsync } from "./build-staging/rsync.js" import { DefaultEnvironmentContext, @@ -127,8 +123,8 @@ import type { CloudApi, CloudProject } from "./cloud/api.js" import { getGardenCloudDomain } from "./cloud/api.js" import { OutputConfigContext } from "./config/template-contexts/module.js" import { ProviderConfigContext } from "./config/template-contexts/provider.js" -import type { ConfigContext } from "./config/template-contexts/base.js" -import { validateSchema, validateWithPath } from "./config/validation.js" +import { GenericContext, type ConfigContext, LayeredContext } from "./config/template-contexts/base.js" +import { validateSchema } from "./config/validation.js" import { pMemoizeDecorator } from "./lib/p-memoize.js" import { ModuleGraph } from "./graph/modules.js" import { @@ -167,12 +163,14 @@ import { configureNoOpExporter } from "./util/open-telemetry/tracing.js" import { detectModuleOverlap, makeOverlapErrors } from "./util/module-overlap.js" import { GotHttpError } from "./util/http.js" import { styles } from "./logger/styles.js" +import { s } from "./config/zod.js" +import { GardenConfig } from "./template-string/validation.js" const defaultLocalAddress = "localhost" export interface GardenOpts { commandInfo: CommandInfo - config?: ProjectConfig + config?: UnrefinedProjectConfig environmentString?: string // Note: This is the string, as e.g. passed with the --env flag forceRefresh?: boolean gardenDirPath?: string @@ -184,7 +182,7 @@ export interface GardenOpts { persistent?: boolean plugins?: RegisterPluginParam[] sessionId?: string - variableOverrides?: PrimitiveMap + variableOverrides?: StringMap cloudApi?: CloudApi } @@ -207,16 +205,14 @@ export interface GardenParams { moduleExcludePatterns?: string[] monitors?: MonitorManager opts: GardenOpts - outputs: OutputSpec[] plugins: RegisterPluginParam[] + production: boolean - projectConfig: ProjectConfig + projectConfig: ProjectConfigWithoutSources projectName: string projectRoot: string - projectSources?: SourceConfig[] - providerConfigs: GenericProviderConfig[] - variables: DeepPrimitiveMap - variableOverrides: DeepPrimitiveMap + variables: ConfigContext + variableOverrides: StringMap secrets: StringMap sessionId: string username: string | undefined @@ -253,7 +249,7 @@ export class Garden { public readonly treeCache: TreeCache public events: EventBus private tools?: { [key: string]: PluginTool } - public configTemplates: { [name: string]: ConfigTemplateConfig } + public configTemplates: { [name: string]: ResolveConfigTemplateResult } private actionTypeBases: ActionTypeMap[]> private emittedWarnings: Set public cloudApi: CloudApi | null @@ -272,26 +268,23 @@ export class Garden { * for the current environment but can be overwritten with the `--env` flag. */ public readonly namespace: string - public readonly variables: DeepPrimitiveMap + public readonly variables: ConfigContext // Any variables passed via the `--var` CLI option (maintained here so that they can be used during module resolution // to override module variables and module varfiles). public readonly variableOverrides: DeepPrimitiveMap public readonly secrets: StringMap - private readonly projectSources: SourceConfig[] public readonly buildStaging: BuildStaging public readonly gardenDirPath: string public readonly artifactsPath: string public readonly vcsInfo: VcsInfo public readonly opts: GardenOpts - private readonly projectConfig: ProjectConfig - private readonly providerConfigs: GenericProviderConfig[] + private readonly projectConfig: ProjectConfigWithoutSources public readonly workingCopyId: string public readonly dotIgnoreFile: string public readonly proxy: ProxyConfig public readonly moduleIncludePatterns?: string[] public readonly moduleExcludePatterns: string[] public readonly persistent: boolean - public readonly rawOutputs: OutputSpec[] public readonly username?: string public readonly version: string private readonly forceRefresh: boolean @@ -314,14 +307,11 @@ export class Garden { this.artifactsPath = params.artifactsPath this.vcsInfo = params.vcsInfo this.opts = params.opts - this.rawOutputs = params.outputs this.production = params.production this.projectConfig = params.projectConfig this.projectName = params.projectName this.projectRoot = params.projectRoot - this.projectSources = params.projectSources || [] this.projectApiVersion = params.projectApiVersion - this.providerConfigs = params.providerConfigs this.variables = params.variables this.variableOverrides = params.variableOverrides this.secrets = params.secrets @@ -344,7 +334,7 @@ export class Garden { this.asyncLock = new AsyncLock() - const gitMode = gardenEnv.GARDEN_GIT_SCAN_MODE || params.projectConfig.scan?.git?.mode + const gitMode = gardenEnv.GARDEN_GIT_SCAN_MODE || params.projectConfig.config.scan?.git?.mode const handlerCls = gitMode === "repo" ? GitRepoHandler : GitHandler this.vcs = new handlerCls({ @@ -421,7 +411,7 @@ export class Garden { // Since we don't have the ability to hook into the post provider init stage from within the provider plugin // especially because it's the absence of said provider that needs to trigger this case, // there isn't really a cleaner way around this for now. - const providerConfigs = this.getRawProviderConfigs() + const providerConfigs = this.getConfiguredProviders() const hasOtelCollectorProvider = providerConfigs.some((providerConfig) => { return providerConfig.name === "otel-collector" @@ -636,7 +626,7 @@ export class Garden { } this.log.silly(() => `Loading plugins`) - const rawConfigs = this.getRawProviderConfigs() + const rawConfigs = this.getConfiguredProviders() this.loadedPlugins = await loadAndResolvePlugins(this.log, this.projectRoot, this.registeredPlugins, rawConfigs) @@ -650,9 +640,9 @@ export class Garden { * Returns plugins that are currently configured in provider configs. */ @pMemoizeDecorator() - async getConfiguredPlugins() { + async getConfiguredPlugins(): Promise { const plugins = await this.getAllPlugins() - const configNames = keyBy(this.getRawProviderConfigs(), "name") + const configNames = keyBy(this.getConfiguredProviders(), "name") return plugins.filter((p) => configNames[p.name]) } @@ -692,10 +682,41 @@ export class Garden { return this.actionTypeBases[kind][type] || [] } - getRawProviderConfigs({ names, allowMissing = false }: { names?: string[]; allowMissing?: boolean } = {}) { - return names - ? findByNames({ names, entries: this.providerConfigs, description: "provider", allowMissing }) - : this.providerConfigs + getConfiguredProviders({ + names, + allowMissing = false, + }: { names?: string[]; allowMissing?: boolean } = {}): BaseProviderConfig[] { + const configs = names + ? findByNames({ names, entries: this.projectConfig.config.providers, description: "provider", allowMissing }) + : this.projectConfig.config.providers + + // we pretend these are configured by default, even when no providers have been actually declared in project.garden.yml + const defaultProviders = fixedPlugins.map((name) => ({ name, environments: undefined, dependencies: [] })) + + const providers: Record = {} + for (const config of [...configs, ...defaultProviders]) { + if (config.environments !== undefined && !config.environments.includes(this.environmentName)) { + // This provider is not configured for the current environment + continue + } + + // merge dependencies and environments + if (!providers[config.name]) { + providers[config.name] = { + name: config.name, + dependencies: config.dependencies, + environments: config.environments, + } + } else { + providers[config.name].dependencies = uniq([...providers[config.name].dependencies, ...config.dependencies]) + providers[config.name].environments = uniq([ + ...(providers[config.name].environments || [this.environmentName]), + ...(config.environments || [this.environmentName]), + ]) + } + } + + return Object.values(providers) } async resolveProvider(log: Log, name: string) { @@ -733,14 +754,12 @@ export class Garden { let providers: Provider[] = [] await this.asyncLock.acquire("resolve-providers", async () => { - const rawConfigs = this.getRawProviderConfigs({ names }) + const rawConfigs = this.getConfiguredProviders({ names }) if (!names) { names = getNames(rawConfigs) } - throwOnMissingSecretKeys(rawConfigs, this.secrets, "Provider", log) - // As an optimization, we return immediately if all requested providers are already resolved const alreadyResolvedProviders = names.map((name) => this.resolvedProviders[name]).filter(Boolean) if (alreadyResolvedProviders.length === names.length) { @@ -879,15 +898,15 @@ export class Garden { /** * When running workflows via the `workflow` command, we only resolve the workflow being executed. */ - async getWorkflowConfig(name: string): Promise { + async getWorkflowConfig(name: string): Promise> { return resolveWorkflowConfig(this, await this.getRawWorkflowConfig(name)) } - async getRawWorkflowConfig(name: string): Promise { + async getRawWorkflowConfig(name: string): Promise { return (await this.getRawWorkflowConfigs([name]))[0] } - async getRawWorkflowConfigs(names?: string[]): Promise { + async getRawWorkflowConfigs(names?: string[]): Promise { if (!this.state.configsScanned) { await this.scanAndAddConfigs() } @@ -913,7 +932,7 @@ export class Garden { const plugins = keyBy(loadedPlugins, "name") // We only pass configured plugins to the router (others won't have the required configuration to call handlers) - const configuredPlugins = this.getRawProviderConfigs().map((c) => plugins[c.name]) + const configuredPlugins = this.getConfiguredProviders().map((c) => plugins[c.name]) return new ActionRouter(this, configuredPlugins, loadedPlugins, moduleTypes) } @@ -1314,21 +1333,22 @@ export class Garden { const allResources = flatten( await Promise.all(configPaths.map(async (path) => (await this.loadResources(path)) || [])) ) - const groupedResources = groupBy(allResources, "kind") + const groupedResources = groupBy(allResources, (r) => r.config.kind) - for (const [kind, configs] of Object.entries(groupedResources)) { - throwOnMissingSecretKeys(configs, this.secrets, kind, this.log) - } + let rawModuleConfigs = [...((groupedResources.Module as UnrefinedModuleConfig[]) || [])].map((c) => { + return c.refineWithZod(s.object({ + type: s.string(), + })) + }) - let rawModuleConfigs = [...((groupedResources.Module as ModuleConfig[]) || [])] - const rawWorkflowConfigs = (groupedResources.Workflow as WorkflowConfig[]) || [] - const rawConfigTemplateResources = (groupedResources[configTemplateKind] as ConfigTemplateResource[]) || [] + const rawWorkflowConfigs = (groupedResources.Workflow as UnrefinedWorkflowConfig[]) || [] + const rawConfigTemplateResources = (groupedResources[configTemplateKind] as UnrefinedConfigTemplateResource[]) || [] // Resolve config templates const configTemplates = await Promise.all(rawConfigTemplateResources.map((r) => resolveConfigTemplate(this, r))) - const templatesByName = keyBy(configTemplates, "name") + const templatesByName = keyBy(configTemplates, (t) => t.refinedTemplate.config.name) // -> detect duplicate templates - const duplicateTemplates = duplicatesByKey(configTemplates, "name") + const duplicateTemplates = duplicatesBy(configTemplates, (t) => t.refinedTemplate.config.name) if (duplicateTemplates.length > 0) { const messages = duplicateTemplates @@ -1336,7 +1356,7 @@ export class Garden { (d) => `Name ${d.value} is used at ${naturalList( d.duplicateItems.map((i) => - relative(this.projectRoot, i.internal.configFilePath || i.internal.basePath) + relative(this.projectRoot, i.refinedTemplate.configFileDirname!) ) )}` ) @@ -1346,16 +1366,17 @@ export class Garden { }) } + // Convert type:templated modules to Render configs // TODO: remove in 0.14 - const rawTemplatedModules = rawModuleConfigs.filter((m) => m.type === "templated") as TemplatedModuleConfig[] + const rawTemplatedModules = rawModuleConfigs.filter((m) => m.config.type === "templated") // -> removed templated modules from the module config list - rawModuleConfigs = rawModuleConfigs.filter((m) => m.type !== "templated") + rawModuleConfigs = rawModuleConfigs.filter((m) => m.config.type !== "templated") const renderConfigs = [ ...(groupedResources[renderTemplateKind] || []), ...rawTemplatedModules.map(convertTemplatedModuleToRender), - ] as RenderTemplateConfig[] + ] as GardenConfig[] // Resolve Render configs const renderResults = await Promise.all( @@ -1470,15 +1491,15 @@ export class Garden { * added workflows, and partially resolving it (i.e. without fully resolving step configs, which * is done just-in-time before a given step is run). */ - private addWorkflow(config: WorkflowConfig) { - const key = config.name + private addWorkflow(config: UnrefinedWorkflowConfig) { + const key = config.config.name this.log.silly(() => `Adding workflow ${key}`) const existing = this.workflowConfigs[key] if (existing) { - const paths = [existing.internal.configFilePath || existing.internal.basePath, config.internal.basePath] - const [pathA, pathB] = paths.map((path) => relative(this.projectRoot, path)).sort() + const paths = [existing.configFileDirname, config.configFileDirname] + const [pathA, pathB] = paths.map((path) => relative(this.projectRoot, path || this.projectRoot)).sort() throw new ConfigurationError({ message: `Workflow ${key} is declared multiple times (in '${pathA}' and '${pathB}')`, @@ -1496,12 +1517,12 @@ export class Garden { @OtelTraced({ name: "loadResources", }) - private async loadResources(configPath: string): Promise<(GardenResource | ModuleConfig)[]> { + private async loadResources(configPath: string): Promise[]> { configPath = resolve(this.projectRoot, configPath) this.log.silly(() => `Load configs from ${configPath}`) const resources = await loadConfigResources(this.log, this.projectRoot, configPath) this.log.silly(() => `Loaded configs from ${configPath}`) - return resources.filter((r) => r.kind && r.kind !== "Project") + return resources.filter((r) => r.config.kind && r.config.kind !== "Project") } //=========================================================================== @@ -1629,7 +1650,7 @@ export class Garden { if (resolveProviders) { providers = Object.values(await this.resolveProviders(log)) } else { - providers = this.getRawProviderConfigs() + providers = this.getConfiguredProviders() } if (!graph && resolveGraph) { @@ -1651,13 +1672,13 @@ export class Garden { workflowConfigs = await this.getRawWorkflowConfigs() } } else { - providers = this.getRawProviderConfigs() + providers = this.getConfiguredProviders() moduleConfigs = await this.getRawModuleConfigs() workflowConfigs = await this.getRawWorkflowConfigs() actionConfigs = this.actionConfigs } - const allEnvironmentNames = this.projectConfig.environments.map((c) => c.name) + const allEnvironmentNames = this.projectConfig.config.environments.map((c) => c.name) return { environmentName: this.environmentName, @@ -1715,12 +1736,19 @@ export async function resolveGardenParamsPartial(currentDirectory: string, opts: } } - gardenDirPath = resolve(config.path, gardenDirPath || DEFAULT_GARDEN_DIR_NAME) + const projectRoot = config.configFileDirname + + if (!projectRoot) { + throw new InternalError({ + message: `Could not determine project root`, + }) + } + + gardenDirPath = resolve(projectRoot, gardenDirPath || DEFAULT_GARDEN_DIR_NAME) const artifactsPath = resolve(gardenDirPath, "artifacts") const _username = (await username()) || "" - const projectName = config.name - const { path: projectRoot } = config + const projectName = config.config.name const commandInfo = opts.commandInfo const treeCache = new TreeCache() @@ -1734,29 +1762,31 @@ export async function resolveGardenParamsPartial(currentDirectory: string, opts: }) const vcsInfo = await gitHandler.getPathInfo(log, projectRoot) - // Since we iterate/traverse them before fully validating them (which we do after resolving template strings), we - // validate that `config.environments` and `config.providers` are both arrays. - // This prevents cryptic type errors when the user mistakenly writes down e.g. a map instead of an array. - validateWithPath({ - config: config.environments, - schema: joi.array().items(joi.object()).min(1).required(), - configType: "project environments", - path: config.path, - projectRoot: config.path, - source: { yamlDoc: config.internal.yamlDoc, basePath: ["environments"] }, - }) + const withEnvironment = config + .withContext( + new DefaultEnvironmentContext({ + projectName, + projectRoot, + artifactsPath, + vcsInfo, + username: _username, + commandInfo, + }) + ) + .refineWithZod( + s.object({ + environments: s + .array( + s.object({ + name: s.string(), + }) + ) + .min(1), + defaultEnvironment: s.string().optional().default(""), + }) + ) - const configDefaultEnvironment = resolveTemplateString({ - string: config.defaultEnvironment || "", - context: new DefaultEnvironmentContext({ - projectName, - projectRoot, - artifactsPath, - vcsInfo, - username: _username, - commandInfo, - }), - }) as string + const configDefaultEnvironment = withEnvironment.config.defaultEnvironment const localConfigStore = new LocalConfigStore(gardenDirPath) @@ -1767,7 +1797,10 @@ export async function resolveGardenParamsPartial(currentDirectory: string, opts: log.info(`Using environment ${localConfigDefaultEnv}, set with the \`set default-env\` command`) } - environmentStr = getDefaultEnvironmentName(localConfigDefaultEnv || configDefaultEnvironment, config) + environmentStr = getDefaultEnvironmentName( + localConfigDefaultEnv || configDefaultEnvironment, + withEnvironment.config + ) } const { environment: environmentName, namespace } = parseEnvironment(environmentStr) @@ -1814,12 +1847,12 @@ export const resolveGardenParams = profileAsync(async function _resolveGardenPar vcsInfo, } = partialResolved - let { config, namespace } = partialResolved + let { namespace } = partialResolved + const { config } = partialResolved await ensureDir(gardenDirPath) await ensureDir(artifactsPath) - const projectApiVersion = config.apiVersion const sessionId = opts.sessionId || uuidv4() const cloudApi = opts.cloudApi || null @@ -1828,7 +1861,7 @@ export const resolveGardenParams = profileAsync(async function _resolveGardenPar // If true, then user is logged in and we fetch the remote project and secrets (if applicable) if (!opts.noEnterprise && cloudApi) { const distroName = getCloudDistributionName(cloudApi.domain) - const isCommunityEdition = !config.domain + const isCommunityEdition = !config.config.domain const cloudLog = log.createLog({ name: getCloudLogSectionName(distroName) }) cloudLog.info(`Connecting to ${distroName}...`) @@ -1868,10 +1901,10 @@ export const resolveGardenParams = profileAsync(async function _resolveGardenPar // If the user is logged in and a cloud project exists we use that ID // but fallback to the one set in the config (even if the user isn't logged in). // Same applies for domains. - const projectId = cloudProject?.id || config.id - const cloudDomain = cloudApi?.domain || getGardenCloudDomain(config.domain) + const projectId = cloudProject?.id || config.config.id + const cloudDomain = cloudApi?.domain || getGardenCloudDomain(config.config.domain) - config = resolveProjectConfig({ + const resolvedConfig = resolveProjectConfig({ log, defaultEnvironmentName: configDefaultEnvironment, config, @@ -1879,24 +1912,24 @@ export const resolveGardenParams = profileAsync(async function _resolveGardenPar vcsInfo, username: _username, loggedIn, - enterpriseDomain: config.domain, + enterpriseDomain: config.config.domain, secrets, commandInfo, }) const pickedEnv = await pickEnvironment({ - projectConfig: config, + projectConfig: resolvedConfig, envString: environmentStr, artifactsPath, vcsInfo, username: _username, loggedIn, - enterpriseDomain: config.domain, + enterpriseDomain: config.config.domain, secrets, commandInfo, }) - const { providers, production } = pickedEnv + const { production, refinedConfig } = pickedEnv let { variables } = pickedEnv // Allow overriding variables @@ -1916,7 +1949,7 @@ export const resolveGardenParams = profileAsync(async function _resolveGardenPar const gardenDirExcludePattern = `${relative(projectRoot, gardenDirPath)}/**/*` const moduleExcludePatterns = [ - ...((config.scan || {}).exclude || []), + ...((refinedConfig.config.scan || {}).exclude || []), gardenDirExcludePattern, ...fixedProjectExcludes, ] @@ -1925,8 +1958,8 @@ export const resolveGardenParams = profileAsync(async function _resolveGardenPar let proxyHostname: string if (gardenEnv.GARDEN_PROXY_DEFAULT_ADDRESS) { proxyHostname = gardenEnv.GARDEN_PROXY_DEFAULT_ADDRESS - } else if (config.proxy?.hostname) { - proxyHostname = config.proxy.hostname + } else if (refinedConfig.config.proxy?.hostname) { + proxyHostname = refinedConfig.config.proxy.hostname } else { proxyHostname = defaultLocalAddress } @@ -1940,7 +1973,7 @@ export const resolveGardenParams = profileAsync(async function _resolveGardenPar sessionId, projectId, cloudDomain, - projectConfig: config, + projectConfig: refinedConfig, projectRoot, projectName, environmentName, @@ -1949,26 +1982,23 @@ export const resolveGardenParams = profileAsync(async function _resolveGardenPar variables, variableOverrides, secrets, - projectSources: config.sources, production, gardenDirPath, globalConfigStore: opts.globalConfigStore, localConfigStore, opts, - outputs: config.outputs || [], plugins: opts.plugins || [], - providerConfigs: providers, moduleExcludePatterns, workingCopyId, - dotIgnoreFile: config.dotIgnoreFile, + dotIgnoreFile: refinedConfig.config.dotIgnoreFile, proxy, log, - moduleIncludePatterns: (config.scan || {}).include, + moduleIncludePatterns: (refinedConfig.config.scan || {}).include, username: _username, forceRefresh: opts.forceRefresh, cloudApi, cache: treeCache, - projectApiVersion, + projectApiVersion: refinedConfig.config.apiVersion, } }) }) @@ -1978,21 +2008,21 @@ export const resolveGardenParams = profileAsync(async function _resolveGardenPar */ async function getCloudProject({ cloudApi, - config, + config: projectConfig, log, isCommunityEdition, projectRoot, projectName, }: { cloudApi: CloudApi - config: ProjectConfig + config: UnrefinedProjectConfig log: Log isCommunityEdition: boolean projectRoot: string projectName: string }) { const distroName = getCloudDistributionName(cloudApi.domain) - const projectIdFromConfig = config.id + const projectIdFromConfig = projectConfig.config.id // If logged into community edition, throw if ID is set if (projectIdFromConfig && isCommunityEdition) { @@ -2072,17 +2102,18 @@ async function getCloudProject({ // Override variables, also allows to override nested variables using dot notation // eslint-disable-next-line @typescript-eslint/no-shadow -export function overrideVariables(variables: DeepPrimitiveMap, overrideVariables: DeepPrimitiveMap): DeepPrimitiveMap { - const objNew = cloneDeep(variables) - Object.keys(overrideVariables).forEach((key) => { - if (objNew.hasOwnProperty(key)) { - // if the original key itself is a string with a dot, then override that - objNew[key] = overrideVariables[key] - } else { - set(objNew, key, overrideVariables[key]) - } - }) - return objNew +export function overrideVariables(variables: ConfigContext, overrideVariables: StringMap): ConfigContext { + // Transform override paths like "foo.bar[0].baz" + // into a nested object like + // { foo: { bar: [{ baz: "foo" }] } } + // which we can then use for the layered context as overrides on the nested structure within + const overrides = {} + for (const key of Object.keys(overrideVariables)) { + set(overrides, key, overrideVariables[key]) + } + + const overridesContext = new GenericContext(overrides) + return new LayeredContext(overridesContext, variables) } /** @@ -2134,11 +2165,12 @@ export interface ConfigDump { environmentName: string // TODO: Remove this? allEnvironmentNames: string[] namespace: string - providers: (Provider | GenericProviderConfig)[] - variables: DeepPrimitiveMap - actionConfigs: ActionConfigMap - moduleConfigs: ModuleConfig[] - workflowConfigs: WorkflowConfig[] + projectConfig: ProjectConfigWithoutSources + providers: (Provider | BaseProviderConfig)[] + variables: ConfigContext + actionConfigs: ActionConfigMap // TODO: GardenConfig wrapper + moduleConfigs: ModuleConfig[] // TODO: GardenConfig wrapper + workflowConfigs: WorkflowConfig[] // TODO: GardenConfig wrapper projectName: string projectRoot: string projectId?: string diff --git a/core/src/plugins.ts b/core/src/plugins.ts index e90a438783..49b8d517b3 100644 --- a/core/src/plugins.ts +++ b/core/src/plugins.ts @@ -15,7 +15,7 @@ import type { GardenPluginReference, } from "./plugin/plugin.js" import { pluginSchema, pluginNodeModuleSchema } from "./plugin/plugin.js" -import type { GenericProviderConfig } from "./config/provider.js" +import type { BaseProviderConfig, GenericProviderConfig } from "./config/provider.js" import { CircularDependenciesError, ConfigurationError, PluginError, RuntimeError } from "./exceptions.js" import { uniq, mapValues, fromPairs, flatten, keyBy, some, isString, sortBy } from "lodash-es" import type { Dictionary, MaybeUndefined } from "./util/util.js" @@ -43,7 +43,7 @@ export async function loadAndResolvePlugins( log: Log, projectRoot: string, registeredPlugins: RegisterPluginParam[], - configs: GenericProviderConfig[] + configs: BaseProviderConfig[] ) { const loadedPlugins = await Promise.all(registeredPlugins.map((p) => loadPlugin(log, projectRoot, p))) const pluginsByName = keyBy(loadedPlugins, "name") @@ -54,7 +54,7 @@ export async function loadAndResolvePlugins( export function resolvePlugins( log: Log, loadedPlugins: Dictionary, - configs: GenericProviderConfig[] + configs: BaseProviderConfig[] ): GardenPluginSpec[] { const initializedPlugins: PluginMap = {} const validatePlugin = (name: string) => { diff --git a/core/src/plugins/container/config.ts b/core/src/plugins/container/config.ts index 41a8685099..45cec72f38 100644 --- a/core/src/plugins/container/config.ts +++ b/core/src/plugins/container/config.ts @@ -97,12 +97,12 @@ export interface ServiceHealthCheckSpec { /** * DEPRECATED: Use {@link ContainerResourcesSpec} instead. */ -export interface ServiceLimitSpec { +export type ServiceLimitSpec = { cpu: number memory: number } -export interface ContainerResourcesSpec { +export type ContainerResourcesSpec = { cpu: { min: number max: number | null diff --git a/core/src/router/base.ts b/core/src/router/base.ts index e870331b38..93e208c7ed 100644 --- a/core/src/router/base.ts +++ b/core/src/router/base.ts @@ -451,7 +451,7 @@ export abstract class BaseActionRouter extends BaseRouter if (filtered.length > 1) { // If we still end up with multiple handlers with no obvious best candidate, we use the order of configuration // as a tie-breaker. - const configs = this.garden.getRawProviderConfigs() + const configs = this.garden.getConfiguredProviders() for (const config of configs.reverse()) { for (const handler of filtered) { diff --git a/core/src/router/module.ts b/core/src/router/module.ts index 47d064625f..0ff578d879 100644 --- a/core/src/router/module.ts +++ b/core/src/router/module.ts @@ -275,7 +275,7 @@ export class ModuleRouter extends BaseRouter { if (filtered.length > 1) { // If we still end up with multiple handlers with no obvious best candidate, we use the order of configuration // as a tie-breaker. - const configs = this.garden.getRawProviderConfigs() + const configs = this.garden.getConfiguredProviders() for (const config of configs.reverse()) { for (const handler of filtered) { diff --git a/core/src/server/commands.ts b/core/src/server/commands.ts index a781aeb4a6..614eaef0ef 100644 --- a/core/src/server/commands.ts +++ b/core/src/server/commands.ts @@ -22,8 +22,7 @@ import type { GardenInstanceManager } from "./instance-manager.js" import { isDirectory } from "../util/fs.js" import fsExtra from "fs-extra" const { pathExists } = fsExtra -import type { ProjectConfig } from "../config/project.js" -import { findProjectConfig } from "../config/base.js" +import { findProjectConfig, type UnrefinedProjectConfig } from "../config/base.js" import type { GlobalConfigStore } from "../config-store/global.js" import type { ParsedArgs } from "minimist" import type { ServeCommand } from "../commands/serve.js" @@ -44,6 +43,7 @@ import { z } from "zod" import { exec } from "../util/util.js" import split2 from "split2" import pProps from "p-props" +import { InternalError } from "../exceptions.js" const autocompleteArguments = { input: new StringParameter({ @@ -400,7 +400,7 @@ export async function resolveRequest({ return { error: { code, message, detail } } } - let projectConfig: ProjectConfig | undefined + let projectConfig: UnrefinedProjectConfig | undefined // TODO: support --root option flag @@ -427,7 +427,13 @@ export async function resolveRequest({ } } - const projectRoot = projectConfig.path + const projectRoot = projectConfig.configFileDirname + + if (!projectRoot) { + throw new InternalError({ + message: `Could not determine project root for project config ${projectConfig.config.name}`, + }) + } const internal = request.internal diff --git a/core/src/server/instance-manager.ts b/core/src/server/instance-manager.ts index 04739885e6..757f517410 100644 --- a/core/src/server/instance-manager.ts +++ b/core/src/server/instance-manager.ts @@ -40,6 +40,8 @@ import { import type { GardenInstanceKeyParams } from "./helpers.js" import { getGardenInstanceKey } from "./helpers.js" import { styles } from "../logger/styles.js" +import { UnrefinedProjectConfig } from "../config/base.js" +import { InternalError } from "../exceptions.js" interface InstanceContext { garden: Garden @@ -349,7 +351,7 @@ export class GardenInstanceManager { sessionId, }: { command?: Command - projectConfig: ProjectConfig + projectConfig: UnrefinedProjectConfig globalConfigStore: GlobalConfigStore log: Log args: ParameterValues @@ -362,7 +364,7 @@ export class GardenInstanceManager { if (!command?.noProject) { cloudApi = await this.getCloudApi({ log, - cloudDomain: getGardenCloudDomain(projectConfig.domain), + cloudDomain: getGardenCloudDomain(projectConfig.config.domain), globalConfigStore, }) } @@ -379,7 +381,13 @@ export class GardenInstanceManager { sessionId, } - const projectRoot = projectConfig.path + const projectRoot = projectConfig.configFileDirname + + if (!projectRoot) { + throw new InternalError({ + message: "Unable to determine project root", + }) + } if (command && command.noProject) { return makeDummyGarden(projectRoot, gardenOpts) diff --git a/core/src/tasks/resolve-provider.ts b/core/src/tasks/resolve-provider.ts index ad3c85117f..62f84567b5 100644 --- a/core/src/tasks/resolve-provider.ts +++ b/core/src/tasks/resolve-provider.ts @@ -101,7 +101,7 @@ export class ResolveProviderTask extends BaseTask { const implicitDeps = getProviderTemplateReferences(this.config).map((name) => ({ name })) const allDeps = uniq([...pluginDeps, ...explicitDeps, ...implicitDeps]) - const rawProviderConfigs = this.garden.getRawProviderConfigs() + const rawProviderConfigs = this.garden.getConfiguredProviders() const plugins = keyBy(this.allPlugins, "name") const matchDependencies = (depName: string) => { diff --git a/core/src/template-string/ast.ts b/core/src/template-string/ast.ts new file mode 100644 index 0000000000..a1d960d8e2 --- /dev/null +++ b/core/src/template-string/ast.ts @@ -0,0 +1,878 @@ +/* + * Copyright (C) 2018-2023 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { isArray, isEmpty, isNumber, isString } from "lodash-es" +import { CONTEXT_RESOLVE_KEY_NOT_FOUND, renderKeyPath, type ConfigContext, type ContextResolveOpts } from "../config/template-contexts/base.js" +import { InternalError, TemplateStringError } from "../exceptions.js" +import { getHelperFunctions } from "./functions.js" +import { + TemplateLeaf, + isTemplateLeaf, + isTemplateLeafValue, + isTemplatePrimitive, + mergeInputs, + templateLeafValueDeepMap, +} from "./inputs.js" +import type { TemplateLeafValue, TemplatePrimitive, TemplateValue } from "./inputs.js" +import { WrapContextLookupInputsLazily, deepEvaluateAndUnwrap, evaluateAndUnwrap, evaluate } from "./lazy.js" +import { Collection, CollectionOrValue, deepMap } from "../util/objects.js" +import { TemplateProvenance } from "./template-string.js" +import { validateSchema } from "../config/validation.js" +import { TemplateExpressionGenerator, containsLazyValues } from "./static-analysis.js" + +type EvaluateArgs = { + context: ConfigContext + opts: ContextResolveOpts + rawTemplateString: string + + /** + * Whether or not to throw an error if ContextLookupExpression fails to resolve variable. + * The FormatStringExpression will set this parameter based on wether the OptionalSuffix (?) is present or not. + */ + optional?: boolean +} + +/** + * Returned by the `location()` helper in PEG.js. + */ +export type Location = { + start: { + offset: number + line: number + column: number + } + end: { + offset: number + line: number + column: number + } + source: TemplateProvenance +} + +function* astVisitAll(e: TemplateExpression): TemplateExpressionGenerator { + for (const propertyValue of Object.values(e)) { + if (propertyValue instanceof TemplateExpression) { + yield propertyValue + yield* astVisitAll(propertyValue) + } else if (Array.isArray(propertyValue)) { + for (const item of propertyValue) { + if (item instanceof TemplateExpression) { + yield item + yield* astVisitAll(item) + } + } + } + } +} + +export abstract class TemplateExpression { + constructor(public readonly loc: Location) {} + + *visitAll(): TemplateExpressionGenerator { + yield* astVisitAll(this) + } + + abstract evaluate(args: EvaluateArgs): CollectionOrValue +} + +export class IdentifierExpression extends TemplateExpression { + constructor( + loc: Location, + public readonly name: string + ) { + if (!isString(name)) { + throw new InternalError({ + message: `IdentifierExpression name must be a string. Got: ${typeof name}`, + }) + } + super(loc) + } + + override evaluate({ rawTemplateString }): TemplateLeaf { + return new TemplateLeaf({ + expr: rawTemplateString, + value: this.name, + inputs: {}, + }) + } +} + +export class LiteralExpression extends TemplateExpression { + constructor( + loc: Location, + public readonly literal: TemplatePrimitive + ) { + super(loc) + } + + override evaluate(args: EvaluateArgs): TemplateLeaf { + return new TemplateLeaf({ + expr: args.rawTemplateString, + value: this.literal, + inputs: {}, + }) + } +} + +export class ArrayLiteralExpression extends TemplateExpression { + constructor( + loc: Location, + // an ArrayLiteralExpression consists of several template expressions, + // for example other literal expressions and context lookup expressions. + public readonly literal: TemplateExpression[] + ) { + super(loc) + } + + override evaluate(args: EvaluateArgs): CollectionOrValue { + // Empty array needs to be wrapped with TemplateLeaf + if (isEmpty(this.literal)) { + return new TemplateLeaf({ + expr: args.rawTemplateString, + value: [], + inputs: {}, + }) + } + + return this.literal.map((expr) => expr.evaluate(args)) + } +} + +export abstract class UnaryExpression extends TemplateExpression { + constructor( + loc: Location, + public readonly innerExpression: TemplateExpression + ) { + super(loc) + } + + override evaluate(args: EvaluateArgs): CollectionOrValue { + const inner = this.innerExpression.evaluate(args) + + const innerValue = evaluateAndUnwrap({ value: inner, context: args.context, opts: args.opts }) + + return mergeInputs( + this.loc.source, + new TemplateLeaf({ + expr: args.rawTemplateString, + value: this.transform(innerValue), + inputs: {}, + }), + inner + ) + } + + abstract transform(value: TemplatePrimitive | Collection): TemplatePrimitive +} + +export class TypeofExpression extends UnaryExpression { + override transform(value: TemplatePrimitive | Collection): string { + return typeof value + } +} + +export class NotExpression extends UnaryExpression { + override transform(value: TemplatePrimitive | Collection): boolean { + return !value + } +} + +export abstract class LogicalExpression extends TemplateExpression { + constructor( + loc: Location, + public readonly operator: string, + public readonly left: TemplateExpression, + public readonly right: TemplateExpression + ) { + super(loc) + } +} + +// you need to call with unwrap: isTruthy(unwrap(value)) +export function isTruthy(v: TemplatePrimitive | Collection): boolean { + if (isTemplatePrimitive(v)) { + return !!v + } else { + // collections are truthy, regardless wether they are empty or not. + v satisfies Collection + return true + } +} + +export class LogicalOrExpression extends LogicalExpression { + override evaluate(args: EvaluateArgs): CollectionOrValue { + const left = this.left.evaluate({ + ...args, + optional: true, + }) + + if (isTruthy(evaluateAndUnwrap({ value: left, context: args.context, opts: args.opts }))) { + return left + } + + if (args.opts.allowPartial) { + // it might be that the left side will become resolvable later. + // TODO: should we maybe explicitly return a symbol, when we couldn't resolve something? + return mergeInputs(this.loc.source, new TemplateLeaf({ + expr: args.rawTemplateString, + value: undefined, + inputs: {}, + }), left) + } + + const right = this.right.evaluate(args) + return mergeInputs(this.loc.source, right, left) + } +} + +export class LogicalAndExpression extends LogicalExpression { + override evaluate(args: EvaluateArgs): CollectionOrValue { + const left = this.left.evaluate({ + ...args, + // TODO: Why optional for &&? + optional: true, + }) + + // NOTE(steffen): I find this logic extremely weird. + // + // I would have expected the following: + // "value" && missing => error + // missing && "value" => error + // false && missing => false + // + // and similarly for ||: + // missing || "value" => "value" + // "value" || missing => "value" + // missing || missing => error + // false || missing => error + + const leftValue = evaluateAndUnwrap({ value: left, context: args.context, opts: args.opts }) + if (!isTruthy(leftValue)) { + // Javascript would return the value on the left; we return false in case the value is undefined. This is a quirk of Garden's template languate that we want to keep for backwards compatibility. + if (leftValue === undefined) { + return mergeInputs( + this.loc.source, + new TemplateLeaf({ + expr: args.rawTemplateString, + value: args.opts.allowPartial ? undefined : false, + inputs: {}, + }), + left + ) + } else { + return left + } + } else { + const right = this.right.evaluate({ + ...args, + // TODO: is this right? + optional: true, + }) + const rightValue = evaluateAndUnwrap({ value: right, context: args.context, opts: args.opts }) + if (rightValue === undefined) { + return mergeInputs( + this.loc.source, + new TemplateLeaf({ + expr: args.rawTemplateString, + value: args.opts.allowPartial ? undefined : false, + inputs: {}, + }), + right, + left + ) + } else { + return mergeInputs(this.loc.source, right, left) + } + } + } +} + +export abstract class BinaryExpression extends TemplateExpression { + constructor( + loc: Location, + public readonly operator: string, + public readonly left: TemplateExpression, + public readonly right: TemplateExpression + ) { + super(loc) + } + + override evaluate(args: EvaluateArgs): CollectionOrValue { + const left = this.left.evaluate(args) + const right = this.right.evaluate(args) + + const leftValue = evaluateAndUnwrap({ value: left, context: args.context, opts: args.opts }) + const rightValue = evaluateAndUnwrap({ value: right, context: args.context, opts: args.opts }) + + const transformed = this.transform(leftValue, rightValue, args) + + if (isTemplatePrimitive(transformed)) { + return mergeInputs( + this.loc.source, + new TemplateLeaf({ + expr: args.rawTemplateString, + value: transformed, + inputs: {}, + }), + left, + right + ) + } + + // We don't need to merge inputs; if transform returns a collection it took care of that already. + // Example: concatenation of strings with the + operator. + return transformed + } + + abstract transform( + left: TemplatePrimitive | Collection, + right: TemplatePrimitive | Collection, + args: EvaluateArgs + ): TemplatePrimitive | Collection +} + +export class EqualExpression extends BinaryExpression { + override transform( + left: TemplatePrimitive | Collection, + right: TemplatePrimitive | Collection + ): boolean { + return left === right + } +} + +export class NotEqualExpression extends BinaryExpression { + override transform( + left: TemplatePrimitive | Collection, + right: TemplatePrimitive | Collection + ): boolean { + return left !== right + } +} + +export class AddExpression extends BinaryExpression { + override transform( + left: TemplatePrimitive | Collection, + right: TemplatePrimitive | Collection, + args: EvaluateArgs + ): TemplatePrimitive | Collection { + if (isNumber(left) && isNumber(right)) { + return left + right + } else if (isString(left) && isString(left)) { + return left + right + } else if (Array.isArray(left) && Array.isArray(right)) { + // In this special case, simply return the concatenated arrays. + // Input tracking has been taken care of already in this case, as leaf objects are preserved. + return left.concat(right) + } else { + throw new TemplateStringError({ + message: `Both terms need to be either arrays or strings or numbers for + operator (got ${typeof left} and ${typeof right}).`, + rawTemplateString: args.rawTemplateString, + loc: this.loc, + }) + } + } +} + +export class ContainsExpression extends BinaryExpression { + override transform( + collection: TemplatePrimitive | Collection, + element: TemplatePrimitive | Collection, + args: EvaluateArgs + ): boolean { + if (!isTemplatePrimitive(element)) { + throw new TemplateStringError({ + message: `The right-hand side of a 'contains' operator must be a string, number, boolean or null (got ${typeof element}).`, + rawTemplateString: args.rawTemplateString, + loc: this.loc, + }) + } + + if (typeof collection === "object" && collection !== null) { + if (isArray(collection)) { + return collection.some((v) => element === evaluateAndUnwrap({ value: v, context: args.context, opts: args.opts })) + } + + return collection.hasOwnProperty(String(element)) + } + + if (typeof collection === "string") { + return collection.includes(String(element)) + } + + throw new TemplateStringError({ + message: `The left-hand side of a 'contains' operator must be a string, array or object (got ${collection}).`, + rawTemplateString: args.rawTemplateString, + loc: this.loc, + }) + } +} + +export abstract class BinaryExpressionOnNumbers extends BinaryExpression { + override transform( + left: TemplatePrimitive | Collection, + right: TemplatePrimitive | Collection, + args: EvaluateArgs + ): TemplatePrimitive | Collection { + // All other operators require numbers to make sense (we're not gonna allow random JS weirdness) + if (!isNumber(left) || !isNumber(right)) { + throw new TemplateStringError({ + message: `Both terms need to be numbers for ${ + this.operator + } operator (got ${typeof left} and ${typeof right}).`, + rawTemplateString: args.rawTemplateString, + loc: this.loc, + }) + } + + return this.calculate(left, right) + } + + abstract calculate(left: number, right: number): number | boolean +} + +export class MultiplyExpression extends BinaryExpressionOnNumbers { + override calculate(left: number, right: number): number { + return left * right + } +} + +export class DivideExpression extends BinaryExpressionOnNumbers { + override calculate(left: number, right: number): number { + return left / right + } +} + +export class ModuloExpression extends BinaryExpressionOnNumbers { + override calculate(left: number, right: number): number { + return left % right + } +} + +export class SubtractExpression extends BinaryExpressionOnNumbers { + override calculate(left: number, right: number): number { + return left - right + } +} + +export class LessThanEqualExpression extends BinaryExpressionOnNumbers { + override calculate(left: number, right: number): boolean { + return left <= right + } +} + +export class GreaterThanEqualExpression extends BinaryExpressionOnNumbers { + override calculate(left: number, right: number): boolean { + return left >= right + } +} + +export class LessThanExpression extends BinaryExpressionOnNumbers { + override calculate(left: number, right: number): boolean { + return left < right + } +} + +export class GreaterThanExpression extends BinaryExpressionOnNumbers { + override calculate(left: number, right: number): boolean { + return left > right + } +} + +export class FormatStringExpression extends TemplateExpression { + constructor( + loc: Location, + public readonly innerExpression: TemplateExpression, + public readonly isOptional: boolean + ) { + super(loc) + } + + override evaluate(args: EvaluateArgs): CollectionOrValue { + const optional = args.optional !== undefined ? args.optional : this.isOptional + + return this.innerExpression.evaluate({ + ...args, + optional, + }) + } +} + +export class ElseBlockExpression extends TemplateExpression { + override evaluate(): never { + // See also `buildConditionalTree` in `parser.pegjs` + throw new InternalError({ + message: `{else} block expression should not end up in the final AST`, + }) + } +} + +export class EndIfBlockExpression extends TemplateExpression { + override evaluate(): never { + // See also `buildConditionalTree` in `parser.pegjs` + throw new InternalError({ + message: `{endif} block expression should not end up in the final AST`, + }) + } +} + +export class IfBlockExpression extends TemplateExpression { + constructor( + loc: Location, + public readonly condition: TemplateExpression, + public ifTrue: TemplateExpression | undefined, + public ifFalse: TemplateExpression | undefined + ) { + super(loc) + } + + override evaluate(args: EvaluateArgs): CollectionOrValue { + const condition = this.condition.evaluate(args) + + const evaluated = isTruthy(evaluateAndUnwrap({ value: condition, context: args.context, opts: args.opts })) + ? this.ifTrue?.evaluate(args) + : this.ifFalse?.evaluate(args) + + return mergeInputs( + this.loc.source, + evaluated || + new TemplateLeaf({ + expr: args.rawTemplateString, + value: "", + inputs: {}, + }), + condition + ) + } +} + +export class StringConcatExpression extends TemplateExpression { + public readonly expressions: TemplateExpression[] + constructor(loc: Location, ...expressions: TemplateExpression[]) { + super(loc) + this.expressions = expressions + } + + override evaluate(args: EvaluateArgs): TemplateLeaf { + const evaluatedExpressions: TemplateLeaf[] = this.expressions.map((expr) => { + const r = evaluate({ value: expr.evaluate(args), context: args.context, opts: args.opts }) + + if (!isTemplateLeaf(r) || !isTemplatePrimitive(r.value)) { + throw new TemplateStringError({ + message: `Cannot concatenate: expected primitive, but expression resolved to ${ + isTemplateLeaf(r) ? typeof r.value : typeof r + }`, + rawTemplateString: args.rawTemplateString, + loc: this.loc, + }) + } + + // The isPrimitive asserts that we are dealing with primitive values, and not empty arrays + return r as TemplateLeaf + }) + + const result = evaluatedExpressions.reduce((acc, expr) => { + return `${acc}${expr.value === undefined ? "" : expr.value}` + }, "") + + return mergeInputs( + this.loc.source, + new TemplateLeaf({ + expr: args.rawTemplateString, + value: result, + inputs: {}, + }), + ...evaluatedExpressions + ) as TemplateLeaf // TODO: fix mergeInputs return type + } +} + +export class MemberExpression extends TemplateExpression { + constructor( + loc: Location, + public readonly innerExpression: TemplateExpression + ) { + super(loc) + } + + override evaluate(args: EvaluateArgs): TemplateLeaf { + const inner = this.innerExpression.evaluate(args) + const innerValue = evaluateAndUnwrap({ value: inner, context: args.context, opts: args.opts }) + + if (typeof innerValue !== "string" && typeof innerValue !== "number") { + throw new TemplateStringError({ + message: `Expression in bracket must resolve to a string or number (got ${typeof innerValue}).`, + rawTemplateString: args.rawTemplateString, + loc: this.loc, + }) + } + + return new TemplateLeaf({ + expr: args.rawTemplateString, + value: innerValue, + inputs: {}, + }) + } +} + +export class ContextLookupExpression extends TemplateExpression { + constructor( + loc: Location, + public readonly keyPath: (IdentifierExpression | MemberExpression)[] + ) { + super(loc) + } + + override evaluate({ context, opts, optional, rawTemplateString }: EvaluateArgs): CollectionOrValue { + const evaluatedKeyPath = this.keyPath.map((k) => k.evaluate({ context, opts, optional, rawTemplateString })) + const keyPath = evaluatedKeyPath.map((k) => k.value) + + const { result } = context.resolve({ + key: keyPath, + nodePath: [], + opts, + }) + + if (result === CONTEXT_RESOLVE_KEY_NOT_FOUND) { + if (optional) { + return new TemplateLeaf({ + expr: rawTemplateString, + value: undefined, + inputs: {}, + }) + } + + throw new TemplateStringError({ + message: `Could not resolve key ${renderKeyPath(keyPath)}`, + rawTemplateString, + loc: this.loc, + }) + } + + let wrappedResult: CollectionOrValue = new WrapContextLookupInputsLazily( + this.loc.source, + result, + keyPath, + rawTemplateString + ) + + // eagerly wrap values if result doesn't contain lazy values anyway. + // otherwise we wrap the values at a later time, when actually necessary. + if (!containsLazyValues(result)) { + wrappedResult = evaluate({ value: wrappedResult, context, opts }) + } + + // Add inputs from the keyPath expressions as well. + return mergeInputs(this.loc.source, wrappedResult, ...evaluatedKeyPath) + } +} + +export class FunctionCallExpression extends TemplateExpression { + constructor( + loc: Location, + public readonly functionName: IdentifierExpression, + public readonly args: TemplateExpression[] + ) { + super(loc) + } + + override evaluate(args: EvaluateArgs): CollectionOrValue { + const functionArgs = this.args.map((arg) => + evaluate({ value: arg.evaluate(args), context: args.context, opts: args.opts }) + ) + const functionName = this.functionName.evaluate(args) + + let result: CollectionOrValue + + result = this.callHelperFunction({ + functionName: functionName.value, + args: functionArgs, + text: args.rawTemplateString, + context: args.context, + opts: args.opts, + }) + + return mergeInputs(this.loc.source, result, functionName) + } + + callHelperFunction({ + functionName, + args, + text, + context, + opts, + }: { + functionName: string + args: CollectionOrValue[] + text: string + context: ConfigContext + opts: ContextResolveOpts + }): CollectionOrValue { + const helperFunctions = getHelperFunctions() + const spec = helperFunctions[functionName] + + if (!spec) { + const availableFns = Object.keys(helperFunctions).join(", ") + throw new TemplateStringError({ + message: `Could not find helper function '${functionName}'. Available helper functions: ${availableFns}`, + rawTemplateString: text, + loc: this.loc, + }) + } + + const resolvedArgs: unknown[] = [] + + for (const arg of args) { + const value = evaluateAndUnwrap({ value: arg, context, opts }) + + // Note: At the moment, we always transform template values to raw values and perform the default input tracking for them; + // We might have to reconsider this once we need template helpers that perform input tracking on its own for non-collection arguments. + if (isTemplateLeafValue(value)) { + resolvedArgs.push(value) + } else if (spec.skipInputTrackingForCollectionValues) { + // This template helper is aware of TemplateValue instances, and will perform input tracking on its own. + resolvedArgs.push(value) + } else { + // This argument is a collection, and the template helper cannot deal with TemplateValue instances. + // We will unwrap this collection and resolve all values, and then perform default input tracking. + resolvedArgs.push(deepEvaluateAndUnwrap({ value: value, context, opts })) + } + } + + // Validate args + let i = 0 + for (const [argName, schema] of Object.entries(spec.arguments)) { + const value = resolvedArgs[i] + const schemaDescription = spec.argumentDescriptions[argName] + + if (value === undefined && schemaDescription.flags?.presence === "required") { + throw new TemplateStringError({ + message: `Missing argument '${argName}' (at index ${i}) for ${functionName} helper function.`, + rawTemplateString: text, + loc: this.loc, + }) + } + + const loc = this.loc + class FunctionCallValidationError extends TemplateStringError { + constructor({ message }: { message: string }) { + super({ + message: message, + rawTemplateString: text, + loc: loc, + }) + } + } + + resolvedArgs[i] = validateSchema(value, schema, { + context: `argument '${argName}' for ${functionName} helper function`, + ErrorClass: FunctionCallValidationError, + }) + i++ + } + + let result: CollectionOrValue + + try { + result = spec.fn(...resolvedArgs) + } catch (error) { + throw new TemplateStringError({ + message: `Error from helper function ${functionName}: ${error}`, + rawTemplateString: text, + loc: this.loc, + }) + } + + // We only need to augment inputs for primitive args in case skipInputTrackingForCollectionValues is true. + const trackedArgs = args.filter((arg) => { + if (isTemplateLeaf(arg)) { + return true + } + + // This argument is a collection; We only apply the default input tracking algorithm for primitive args if skipInputTrackingForCollectionValues is NOT true. + return spec.skipInputTrackingForCollectionValues !== true + }) + + // e.g. result of join() is a string, so we need to wrap it in a TemplateValue instance and merge inputs + // even though slice() returns an array, if the resulting array is empty, it's a template primitive and thus we need to wrap it in a TemplateValue instance + if (isTemplateLeafValue(result)) { + return mergeInputs( + this.loc.source, + new TemplateLeaf({ + expr: text, + value: result, + // inputs will be augmented by mergeInputs + inputs: {}, + }), + ...trackedArgs + ) + } else if (isTemplateLeaf(result)) { + if (!spec.skipInputTrackingForCollectionValues) { + throw new InternalError({ + message: `Helper function ${functionName} returned a TemplateValue instance, but skipInputTrackingForCollectionValues is not true`, + }) + } + return mergeInputs(this.loc.source, result, ...trackedArgs) + } else { + // Result is a collection; + + // if skipInputTrackingForCollectionValues is true, the function handles input tracking, so leafs are TemplateValue instances. + if (spec.skipInputTrackingForCollectionValues) { + return deepMap(result, (v) => { + if (isTemplatePrimitive(v)) { + throw new InternalError({ + message: `Helper function ${functionName} returned a collection, skipInputTrackingForCollectionValues is true and collection values are not TemplateValue instances`, + }) + } + return mergeInputs(this.loc.source, v, ...trackedArgs) + }) + } else { + // if skipInputTrackingForCollectionValues is false; Now the values are TemplatePrimitives. + // E.g. this would be the case for split() which turns a string input into a primitive string array. + // templatePrimitiveDeepMap will crash if the function misbehaved and returned TemplateValue + return templateLeafValueDeepMap(result as CollectionOrValue, (v) => { + return mergeInputs( + this.loc.source, + new TemplateLeaf({ + expr: text, + value: v, + // inputs will be augmented by mergeInputs + inputs: {}, + }), + ...trackedArgs + ) + }) + } + } + } +} + +export class TernaryExpression extends TemplateExpression { + constructor( + loc: Location, + public readonly condition: TemplateExpression, + public readonly ifTrue: TemplateExpression, + public readonly ifFalse: TemplateExpression + ) { + super(loc) + } + + override evaluate(args: EvaluateArgs): CollectionOrValue { + const conditionResult = this.condition.evaluate({ + ...args, + optional: true, + }) + + // evaluate ternary expression + const evaluationResult = isTruthy(evaluateAndUnwrap({ value: conditionResult, context: args.context, opts: args.opts })) + ? this.ifTrue.evaluate(args) + : this.ifFalse.evaluate(args) + + // merge inputs from the condition and the side that was evaluated + return mergeInputs(this.loc.source, evaluationResult, conditionResult) + } +} diff --git a/core/src/template-string/functions.ts b/core/src/template-string/functions.ts index a196f20c71..9c8e0fe456 100644 --- a/core/src/template-string/functions.ts +++ b/core/src/template-string/functions.ts @@ -8,16 +8,16 @@ import { v4 as uuidv4 } from "uuid" import { createHash } from "node:crypto" -import { TemplateStringError } from "../exceptions.js" -import { camelCase, escapeRegExp, isArrayLike, isEmpty, isString, kebabCase, keyBy, mapValues, trim } from "lodash-es" +import { camelCase, isArrayLike, isEmpty, isString, kebabCase, keyBy, mapValues, trim } from "lodash-es" import type { JoiDescription, Primitive } from "../config/common.js" import { joi, joiPrimitive } from "../config/common.js" import type Joi from "@hapi/joi" -import { validateSchema } from "../config/validation.js" import { load, loadAll } from "js-yaml" import { safeDumpYaml } from "../util/serialization.js" import indentString from "indent-string" -import { maybeTemplateString } from "./template-string.js" +import type { TemplateValue } from "./inputs.js" +import type { CollectionOrValue } from "../util/objects.js" +import { TemplateFunctionCallError } from "../exceptions.js" interface ExampleArgument { input: any[] @@ -32,6 +32,15 @@ interface TemplateHelperFunction { outputSchema: Joi.Schema exampleArguments: ExampleArgument[] fn: Function + + /** + * If the function does input tracking on its own for collection arguments, it can set this to true to avoid the default input tracking. + * + * This is useful in case the function concatenates arrays or objects, or filters arrays or objects. + * + * @default false + */ + skipInputTrackingForCollectionValues?: boolean } const helperFunctionSpecs: TemplateHelperFunction[] = [ @@ -85,7 +94,7 @@ const helperFunctionSpecs: TemplateHelperFunction[] = [ .required() .description("The array or string to append."), }, - outputSchema: joi.string(), + outputSchema: joi.alternatives(joi.string(), joi.array()), exampleArguments: [ { input: [ @@ -103,13 +112,16 @@ const helperFunctionSpecs: TemplateHelperFunction[] = [ }, { input: ["string1", "string2"], output: "string1string2" }, ], - fn: (arg1: any, arg2: any) => { + + // If we receive an array, it will be raw collection values for correct input tracking when merely concatenating arrays. + skipInputTrackingForCollectionValues: true, + fn: (arg1: string | CollectionOrValue[], arg2: string | CollectionOrValue[]) => { if (isString(arg1) && isString(arg2)) { return arg1 + arg2 } else if (Array.isArray(arg1) && Array.isArray(arg2)) { return [...arg1, ...arg2] } else { - throw new TemplateStringError({ + throw new TemplateFunctionCallError({ message: `Both terms need to be either arrays or strings (got ${typeof arg1} and ${typeof arg2}).`, }) } @@ -148,7 +160,7 @@ const helperFunctionSpecs: TemplateHelperFunction[] = [ { input: ["not empty"], output: false }, { input: [null], output: true }, ], - fn: (value: any) => value === undefined || isEmpty(value), + fn: (value: unknown) => value === undefined || isEmpty(value), }, { name: "join", @@ -190,7 +202,7 @@ const helperFunctionSpecs: TemplateHelperFunction[] = [ { input: [["some", "array"]], output: '["some","array"]' }, { input: [{ some: "object" }], output: '{"some":"object"}' }, ], - fn: (value: any) => JSON.stringify(value), + fn: (value: unknown) => JSON.stringify(value), }, { name: "kebabCase", @@ -234,8 +246,7 @@ const helperFunctionSpecs: TemplateHelperFunction[] = [ { input: ["string_with_underscores", "_", "-"], output: "string-with-underscores" }, { input: ["remove.these.dots", ".", ""], output: "removethesedots" }, ], - fn: (str: string, substring: string, replacement: string) => - str.replace(new RegExp(escapeRegExp(substring), "g"), replacement), + fn: (str: string, substring: string, replacement: string) => str.replaceAll(substring, replacement), }, { name: "sha256", @@ -270,7 +281,9 @@ const helperFunctionSpecs: TemplateHelperFunction[] = [ { input: ["ThisIsALongStringThatINeedAPartOf", 11, -7], output: "StringThatINeed" }, { input: [".foo", 1], output: "foo" }, ], - fn: (stringOrArray: string | any[], start: number | string, end?: number | string) => { + // If we receive an array, it will be raw template values for correct input tracking when merely slicing arrays. + skipInputTrackingForCollectionValues: true, + fn: (stringOrArray: string | CollectionOrValue[], start: number | string, end?: number | string) => { const parseInt = (value: number | string, name: string): number => { if (typeof value === "number") { return value @@ -278,7 +291,7 @@ const helperFunctionSpecs: TemplateHelperFunction[] = [ const result = Number.parseInt(value, 10) if (Number.isNaN(result)) { - throw new TemplateStringError({ + throw new TemplateFunctionCallError({ message: `${name} index must be a number or a numeric string (got "${value}")`, }) } @@ -340,10 +353,11 @@ const helperFunctionSpecs: TemplateHelperFunction[] = [ { input: [1], output: "1" }, { input: [true], output: "true" }, ], - fn: (val: any) => { + fn: (val: unknown) => { return String(val) }, }, + // TODO: What are the implications of `uuidv4` on input tracking? { name: "uuidv4", description: "Generates a random v4 UUID.", @@ -397,7 +411,7 @@ const helperFunctionSpecs: TemplateHelperFunction[] = [ fn: (value: any, multiDocument?: boolean) => { if (multiDocument) { if (!isArrayLike(value)) { - throw new TemplateStringError({ + throw new TemplateFunctionCallError({ message: `yamlEncode: Set multiDocument=true but value is not an array (got ${typeof value})`, }) } @@ -449,95 +463,3 @@ export function getHelperFunctions(): HelperFunctions { return _helperFunctions } - -export function callHelperFunction({ - functionName, - args, - text, - allowPartial, -}: { - functionName: string - args: any[] - text: string - allowPartial: boolean -}) { - const helperFunctions = getHelperFunctions() - const spec = helperFunctions[functionName] - - if (!spec) { - const availableFns = Object.keys(helperFunctions).join(", ") - const _error = new TemplateStringError({ - message: `Could not find helper function '${functionName}'. Available helper functions: ${availableFns}`, - }) - return { _error } - } - - const resolvedArgs: any[] = [] - - for (const arg of args) { - // arg can be null here because some helpers allow nulls as valid args - if (arg && arg._error) { - return arg - } - - // allow nulls as valid arg values - if (arg && arg.resolved !== undefined) { - resolvedArgs.push(arg.resolved) - } else { - resolvedArgs.push(arg) - } - } - - // Validate args - let i = 0 - - for (const [argName, schema] of Object.entries(spec.arguments)) { - const value = resolvedArgs[i] - const schemaDescription = spec.argumentDescriptions[argName] - - if (value === undefined && schemaDescription.flags?.presence === "required") { - return { - _error: new TemplateStringError({ - message: `Missing argument '${argName}' (at index ${i}) for ${functionName} helper function.`, - }), - } - } - - try { - resolvedArgs[i] = validateSchema(value, schema, { - context: `argument '${argName}' for ${functionName} helper function`, - ErrorClass: TemplateStringError, - }) - - // do not apply helper function for an unresolved template string - if (maybeTemplateString(value)) { - if (allowPartial) { - return { resolved: "${" + text + "}" } - } else { - const _error = new TemplateStringError({ - message: `Function '${functionName}' cannot be applied on unresolved string`, - }) - return { _error } - } - } - } catch (_error) { - if (allowPartial) { - return { resolved: text } - } else { - return { _error } - } - } - - i++ - } - - try { - const resolved = spec.fn(...resolvedArgs) - return { resolved } - } catch (error) { - const _error = new TemplateStringError({ - message: `Error from helper function ${functionName}: ${error}`, - }) - return { _error } - } -} diff --git a/core/src/template-string/inputs.ts b/core/src/template-string/inputs.ts new file mode 100644 index 0000000000..29ae23fcb5 --- /dev/null +++ b/core/src/template-string/inputs.ts @@ -0,0 +1,142 @@ +/* + * Copyright (C) 2018-2023 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { Primitive, isPrimitive } from "utility-types" +import { ObjectPath } from "../config/template-contexts/base.js" +import { InternalError } from "../exceptions.js" +import { clone, isArray, isPlainObject, mapValues } from "lodash-es" +import { CollectionOrValue, deepMap } from "../util/objects.js" +import { LazyValue, MergeInputsLazily } from "./lazy.js" +import { TemplateProvenance } from "./template-string.js" +import { containsLazyValues } from "./static-analysis.js" + +export function isTemplateLeafValue(value: unknown): value is TemplateLeafValue { + return ( + isPrimitive(value) || + (isPlainObject(value) && Object.keys(value).length === 0) || + (Array.isArray(value) && value.length === 0) + ) +} + +export function isTemplatePrimitive(value: unknown): value is TemplatePrimitive { + return isPrimitive(value) && typeof value !== "symbol" +} + +export type EmptyArray = never[] +export type EmptyObject = { [key: string]: never } + +export type TemplatePrimitive = Exclude + +export type TemplateLeafValue = + | TemplatePrimitive + // We need an instance of TemplateValue to wrap /empty/ Arrays and /empty/ Objects, so we can track their inputs. + // If the array/object has elements, those will be wrapped in TemplateValue instances. + | EmptyArray + | EmptyObject + +export function isTemplateLeaf(value: unknown): value is TemplateLeaf { + return value instanceof TemplateLeaf +} + +export function isTemplateValue(value: unknown): value is TemplateValue { + return isTemplateLeaf(value) || isLazyValue(value) +} + +export function isLazyValue(value: unknown): value is LazyValue { + return value instanceof LazyValue +} + +export type TemplateInputs = { + // key is the input variable name, e.g. secrets.someSecret, local.env.SOME_VARIABLE, etc + [contextKeyPath: string]: TemplateLeaf +} + +export class TemplateLeaf { + public readonly expr: string | undefined + public readonly value: T + public inputs: TemplateInputs + + constructor({ expr, value, inputs }: { expr: string | undefined; value: T; inputs: TemplateInputs }) { + if (!isTemplateLeafValue(value)) { + throw new InternalError({ message: `Invalid template leaf value type: ${typeof value}` }) + } + this.expr = expr + this.value = value + this.inputs = inputs + } + + public addInputs(additionalInputs: TemplateLeaf["inputs"]): TemplateLeaf { + const newLeaf = clone(this) + newLeaf["inputs"] = { + ...newLeaf.inputs, + ...additionalInputs, + } + return newLeaf + } + + static from(staticValue: TemplateLeafValue): TemplateLeaf { + return new TemplateLeaf({ + expr: undefined, + value: staticValue, + inputs: {}, + }) + } +} + +export type TemplateValue = TemplateLeaf | LazyValue + +// helpers + +// Similar to deepMap, but treats empty collections as leaves, because they are template primitives. +// The main difference to deepMap is that it calls the callback for empty arrays and objects, so we can wrap them in TemplateLeaf +export function templateLeafValueDeepMap

( + value: CollectionOrValue

, + fn: (value: TemplateLeafValue, keyPath: ObjectPath) => CollectionOrValue, + keyPath: ObjectPath = [] +): CollectionOrValue { + if (isTemplateLeafValue(value)) { + // This also handles empty collections + return fn(value, keyPath) + } else if (isArray(value)) { + return value.map((v, k) => templateLeafValueDeepMap(v, fn, [...keyPath, k])) + } else if (isPlainObject(value)) { + // we know we can use mapValues, as this was a plain object + return mapValues(value as any, (v, k) => templateLeafValueDeepMap(v, fn, [...keyPath, k])) + } else { + throw new InternalError({ message: `Unexpected value type: ${typeof value}` }) + } +} + +export function mergeInputs( + source: TemplateProvenance, + result: CollectionOrValue, + ...relevantValues: CollectionOrValue[] +): CollectionOrValue { + let additionalInputs: TemplateLeaf["inputs"] = {} + + if (containsLazyValues(relevantValues)) { + return new MergeInputsLazily(source, result, relevantValues) + } + + for (const v of relevantValues as CollectionOrValue[]) { + deepMap(v, (leaf) => { + for (const [k, v] of Object.entries(leaf.inputs)) { + additionalInputs[k] = v + } + }) + } + + if (Object.keys(additionalInputs).length === 0) { + return result + } + + return deepMap(result, (v) => { + // we can't mutate here, otherwise we'll mix up inputs. addInputs clones the value and returns a new instance with additional inputs. + return v.addInputs(additionalInputs) + }) +} diff --git a/core/src/template-string/lazy.ts b/core/src/template-string/lazy.ts new file mode 100644 index 0000000000..b392e319cc --- /dev/null +++ b/core/src/template-string/lazy.ts @@ -0,0 +1,522 @@ +/* + * Copyright (C) 2018-2023 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { clone, isBoolean, isEmpty } from "lodash-es" +import { + arrayConcatKey, + arrayForEachFilterKey, + arrayForEachKey, + arrayForEachReturnKey, + conditionalElseKey, + conditionalKey, + conditionalThenKey, + objectSpreadKey, +} from "../config/constants.js" +import type { ConfigContext, ContextResolveOpts, ObjectPath } from "../config/template-contexts/base.js" +import { GenericContext, LayeredContext, renderKeyPath } from "../config/template-contexts/base.js" +import type { Collection, CollectionOrValue } from "../util/objects.js" +import { isPlainObject, isArray, deepMap } from "../util/objects.js" +import { naturalList } from "../util/string.js" +import type { TemplateExpression } from "./ast.js" +import { isTruthy } from "./ast.js" +import type { TemplateLeafValue, TemplatePrimitive, TemplateValue } from "./inputs.js" +import { TemplateLeaf, isTemplateLeaf, isTemplatePrimitive, mergeInputs } from "./inputs.js" +import type { TemplateProvenance } from "./template-string.js" +import { TemplateError, pushYamlPath } from "./template-string.js" +import { visitAll, type TemplateExpressionGenerator, containsContextLookupReferences } from "./static-analysis.js" +import { InternalError } from "../exceptions.js" + +type UnwrapParams = { + value: CollectionOrValue + context: ConfigContext + opts: ContextResolveOpts +} + +type EvaluatePredicate = (value: LazyValue) => boolean +type ConditionallyEvaluateParams = UnwrapParams & { + predicate: EvaluatePredicate +} +export function conditionallyEvaluate(params: ConditionallyEvaluateParams): CollectionOrValue { + return deepMap(params.value, (v) => { + if (v instanceof LazyValue && params.predicate(v)) { + return conditionallyEvaluate({ ...params, value: v.evaluate(params.context, params.opts) }) + } + + return v + }) +} + +export function deepEvaluateAndUnwrap({ value, context, opts }: UnwrapParams): CollectionOrValue { + return deepMap(value, (v) => { + if (v instanceof LazyValue) { + return deepEvaluateAndUnwrap({ value: v.evaluate(context, opts), context, opts }) + } + + return v.value + }) +} + +export function deepEvaluate({ value, context, opts }: UnwrapParams): CollectionOrValue { + return deepMap(value, (v) => { + if (v instanceof LazyValue) { + return deepEvaluate({ value: v.evaluate(context, opts), context, opts }) + } + + return v + }) +} + +/** + * Recursively calls .evaluate() method on the lazy value, if value is a lazy value, until it finds a collection or template leaf. + */ +export function evaluate({ value, context, opts }: UnwrapParams): TemplateLeaf | Collection { + if (value instanceof LazyValue) { + // We recursively unwrap, because the value might be a LazyValue> + // We do not need to worry about infinite recursion here, because it's not possible to declare infinitely recursive structures in garden.yaml configs. + return evaluate({ value: value.evaluate(context, opts), context, opts }) + } + + return value +} + +/** + * Same as evaluate, but if encountering a TemplateLeaf, return the leaf's primitive value. Otherwise, return the collection. + * + * The result is definitely not a LazyValue or a TemplateLeaf. It's either a TemplatePrimitive or a Collection. + * + * This is helpful for making decisions about how to proceed in when evaluating template expressions or block operators. + */ +export function evaluateAndUnwrap(params: UnwrapParams): TemplateLeafValue | Collection { + const evaluated = evaluate(params) + + if (evaluated instanceof TemplateLeaf) { + return evaluated.value + } + + // it's a collection + return evaluated +} + +export abstract class LazyValue = CollectionOrValue> { + private additionalInputs: TemplateLeaf["inputs"] = {} + + public addInputs(inputs: TemplateLeaf["inputs"]): LazyValue { + const newLazyValue = clone(this) + newLazyValue["additionalInputs"] = { + ...newLazyValue.additionalInputs, + ...inputs, + } + return newLazyValue + } + + constructor(public readonly source: TemplateProvenance) {} + + public evaluate(context: ConfigContext, opts: ContextResolveOpts): R { + const result = this.evaluateImpl(context, opts) + return mergeInputs( + this.source, + result, + // It would be nice if mergeInputs would allow passing `additionalInputs` directly, without wrapping it in `TemplateLeaf`. + new TemplateLeaf({ expr: undefined, value: undefined, inputs: this.additionalInputs }) + ) as R + } + + abstract evaluateImpl(context: ConfigContext, opts: ContextResolveOpts): R + abstract visitAll(): TemplateExpressionGenerator +} + +export class OverrideKeyPathLazily extends LazyValue { + constructor( + private readonly backingCollection: CollectionOrValue, + private readonly keyPath: ObjectPath, + private readonly override: CollectionOrValue + ) { + super({ yamlPath: [], source: undefined }) + } + + override *visitAll(): TemplateExpressionGenerator { + // ??? + yield* visitAll(this.backingCollection) + yield* visitAll(this.override) + } + + override evaluateImpl(context: ConfigContext, opts: ContextResolveOpts): CollectionOrValue { + const evaluated = evaluate({ value: this.backingCollection, context, opts }) + + let currentValue = evaluated + const remainingKeys = clone(this.keyPath.slice(0, -1)) + const targetKey = this.keyPath[this.keyPath.length - 1] + + do { + const key = remainingKeys.shift() + + if (key === undefined) { + break + } + + if (currentValue[key] instanceof LazyValue) { + currentValue[key] = new OverrideKeyPathLazily(currentValue[key], [...remainingKeys, targetKey], this.override) + + // we don't want to override here, our child instance will do that for us + return evaluated + } + + currentValue = currentValue[key] + + if (isTemplateLeaf(currentValue)) { + if (isArray(currentValue.value) || isPlainObject(currentValue.value)) { + currentValue = currentValue.value + } else { + throw new InternalError({ + message: `Expected a collection or array, got ${typeof currentValue.value}`, + }) + } + } + } while (remainingKeys.length > 0) + + // We arrived at the destination. Override! + currentValue[targetKey] = this.override + + return evaluated + } +} + +export class MergeInputsLazily extends LazyValue { + constructor( + source: TemplateProvenance, + private readonly value: CollectionOrValue, + private readonly relevantValues: CollectionOrValue[] + ) { + super(source) + } + + override *visitAll(): TemplateExpressionGenerator { + yield* visitAll(this.relevantValues) + yield* visitAll(this.value) + } + + override evaluateImpl(context: ConfigContext, opts: ContextResolveOpts): CollectionOrValue { + const unwrapped = evaluate({ value: this.value, context, opts }) + const unwrappedRelevantValues = this.relevantValues.map((v) => evaluate({ value: v, context, opts })) + return deepMap(unwrapped, (v) => { + if (v instanceof LazyValue) { + return new MergeInputsLazily(this.source, v, unwrappedRelevantValues) + } + return mergeInputs(this.source, v satisfies TemplateLeaf, ...unwrappedRelevantValues) + }) + } +} + +export class WrapContextLookupInputsLazily extends LazyValue { + constructor( + source: TemplateProvenance, + private readonly value: CollectionOrValue, + private readonly contextKeyPath: ObjectPath, + private readonly expr: string, + private readonly collectionKeyPathPrefix: ObjectPath = [] + ) { + super(source) + } + + override *visitAll(): TemplateExpressionGenerator { + yield* visitAll(this.value) + } + + override evaluateImpl(context: ConfigContext, opts: ContextResolveOpts): CollectionOrValue { + const unwrapped = evaluate({ value: this.value, context, opts }) + return deepMap(unwrapped, (v, _k, collectionKeyPath) => { + // Wrap it lazily + if (v instanceof LazyValue) { + return new WrapContextLookupInputsLazily(this.source, v, this.contextKeyPath, this.expr, collectionKeyPath) + } + + return new TemplateLeaf({ + expr: this.expr, + value: v.value, + inputs: { + // key might be something like ["var", "foo", "bar"] + // We also add the keypath to get separate keys for every level of the keypath + [renderKeyPath([...this.contextKeyPath, ...this.collectionKeyPathPrefix, ...collectionKeyPath])]: v, + }, + }) + }) + } +} + +type TemplateStringLazyValueArgs = { + source: TemplateProvenance + astRootNode: TemplateExpression + expr: string +} +export class TemplateStringLazyValue extends LazyValue { + private readonly astRootNode: TemplateExpression + public readonly expr: string + + constructor({ source, expr, astRootNode }: TemplateStringLazyValueArgs) { + super(source) + this.expr = expr + this.astRootNode = astRootNode + } + + override *visitAll(): TemplateExpressionGenerator { + yield* this.astRootNode.visitAll() + } + + override evaluateImpl(context: ConfigContext, opts: ContextResolveOpts): CollectionOrValue { + return this.astRootNode.evaluate({ rawTemplateString: this.expr, context, opts }) + } +} + +type ConcatOperator = { [arrayConcatKey]: CollectionOrValue } + +export class ConcatLazyValue extends LazyValue[]> { + constructor( + source: TemplateProvenance, + private readonly yaml: (ConcatOperator | CollectionOrValue)[] | ForEachLazyValue + ) { + super(source) + } + + override *visitAll(): TemplateExpressionGenerator { + yield* visitAll(this.yaml as CollectionOrValue) + } + + override evaluateImpl(context: ConfigContext, opts: ContextResolveOpts): CollectionOrValue[] { + const output: CollectionOrValue[] = [] + + let concatYaml: (ConcatOperator | CollectionOrValue)[] + if (this.yaml instanceof ForEachLazyValue) { + concatYaml = this.yaml.evaluate(context, opts) + } else { + concatYaml = this.yaml + } + + for (const v of concatYaml) { + // handle concat operator + if (this.isConcatOperator(v)) { + const unwrapped = evaluateAndUnwrap({ value: v[arrayConcatKey], context, opts }) + + if (!isTemplatePrimitive(unwrapped) && isArray(unwrapped)) { + output.push(...unwrapped) + } else { + throw new TemplateError({ + message: `Value of ${arrayConcatKey} key must be (or resolve to) an array (got ${typeof unwrapped})`, + source: pushYamlPath(arrayConcatKey, this.source), + }) + } + } else { + // it's not a concat operator, it's a list element. + output.push(v) + } + } + + // input tracking is already being taken care of as we just concatenate arrays + return output + } + + isConcatOperator(v: ConcatOperator | CollectionOrValue): v is ConcatOperator { + if (isPlainObject(v) && v[arrayConcatKey] !== undefined) { + if (Object.keys(v).length > 1) { + const extraKeys = naturalList( + Object.keys(v) + .filter((k) => k !== arrayConcatKey) + .map((k) => JSON.stringify(k)) + ) + throw new TemplateError({ + message: `A list item with a ${arrayConcatKey} key cannot have any other keys (found ${extraKeys})`, + source: pushYamlPath(arrayConcatKey, this.source), + }) + } + return true + } + return false + } +} + +type ForEachClause = { + [arrayForEachKey]: CollectionOrValue // must resolve to an array or plain object, but might be a lazy value + [arrayForEachFilterKey]: CollectionOrValue | undefined // must resolve to boolean, but might be lazy value + [arrayForEachReturnKey]: CollectionOrValue +} + +export class ForEachLazyValue extends LazyValue[]> { + static allowedForEachKeys = [arrayForEachKey, arrayForEachReturnKey, arrayForEachFilterKey] + constructor( + source: TemplateProvenance, + private readonly yaml: ForEachClause + ) { + super(source) + } + + override *visitAll(): TemplateExpressionGenerator { + yield* visitAll(this.yaml as CollectionOrValue) + } + + override evaluateImpl(context: ConfigContext, opts: ContextResolveOpts): CollectionOrValue[] { + const collectionExpressionResult = evaluate({ value: this.yaml[arrayForEachKey], context, opts }) + const collectionExpressionValue = evaluateAndUnwrap({ value: collectionExpressionResult, context, opts }) + + const isObj = !isTemplatePrimitive(collectionExpressionValue) && isPlainObject(collectionExpressionValue) + const isArr = !isTemplatePrimitive(collectionExpressionValue) && isArray(collectionExpressionValue) + if (!isArr && !isObj) { + throw new TemplateError({ + message: `Value of ${arrayForEachKey} key must be (or resolve to) an array or mapping object (got ${typeof collectionExpressionValue})`, + source: pushYamlPath(arrayForEachKey, this.source), + }) + } + + const filterExpression = this.yaml[arrayForEachFilterKey] + + const output: CollectionOrValue[] = [] + + for (const i of Object.keys(collectionExpressionValue)) { + // put the TemplateValue in the context, not the primitive value, so we have input tracking + const contextForIndex = new GenericContext({ + item: { key: i, value: collectionExpressionResult[i] }, + }) + const loopContext = new LayeredContext(contextForIndex, context) + + let filterResult: CollectionOrValue | undefined + // Check $filter clause output, if applicable + if (filterExpression !== undefined) { + filterResult = evaluate({ value: filterExpression, context: loopContext, opts }) + const filterResultValue = evaluateAndUnwrap({ value: filterResult, context: loopContext, opts }) + + if (isBoolean(filterResultValue)) { + if (!filterResultValue) { + continue + } + } else { + throw new TemplateError({ + message: `${arrayForEachFilterKey} clause in ${arrayForEachKey} loop must resolve to a boolean value (got ${typeof filterResultValue})`, + source: pushYamlPath(arrayForEachFilterKey, this.source), + }) + } + } + + const returnExpression = this.yaml[arrayForEachReturnKey] + + // we have to eagerly resolve everything that references item, because the variable will not be available in the future anymore. + // TODO: Instead of evaluating item eagerly, we should implement a CaptureLazyValue, that uses LayeredContext to capture contexts for later. + const returnResult = conditionallyEvaluate({ + value: returnExpression, + context: loopContext, + opts, + predicate: (v) => containsContextLookupReferences(v, ["item"]), + }) + + if (!containsContextLookupReferences(returnExpression, ["item", "value"])) { + // force collectionExpressionResult onto the inputs, as the result still depends on the number of elements in the collection expression, even if we do not access item.value + output.push( + mergeInputs(this.source, returnResult, collectionExpressionResult[i], ...(filterResult ? [filterResult] : [])) + ) + } else { + output.push(mergeInputs(this.source, returnResult, ...(filterResult ? [filterResult] : []))) + } + } + + return output + } +} + +export type ObjectSpreadOperation = { + [objectSpreadKey]: CollectionOrValue + [staticKeys: string]: CollectionOrValue +} +export class ObjectSpreadLazyValue extends LazyValue>> { + constructor( + source: TemplateProvenance, + private readonly yaml: ObjectSpreadOperation + ) { + super(source) + } + + override *visitAll(): TemplateExpressionGenerator { + yield* visitAll(this.yaml as CollectionOrValue) + } + + override evaluateImpl( + context: ConfigContext, + opts: ContextResolveOpts + ): Record> { + // Resolve $merge keys, depth-first, leaves-first + let output = {} + + for (const [k, v] of Object.entries(this.yaml)) { + const resolved = evaluate({ value: v, context, opts }) + + if (k === objectSpreadKey) { + if (isPlainObject(resolved)) { + output = { ...output, ...resolved } + } else if (isTemplateLeaf(resolved) && isEmpty(resolved.value)) { + // nothing to do, we just ignore empty objects + } else { + const resolvedValue = evaluateAndUnwrap({ value: resolved, context, opts }) + throw new TemplateError({ + message: `Value of ${objectSpreadKey} key must be (or resolve to) a mapping object (got ${typeof resolvedValue})`, + source: pushYamlPath(k, this.source), + }) + } + } else { + output[k] = resolved + } + } + + return output + } +} + +export type ConditionalClause = { + [conditionalKey]: CollectionOrValue // must resolve to a boolean, but might be a lazy value + [conditionalThenKey]: CollectionOrValue + [conditionalElseKey]?: CollectionOrValue +} +export class ConditionalLazyValue extends LazyValue { + static allowedConditionalKeys = [conditionalKey, conditionalThenKey, conditionalElseKey] + + constructor( + source: TemplateProvenance, + private readonly yaml: ConditionalClause + ) { + super(source) + } + + override *visitAll(): TemplateExpressionGenerator { + yield* visitAll(this.yaml as CollectionOrValue) + } + + override evaluateImpl(context: ConfigContext, opts: ContextResolveOpts): CollectionOrValue { + const conditional = this.yaml[conditionalKey] + const conditionalValue = evaluateAndUnwrap({ value: conditional, context, opts }) + + if (typeof conditionalValue !== "boolean") { + throw new TemplateError({ + message: `Value of ${conditionalKey} key must be (or resolve to) a boolean (got ${typeof conditionalValue})`, + source: pushYamlPath(conditionalKey, this.source), + }) + } + + const thenClause = this.yaml[conditionalThenKey] + + // We default the $else value to undefined, if it's not specified + const elseClause = + this.yaml[conditionalElseKey] === undefined + ? new TemplateLeaf({ + value: undefined, + inputs: {}, + expr: conditionalElseKey, + }) + : this.yaml[conditionalElseKey] + + if (isTruthy(conditionalValue)) { + return mergeInputs(this.source, thenClause, conditional) + } else { + return mergeInputs(this.source, elseClause, conditional) + } + } +} diff --git a/core/src/template-string/parser.pegjs b/core/src/template-string/parser.pegjs index 9ffc691de2..42a6affa43 100644 --- a/core/src/template-string/parser.pegjs +++ b/core/src/template-string/parser.pegjs @@ -8,22 +8,18 @@ { const { - buildBinaryExpression, - buildLogicalExpression, - callHelperFunction, + ast, escapePrefix, - getKey, - getValue, - isArray, - isPlainObject, - isPrimitive, optionalSuffix, - missingKeyExceptionType, - passthroughExceptionType, - resolveNested, + parseNested, TemplateStringError, + rawTemplateString, } = options + function filledArray(count, value) { + return Array.from({ length: count }, () => value); + } + function extractOptional(optional, index) { return optional ? optional[index] : null; } @@ -36,97 +32,226 @@ return [head].concat(extractList(tail, index)); } - function optionalList(value) { - return value !== null ? value : []; + function buildBinaryExpression(head, tail) { + return tail.reduce(function(result, element) { + const operator = element[1] + const left = result + const right = element[3] + + if (operator === undefined && right === undefined) { + return left + } + + switch (operator) { + case "==": + return new ast.EqualExpression(location(), operator, left, right) + case "!=": + return new ast.NotEqualExpression(location(), operator, left, right) + case "<=": + return new ast.LessThanEqualExpression(location(), operator, left, right) + case ">=": + return new ast.GreaterThanEqualExpression(location(), operator, left, right) + case "<": + return new ast.LessThanExpression(location(), operator, left, right) + case ">": + return new ast.GreaterThanExpression(location(), operator, left, right) + case "+": + return new ast.AddExpression(location(), operator, left, right) + case "-": + return new ast.SubtractExpression(location(), operator, left, right) + case "*": + return new ast.MultiplyExpression(location(), operator, left, right) + case "/": + return new ast.DivideExpression(location(), operator, left, right) + case "%": + return new ast.ModuloExpression(location(), operator, left, right) + default: + throw new TemplateStringError({ message: `Unrecognized logical operator: ${operator}`, rawTemplateString, loc: location() }) + } + }, head); + } + + function buildLogicalExpression(head, tail) { + return tail.reduce(function(result, element) { + const operator = element[1] + const left = result + const right = element[3] + + if (operator === undefined && right === undefined) { + return left + } + + switch (operator) { + case "&&": + return new ast.LogicalAndExpression(location(), operator, left, right) + case "||": + return new ast.LogicalOrExpression(location(), operator, left, right) + default: + throw new TemplateStringError({ message: `Unrecognized logical operator: ${operator}`, rawTemplateString, loc: location() }) + } + }, head); } - function resolveList(items) { - for (const part of items) { - if (part._error) { - return part + /** + * Transforms a flat list of expressions with block operators in between to the proper ast structure, nesting expressions within blocks in the respective conditionals. + * + * @arg {(ast.TemplateExpression | Symbol)[]} elements - List of block operators and ast.TemplateExpression instances + **/ + function buildConditionalTree(...elements) { + // root level expressions + let rootExpressions = [] + + let currentCondition = undefined + let ifTrue = [] + let ifFalse = [] + let nestingLevel = 0 + let encounteredElse = false + const pushElement = (e) => { + if (!currentCondition) { + return rootExpressions.push(e) + } + if (encounteredElse) { + ifFalse.push(e) + } else { + ifTrue.push(e) } } - return items.map((part) => part.resolved || part) + for (const e of elements) { + if (e instanceof ast.IfBlockExpression) { + if (currentCondition) { + pushElement(e) + nestingLevel++ + } else { + currentCondition = e + } + } else if (e instanceof ast.ElseBlockExpression) { + if (currentCondition === undefined) { + throw new TemplateStringError({ message: "Found ${else} block without a preceding ${if...} block.", rawTemplateString, loc: location() }) + } + if (encounteredElse && nestingLevel === 0) { + throw new TemplateStringError({ message: "Encountered multiple ${else} blocks on the same ${if...} block nesting level.", rawTemplateString, loc: location() }) + } + + if (currentCondition && nestingLevel === 0) { + encounteredElse = true + } else { + pushElement(e) + } + } else if (e instanceof ast.EndIfBlockExpression) { + if (currentCondition === undefined) { + throw new TemplateStringError({ message: "Found ${endif} block without a preceding ${if...} block.", rawTemplateString, loc: location() }) + } + if (nestingLevel === 0) { + currentCondition.ifTrue = buildConditionalTree(...ifTrue) + currentCondition.ifFalse = buildConditionalTree(...ifFalse) + currentCondition.loc.end = e.loc.end + rootExpressions.push(currentCondition) + currentCondition = undefined + } else { + nestingLevel-- + pushElement(e) + } + } else { + pushElement(e) + } + } + + if (currentCondition) { + throw new TemplateStringError({ message: "Missing ${endif} after ${if ...} block.", rawTemplateString, loc: location() }) + } + + if (rootExpressions.length === 0) { + return undefined + } + if (rootExpressions.length === 1) { + return rootExpressions[0] + } + + return new ast.StringConcatExpression(location(), ...rootExpressions) + } + + function isBlockExpression(e) { + return e instanceof ast.IfBlockExpression || e instanceof ast.ElseBlockExpression || e instanceof ast.EndIfBlockExpression + } + + function optionalList(value) { + return value !== null ? value : []; } } -TemplateString - = a:(FormatString)+ b:TemplateString? { return [...a, ...(b || [])] } - / a:Prefix b:(FormatString)+ c:TemplateString? { return [a, ...b, ...(c || [])] } - / InvalidFormatString - / $(.*) { return text() === "" ? [] : [{ resolved: text() }] } +Start + = elements:TemplateStrings { + // If there is only one format string, return it's result directly without wrapping in StringConcatExpression + if (elements.length === 1 && !(isBlockExpression(elements[0]))) { + return elements[0] + } + + return buildConditionalTree(...elements) + } + +TemplateStrings + = head:(FormatString)+ tail:TemplateStrings? { + // This means we are concatenating strings, and there may be conditional block statements + return [...head, ...(tail ? tail : [])] + } + / a:Prefix b:(FormatString)+ c:TemplateStrings? { + // This means we are concatenating strings, and there may be conditional block statements + return [a, ...b, ...(c ? c : [])] + } + / UnclosedFormatString + / $(.+) { + return [new ast.LiteralExpression(location(), text())] + } FormatString = EscapeStart SourceCharacter* FormatEndWithOptional { if (options.unescape) { - return text().slice(1) + return new ast.LiteralExpression(location(), text().slice(1)) } else { - return text() + return new ast.LiteralExpression(location(), text()) } } / FormatStart op:BlockOperator FormatEnd { - return { block: op } + // These expressions will not show up in the final AST, but will be used to build the conditional tree + // We instantiate expressions here to get the correct locations for constructng good error messages + switch (op) { + case "else": + return new ast.ElseBlockExpression(location()) + case "endif": + return new ast.EndIfBlockExpression(location()) + default: + throw new TemplateStringError({ message: `Unrecognized block operator: ${op}`, rawTemplateString, loc: location() }) + } } / pre:FormatStartWithEscape blockOperator:(ExpressionBlockOperator __)* e:Expression end:FormatEndWithOptional { if (pre[0] === escapePrefix) { if (options.unescape) { - return text().slice(1) + return new ast.LiteralExpression(location(), text().slice(1)) } else { - return text() + return new ast.LiteralExpression(location(), text()) } } - // Any unexpected error is returned immediately. Certain exceptions have special semantics that are caught below. - if (e && e._error && e._error.type !== missingKeyExceptionType && e._error.type !== passthroughExceptionType) { - return e - } - - // Need to provide the optional suffix as a variable because of a parsing bug in pegjs - const allowUndefined = end[1] === optionalSuffix + const isOptional = end[1] === optionalSuffix + const expression = new ast.FormatStringExpression(location(), e, isOptional) - if (!isPlainObject(e)) { - e = { resolved: e } - } - - if (e && blockOperator[0] && blockOperator[0][0]) { - e.block = blockOperator[0][0] - } + if (blockOperator && blockOperator.length > 0) { + if (isOptional) { + throw new TemplateStringError({ message: "Cannot specify optional suffix in if-block.", rawTemplateString, loc: location() }) + } - if (e && e.block && allowUndefined) { - const _error = new TemplateStringError({ message: "Cannot specify optional suffix in if-block.", detail: { - text: text(), - }}) - return { _error } + // ifTrue and ifFalse will be filled in by `buildConditionalTree` + const ifTrue = undefined + const ifFalse = undefined + return new ast.IfBlockExpression(location(), expression, ifTrue, ifFalse, isOptional) } - if (getValue(e) === undefined) { - if (e && e._error && e._error.type === passthroughExceptionType) { - // We allow certain configuration contexts (e.g. placeholders for runtime.*) to indicate that a template - // string should be returned partially resolved even if allowPartial=false. - return text() - } else if (options.allowPartial) { - return text() - } else if (allowUndefined) { - if (e && e._error) { - return { ...e, _error: undefined } - } else { - return e - } - } else if (e && e._error) { - return e - } else { - const _error = new TemplateStringError({ message: e.message || "Unable to resolve one or more keys.", detail: { - text: text(), - }}) - return { _error } - } - } - return e + return expression } -InvalidFormatString +UnclosedFormatString = Prefix? FormatStart .* { - throw new TemplateStringError({ message: "Unable to parse as valid template string.", detail: {}}) + throw new TemplateStringError({ message: "Unable to parse as valid template string.", rawTemplateString, loc: location() }) } EscapeStart @@ -157,10 +282,10 @@ ExpressionBlockOperator = "if" Prefix - = !FormatStartWithEscape (. ! FormatStartWithEscape)* . { return text() } + = !FormatStartWithEscape (. ! FormatStartWithEscape)* . { return new ast.LiteralExpression(location(), text()) } Suffix - = !FormatEnd (. ! FormatEnd)* . { return text() } + = !FormatEnd (. ! FormatEnd)* . { return new ast.LiteralExpression(location(), text()) } // ---- expressions ----- // Reduced and adapted from: https://github.com/pegjs/pegjs/blob/master/examples/javascript.pegjs @@ -168,14 +293,7 @@ MemberExpression = head:Identifier tail:( "[" __ e:Expression __ "]" { - if (e.resolved && !isPrimitive(e.resolved)) { - const _error = new TemplateStringError( - { message: `Expression in bracket must resolve to a primitive (got ${typeof e}).`, - detail: { text: e.resolved }} - ) - return { _error } - } - return e + return new ast.MemberExpression(location(), e) } / "." e:Identifier { return e @@ -186,33 +304,36 @@ MemberExpression } CallExpression - = callee:Identifier __ args:Arguments { - // Workaround for parser issue (calling text() before referencing other values) - const functionName = callee - const _args = args - - return callHelperFunction({ functionName, args: _args, text: text(), allowPartial: options.allowPartial }) + = functionName:Identifier __ args:Arguments { + return new ast.FunctionCallExpression(location(), functionName, args) } Arguments - = "(" __ args:(ArgumentList __)? ")" { - return optionalList(extractOptional(args, 0)); + = "(" __ args:ArgumentList? __ ")" { + return args || []; } - ArgumentList - = head:Expression tail:(__ "," __ Expression)* { - return buildList(head, tail, 3); + = head:Expression tail:ArgumentListTail* { + return [head, ...(tail || [])]; + } +ArgumentListTail + = tail:(__ "," __ Expression) { + return tail[3] } ArrayLiteral + = v:_ArrayLiteral { + return new ast.ArrayLiteralExpression(location(), v) + } +_ArrayLiteral = "[" __ elision:(Elision __)? "]" { - return resolveList(optionalList(extractOptional(elision, 0))); + return optionalList(extractOptional(elision, 0)); } / "[" __ elements:ElementList __ "]" { - return resolveList(elements); + return elements; } / "[" __ elements:ElementList __ "," __ elision:(Elision __)? "]" { - return resolveList(elements.concat(optionalList(extractOptional(elision, 0)))); + return elements.concat(optionalList(extractOptional(elision, 0))); } ElementList @@ -233,24 +354,20 @@ Elision PrimaryExpression = v:NonStringLiteral { - return v + return new ast.LiteralExpression(location(), v) } / v:StringLiteral { + // Do not parse empty strings. + if (v === "") { + return new ast.LiteralExpression(location(), "") + } // Allow nested template strings in literals - return resolveNested(v) + return parseNested(v) } / ArrayLiteral / CallExpression / key:MemberExpression { - key = resolveList(key) - if (key._error) { - return key - } - try { - return getKey(key, { allowPartial: options.allowPartial }) - } catch (err) { - return { _error: err } - } + return new ast.ContextLookupExpression(location(), key) } / "(" __ e:Expression __ ")" { return e @@ -259,16 +376,13 @@ PrimaryExpression UnaryExpression = PrimaryExpression / operator:UnaryOperator __ argument:UnaryExpression { - const v = argument - - if (v && v._error) { - return v - } - - if (operator === "typeof") { - return typeof getValue(v) - } else if (operator === "!") { - return !getValue(v) + switch (operator) { + case "typeof": + return new ast.TypeofExpression(location(), argument) + case "!": + return new ast.NotExpression(location(), argument) + default: + throw new TemplateStringError({ message: `Unrecognized unary operator: ${operator}`, rawTemplateString, loc: location() }) } } @@ -277,44 +391,8 @@ UnaryOperator / "!" ContainsExpression - = head:UnaryExpression __ ContainsOperator __ tail:UnaryExpression { - if (head && head._error) { - return head - } - if (tail && tail._error) { - return tail - } - - head = getValue(head) - tail = getValue(tail) - - if (!isPrimitive(tail)) { - return { - _error: new TemplateStringError({ - message:`The right-hand side of a 'contains' operator must be a string, number, boolean or null (got ${typeof tail}).`, - detail: {} - }) - } - } - - const headType = head === null ? "null" : typeof head - - if (headType === "object") { - if (isArray(head)) { - return head.includes(tail) - } else { - return head.hasOwnProperty(tail) - } - } else if (headType === "string") { - return head.includes(tail.toString()) - } else { - return { - _error: new TemplateStringError({ - message: `The left-hand side of a 'contains' operator must be a string, array or object (got ${headType}).`, - detail: {} - }) - } - } + = iterable:UnaryExpression __ ContainsOperator __ element:UnaryExpression { + return new ast.ContainsExpression(location(), "contains", iterable, element) } / UnaryExpression @@ -364,7 +442,7 @@ EqualityOperator LogicalANDExpression = head:EqualityExpression tail:(__ LogicalANDOperator __ EqualityExpression)* - { return buildLogicalExpression(head, tail, options); } + { return buildLogicalExpression(head, tail); } LogicalANDOperator = "&&" @@ -372,7 +450,7 @@ LogicalANDOperator LogicalORExpression = head:LogicalANDExpression tail:(__ LogicalOROperator __ LogicalANDExpression)* - { return buildLogicalExpression(head, tail, options); } + { return buildLogicalExpression(head, tail); } LogicalOROperator = "||" @@ -382,10 +460,7 @@ ConditionalExpression "?" __ consequent:Expression __ ":" __ alternate:Expression { - if (test && test._error) { - return test - } - return getValue(test) ? consequent : alternate + return new ast.TernaryExpression(location(), test, consequent, alternate) } / LogicalORExpression @@ -430,13 +505,15 @@ SingleLineComment = "//" (!LineTerminator SourceCharacter)* Identifier - = !ReservedWord name:IdentifierName { return name; } + = !ReservedWord name:IdentifierName { return new ast.IdentifierExpression(location(), name) } IdentifierName "identifier" = head:IdentifierStart tail:IdentifierPart* { return head + tail.join("") } - / Integer + / Integer { + return text(); + } IdentifierStart = UnicodeLetter diff --git a/core/src/template-string/proxy.ts b/core/src/template-string/proxy.ts new file mode 100644 index 0000000000..e5ceec8a1a --- /dev/null +++ b/core/src/template-string/proxy.ts @@ -0,0 +1,114 @@ +/* + * Copyright (C) 2018-2023 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +import { memoize } from "lodash-es" +import type { ConfigContext, ContextResolveOpts } from "../config/template-contexts/base.js" +import type { Collection, CollectionOrValue } from "../util/objects.js" +import { isArray, isPlainObject } from "../util/objects.js" +import type { TemplatePrimitive, TemplateValue } from "./inputs.js" +import { isTemplateLeaf, isTemplateValue } from "./inputs.js" +import { evaluate } from "./lazy.js" +import { InternalError } from "../exceptions.js" + +export const getCollectionSymbol = Symbol("GetCollection") + +type LazyConfigProxyParams = { + parsedConfig: CollectionOrValue + expectedCollectionType?: "object" | "array" + context: ConfigContext + opts: ContextResolveOpts +} +export function getLazyConfigProxy({ + parsedConfig, + expectedCollectionType = "object", + context, + opts, +}: LazyConfigProxyParams): Collection { + const getCollection = memoize(() => { + const collection = evaluate({ value: parsedConfig, context, opts }) + + if (isTemplateLeaf(collection)) { + throw new InternalError({ + message: "getLazyConfigProxy: Expected a collection, got a leaf value", + }) + } + + if (expectedCollectionType === "object" && !isPlainObject(collection)) { + throw new InternalError({ + message: `getLazyConfigProxy: Expected an object, got array`, + }) + } + + if (expectedCollectionType === "array" && !isArray(collection)) { + throw new InternalError({ + message: `getLazyConfigProxy: Expected an array, got object`, + }) + } + + return collection + }) + + const proxy = new Proxy(expectedCollectionType === "array" ? [] : {}, { + get(_, prop) { + if (prop === getCollectionSymbol) { + return getCollection() + } + + const collection = getCollection() + + const value = collection[prop] + + if (!isTemplateValue(value) && !isArray(value) && !isPlainObject(value)) { + return value + } + + if (typeof prop === "symbol") { + return value + } + + const evaluated = evaluate({ value, context, opts }) + + if (isTemplateLeaf(evaluated)) { + return evaluated.value + } + + if (isArray(evaluated)) { + return getLazyConfigProxy({ + parsedConfig: evaluated, + expectedCollectionType: "array", + context, + opts, + }) + } + + return getLazyConfigProxy({ parsedConfig: evaluated, context, opts }) + }, + ownKeys() { + return Object.getOwnPropertyNames(getCollection()) + }, + has(_, key) { + return key in getCollection() || Object.hasOwn(getCollection(), key) + }, + getOwnPropertyDescriptor(_, key) { + return Object.getOwnPropertyDescriptor(getCollection(), key) + }, + set(_, key, value) { + throw new InternalError({ + message: `getLazyConfigProxy: Attempted to set key ${String(key)} to value ${JSON.stringify( + value + )} on lazy config proxy`, + }) + }, + }) as Collection + + // TODO + // // This helps when looking at proxy instances in the debugger. + // // The debugger lists symbol properties and that enables you to dig into the backing AST, if you want. + // proxy[Symbol.for("BackingCollection")] = parsedConfig + + return proxy +} diff --git a/core/src/template-string/static-analysis.ts b/core/src/template-string/static-analysis.ts new file mode 100644 index 0000000000..0cf99c943f --- /dev/null +++ b/core/src/template-string/static-analysis.ts @@ -0,0 +1,87 @@ +/* + * Copyright (C) 2018-2023 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { isNumber, isString, startsWith } from "lodash-es" +import { CollectionOrValue, isArray, isPlainObject } from "../util/objects.js" +import { ContextLookupExpression, IdentifierExpression, LiteralExpression, MemberExpression, TemplateExpression } from "./ast.js" +import { TemplateValue } from "./inputs.js" +import { LazyValue } from "./lazy.js" +import { ObjectPath } from "../config/template-contexts/base.js" + +export type TemplateExpressionGenerator = Generator +export function* visitAll(value: CollectionOrValue): TemplateExpressionGenerator { + if (isArray(value)) { + for (const [k, v] of value.entries()) { + yield* visitAll(v) + } + } else if (isPlainObject(value)) { + for (const k of Object.keys(value)) { + yield* visitAll(value[k]) + } + } else { + yield value + + if (value instanceof LazyValue) { + yield* value.visitAll() + } + } +} + +export function containsLazyValues(value: CollectionOrValue): boolean { + for (const node of visitAll(value)) { + if (node instanceof LazyValue) { + return true + } + } + + return false +} + +export function containsContextLookupReferences(value: CollectionOrValue, path: ObjectPath): boolean { + for (const keyPath of getContextLookupReferences(value)) { + if (startsWith(`${keyPath.join(".")}.`, `${path.join(".")}.`)) { + return true + } + } + + return false +} + +export function* getContextLookupReferences( + value: CollectionOrValue +): Generator { + for (const expression of visitAll(value)) { + if (expression instanceof ContextLookupExpression) { + const keyPath: (string | number)[] = [] + + for (const v of expression.keyPath.values()) { + if (v instanceof IdentifierExpression) { + keyPath.push(v.name) + } else if (v instanceof MemberExpression) { + if (v.innerExpression instanceof LiteralExpression) { + if (isString(v.innerExpression.literal) || isNumber(v.innerExpression.literal)) { + keyPath.push(v.innerExpression.literal) + } else { + // only strings and numbers are valid here + break + } + } else { + // it's a dynamic key, so we can't know the value + break + } + } else { + v satisfies never + } + } + + if (keyPath.length > 0) { + yield keyPath + } + } + } +} diff --git a/core/src/template-string/template-string.ts b/core/src/template-string/template-string.ts index 3ee157fc89..d74962529b 100644 --- a/core/src/template-string/template-string.ts +++ b/core/src/template-string/template-string.ts @@ -6,17 +6,20 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import type { GardenErrorParams } from "../exceptions.js" -import { ConfigurationError, GardenError, TemplateStringError } from "../exceptions.js" +import { + ConfigurationError, + GardenError, + InternalError, + NotImplementedError, + TemplateStringError, +} from "../exceptions.js" import type { ConfigContext, ContextKeySegment, ContextResolveOpts, - ContextResolveOutput, + ObjectPath, } from "../config/template-contexts/base.js" -import { GenericContext, ScanContext } from "../config/template-contexts/base.js" -import cloneDeep from "fast-copy" -import { difference, isNumber, isPlainObject, isString, uniq } from "lodash-es" +import { difference, isPlainObject, isString, mapValues } from "lodash-es" import type { ActionReference, Primitive, StringMap } from "../config/common.js" import { arrayConcatKey, @@ -27,64 +30,63 @@ import { conditionalKey, conditionalThenKey, isPrimitive, - isSpecialKey, objectSpreadKey, } from "../config/common.js" -import { dedent, deline, naturalList, titleize, truncate } from "../util/string.js" -import type { ObjectWithName } from "../util/util.js" -import type { Log } from "../logger/log-entry.js" +import { naturalList, titleize } from "../util/string.js" import type { ModuleConfigContext } from "../config/template-contexts/module.js" -import { callHelperFunction } from "./functions.js" import type { ActionKind } from "../actions/types.js" import { actionKindsLower } from "../actions/types.js" -import { deepMap } from "../util/objects.js" -import type { ConfigSource } from "../config/validation.js" +import { type CollectionOrValue, deepMap, isArray } from "../util/objects.js" import * as parser from "./parser.js" -import { styles } from "../logger/styles.js" +import * as ast from "./ast.js" +import { TemplateLeaf, isTemplateLeafValue, templateLeafValueDeepMap } from "./inputs.js" +import type { TemplateLeafValue, TemplateValue } from "./inputs.js" +import { + ConcatLazyValue, + ConditionalLazyValue, + ForEachLazyValue, + ObjectSpreadLazyValue, + ObjectSpreadOperation, + TemplateStringLazyValue, + deepEvaluateAndUnwrap, +} from "./lazy.js" +import { ConfigSource } from "../config/validation.js" +import { Optional } from "utility-types" -const missingKeyExceptionType = "template-string-missing-key" -const passthroughExceptionType = "template-string-passthrough" const escapePrefix = "$${" -export class TemplateStringMissingKeyException extends GardenError { - type = missingKeyExceptionType -} - -export class TemplateStringPassthroughException extends GardenError { - type = passthroughExceptionType +type TemplateErrorParams = { + message: string + source: TemplateProvenance } -interface ResolvedClause extends ContextResolveOutput { - block?: "if" | "else" | "else if" | "endif" - _error?: Error -} +export class TemplateError extends GardenError { + type = "template" -interface ConditionalTree { - type: "root" | "if" | "else" | "value" - value?: any - children: ConditionalTree[] - parent?: ConditionalTree + constructor(params: TemplateErrorParams) { + // TODO: use params.source to improve error message quality + super({ message: params.message }) + } } -function getValue(v: Primitive | undefined | ResolvedClause) { - return isPlainObject(v) ? (v).resolved : v +export type TemplateProvenance = { + yamlPath: ObjectPath + source: ConfigSource | undefined } -type ObjectPath = (string | number)[] - -export class TemplateError extends GardenError { - type = "template" - - path: ObjectPath | undefined - value: any - resolved: any - - constructor(params: GardenErrorParams & { path: ObjectPath | undefined; value: any; resolved: any }) { - super(params) - this.path = params.path - this.value = params.value - this.resolved = params.resolved - } +export function resolveTemplateString({ + string, + context, + contextOpts, + source, +}: { + string: string + context: ConfigContext + contextOpts?: ContextResolveOpts + source?: TemplateProvenance +}): any { + const result = parseTemplateString({ string, source, unescape: contextOpts?.unescape }) + return deepEvaluateAndUnwrap({ value: result, context, opts: contextOpts || {} }) } /** @@ -94,495 +96,267 @@ export class TemplateError extends GardenError { * * The context should be a ConfigContext instance. The optional `stack` parameter is used to detect circular * dependencies when resolving context variables. + * + * TODO: Update docstring to also talk about resolved reference tracking. */ -export function resolveTemplateString({ +export function parseTemplateString({ string, - context, - contextOpts = {}, - path, + source, + // TODO: remove unescape + unescape = false, }: { string: string - context: ConfigContext - contextOpts?: ContextResolveOpts - path?: ObjectPath -}): any { + source?: TemplateProvenance + unescape?: boolean +}): CollectionOrValue { + if (source === undefined) { + source = { + yamlPath: [], + source: undefined, + } + } + // Just return immediately if this is definitely not a template string if (!maybeTemplateString(string)) { - return string + return new TemplateLeaf({ + expr: undefined, + value: string, + inputs: {}, + }) } - try { - const parsed = parser.parse(string, { - getKey: (key: string[], resolveOpts?: ContextResolveOpts) => { - return context.resolve({ key, nodePath: [], opts: { ...contextOpts, ...(resolveOpts || {}) } }) - }, - getValue, - resolveNested: (nested: string) => resolveTemplateString({ string: nested, context, contextOpts }), - buildBinaryExpression, - buildLogicalExpression, - isArray: Array.isArray, - ConfigurationError, + const parse = (str: string) => { + const parsed: ast.TemplateExpression = parser.parse(str, { + grammarSource: source, + rawTemplateString: string, + ast, TemplateStringError, - missingKeyExceptionType, - passthroughExceptionType, - allowPartial: !!contextOpts.allowPartial, - unescape: !!contextOpts.unescape, + // TODO: What is unescape? + unescape, escapePrefix, optionalSuffix: "}?", - isPlainObject, - isPrimitive, - callHelperFunction, - }) - - const outputs: ResolvedClause[] = parsed.map((p: any) => { - return isPlainObject(p) ? p : { resolved: getValue(p) } + // TODO: This should not be done via recursion, but should be handled in the pegjs grammar. + parseNested: (nested: string) => { + return parse(nested) + }, }) - // We need to manually propagate errors in the parser, so we catch them here - for (const r of outputs) { - if (r && r["_error"]) { - throw r["_error"] - } - } - - // Use value directly if there is only one (or no) value in the output. - let resolved: any = outputs[0]?.resolved - - if (outputs.length > 1) { - // Assemble the parts into a conditional tree - const tree: ConditionalTree = { - type: "root", - children: [], - } - let currentNode = tree - - for (const part of outputs) { - if (part.block === "if") { - const node: ConditionalTree = { - type: "if", - value: !!part.resolved, - children: [], - parent: currentNode, - } - currentNode.children.push(node) - currentNode = node - } else if (part.block === "else") { - if (currentNode.type !== "if") { - throw new TemplateStringError({ - message: "Found ${else} block without a preceding ${if...} block.", - }) - } - const node: ConditionalTree = { - type: "else", - value: !currentNode.value, - children: [], - parent: currentNode.parent, - } - currentNode.parent!.children.push(node) - currentNode = node - } else if (part.block === "endif") { - if (currentNode.type === "if" || currentNode.type === "else") { - currentNode = currentNode.parent! - } else { - throw new TemplateStringError({ - message: "Found ${endif} block without a preceding ${if...} block.", - }) - } - } else { - const v = getValue(part) - - currentNode.children.push({ - type: "value", - value: v === null ? "null" : v, - children: [], - }) - } - } - - if (currentNode.type === "if" || currentNode.type === "else") { - throw new TemplateStringError({ message: "Missing ${endif} after ${if ...} block." }) - } + return parsed + } - // Walk down tree and resolve the output string - resolved = "" - function resolveTree(node: ConditionalTree) { - if (node.type === "value" && node.value !== undefined) { - resolved += node.value - } else if (node.type === "root" || ((node.type === "if" || node.type === "else") && !!node.value)) { - for (const child of node.children) { - resolveTree(child) - } - } - } - - resolveTree(tree) - } + const parsed = parse(string) - return resolved - } catch (err) { - if (!(err instanceof GardenError)) { - throw err - } - const prefix = `Invalid template string (${styles.accent(truncate(string, 35).replace(/\n/g, "\\n"))}): ` - const message = err.message.startsWith(prefix) ? err.message : prefix + err.message + return new TemplateStringLazyValue({ + source, + astRootNode: parsed, + expr: string, + }) +} - throw new TemplateStringError({ message, path }) +/** + * Returns a new ContextResolveOpts where part is appended to only yamlPath in contextOpts + * + * @param part ObjectPath element to append to yamlPath in contextOpts + * @param contextOpts ContextResolveOpts + * @returns + */ +export function pushYamlPath( + part: ObjectPath[0], + contextOpts: T +): T & { yamlPath: ObjectPath } { + return { + ...contextOpts, + yamlPath: [...(contextOpts.yamlPath || []), part], } } /** * Recursively parses and resolves all templated strings in the given object. */ - -// `extends any` here isn't pretty but this function is hard to type correctly -// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-constraint -export function resolveTemplateStrings({ +export function parseTemplateCollection({ value, - context, - contextOpts = {}, - path, - source, + source: _source, + untemplatableKeys = [], }: { - value: T - context: ConfigContext - contextOpts?: ContextResolveOpts - path?: ObjectPath - source: ConfigSource | undefined -}): T { - if (value === null) { - return null as T - } - if (value === undefined) { - return undefined as T - } - - if (!path) { - path = [] + value: CollectionOrValue + source: Optional + untemplatableKeys?: string[] +}): CollectionOrValue { + let source: TemplateProvenance = { + ..._source, + yamlPath: _source.yamlPath || [], } if (typeof value === "string") { - return resolveTemplateString({ string: value, context, path, contextOpts }) - } else if (Array.isArray(value)) { - const output: unknown[] = [] - - value.forEach((v, i) => { - if (isPlainObject(v) && v[arrayConcatKey] !== undefined) { - if (Object.keys(v).length > 1) { - const extraKeys = naturalList( - Object.keys(v) - .filter((k) => k !== arrayConcatKey) - .map((k) => JSON.stringify(k)) - ) - throw new TemplateError({ - message: `A list item with a ${arrayConcatKey} key cannot have any other keys (found ${extraKeys})`, - path, - value, - resolved: undefined, - }) - } - - // Handle array concatenation via $concat - const resolved = resolveTemplateStrings({ - value: v[arrayConcatKey], - context, - contextOpts: { - ...contextOpts, - }, - path: path && [...path, arrayConcatKey], - source, - }) - - if (Array.isArray(resolved)) { - output.push(...resolved) - } else if (contextOpts.allowPartial) { - output.push({ $concat: resolved }) - } else { - throw new TemplateError({ - message: `Value of ${arrayConcatKey} key must be (or resolve to) an array (got ${typeof resolved})`, - path, - value, - resolved, - }) - } - } else { - output.push(resolveTemplateStrings({ value: v, context, contextOpts, source, path: path && [...path, i] })) - } + return parseTemplateString({ string: value, source }) + } else if (isTemplateLeafValue(value)) { + // here we handle things static numbers, empty array etc + // we also handle null and undefined + return new TemplateLeaf({ + expr: undefined, + value, + inputs: {}, }) - - return (output) + } else if (isArray(value)) { + const resolvedValues = value.map((v, i) => + parseTemplateCollection({ value: v, source: pushYamlPath(i, source) }) + ) + // we know that this is not handling an empty array, as that would have been a TemplatePrimitive. + if (value.some((v) => v?.[arrayConcatKey] !== undefined)) { + return new ConcatLazyValue(source, resolvedValues) + } else { + return resolvedValues + } } else if (isPlainObject(value)) { if (value[arrayForEachKey] !== undefined) { - // Handle $forEach loop - return handleForEachObject({ value, context, contextOpts, path, source }) - } else if (value[conditionalKey] !== undefined) { - // Handle $if conditional - return handleConditional({ value, context, contextOpts, path, source }) - } else { - // Resolve $merge keys, depth-first, leaves-first - let output = {} - - for (const [k, v] of Object.entries(value)) { - const resolved = resolveTemplateStrings({ value: v, context, contextOpts, source, path: path && [...path, k] }) - - if (k === objectSpreadKey) { - if (isPlainObject(resolved)) { - output = { ...output, ...resolved } - } else if (contextOpts.allowPartial) { - output[k] = resolved - } else { - throw new TemplateError({ - message: `Value of ${objectSpreadKey} key must be (or resolve to) a mapping object (got ${typeof resolved})`, - path: [...path, k], - value, - resolved, - }) - } - } else { - output[k] = resolved - } - } + const unexpectedKeys = Object.keys(value).filter((k) => !ForEachLazyValue.allowedForEachKeys.includes(k)) - return output - } - } else { - return value - } -} + if (unexpectedKeys.length > 0) { + const extraKeys = naturalList(unexpectedKeys.map((k) => JSON.stringify(k))) -const expectedForEachKeys = [arrayForEachKey, arrayForEachReturnKey, arrayForEachFilterKey] + throw new TemplateError({ + message: `Found one or more unexpected keys on ${arrayForEachKey} object: ${extraKeys}. Allowed keys: ${naturalList( + ForEachLazyValue.allowedForEachKeys + )}`, + source: pushYamlPath(extraKeys[0], source), + }) + } -function handleForEachObject({ - value, - context, - contextOpts, - path, - source, -}: { - value: any - context: ConfigContext - contextOpts: ContextResolveOpts - path: ObjectPath | undefined - source: ConfigSource | undefined -}) { - // Validate input object - if (value[arrayForEachReturnKey] === undefined) { - throw new TemplateError({ - message: `Missing ${arrayForEachReturnKey} field next to ${arrayForEachKey} field. Got ${naturalList( - Object.keys(value) - )}`, - path: path && [...path, arrayForEachKey], - value, - resolved: undefined, - }) - } + if (value[arrayForEachReturnKey] === undefined) { + throw new TemplateError({ + message: `Missing ${arrayForEachReturnKey} field next to ${arrayForEachKey} field. Got ${naturalList( + Object.keys(value) + )}`, + source: pushYamlPath(arrayForEachReturnKey, source), + }) + } - const unexpectedKeys = Object.keys(value).filter((k) => !expectedForEachKeys.includes(k)) + const resolvedCollectionExpression = parseTemplateCollection({ + value: value[arrayForEachKey], + source: pushYamlPath(arrayForEachKey, source), + }) - if (unexpectedKeys.length > 0) { - const extraKeys = naturalList(unexpectedKeys.map((k) => JSON.stringify(k))) + const resolvedReturnExpression = parseTemplateCollection({ + value: value[arrayForEachReturnKey], + source: pushYamlPath(arrayForEachReturnKey, source), + }) - throw new TemplateError({ - message: `Found one or more unexpected keys on ${arrayForEachKey} object: ${extraKeys}. Expected keys: ${naturalList( - expectedForEachKeys - )}`, - path, - value, - resolved: undefined, - }) - } - // Try resolving the value of the $forEach key - let resolvedInput = resolveTemplateStrings({ - value: value[arrayForEachKey], - context, - contextOpts, - source, - path: path && [...path, arrayForEachKey], - }) - const isObject = isPlainObject(resolvedInput) + const resolvedFilterExpression = + value[arrayForEachFilterKey] === undefined + ? undefined + : parseTemplateCollection({ + value: value[arrayForEachFilterKey], + source: pushYamlPath(arrayForEachFilterKey, source), + }) - if (!Array.isArray(resolvedInput) && !isObject) { - if (contextOpts.allowPartial) { - return value - } else { - throw new TemplateError({ - message: `Value of ${arrayForEachKey} key must be (or resolve to) an array or mapping object (got ${typeof resolvedInput})`, - path: path && [...path, arrayForEachKey], - value, - resolved: resolvedInput, + const forEach = new ForEachLazyValue(source, { + [arrayForEachKey]: resolvedCollectionExpression, + [arrayForEachReturnKey]: resolvedReturnExpression, + [arrayForEachFilterKey]: resolvedFilterExpression, }) - } - } - if (isObject) { - const keys = Object.keys(resolvedInput) - const inputContainsSpecialKeys = keys.some((key) => isSpecialKey(key)) - - if (inputContainsSpecialKeys) { - // If partial application is enabled - // we cannot be sure if the object can be evaluated correctly. - // There could be an expression in there that goes `{foo || bar}` - // and `foo` is only to be filled in at a later time, so resolving now would force it to be `bar`. - // Thus we return the entire object - // - // If partial application is disabled - // then we need to make sure that the resulting expression is evaluated again - // since the magic keys only get resolved via `resolveTemplateStrings` - if (contextOpts.allowPartial) { - return value + // This ensures that we only handle $concat operators that literally are hardcoded in the yaml, + // and not ones that are results of other expressions. + if (resolvedReturnExpression[arrayConcatKey] !== undefined) { + return new ConcatLazyValue(source, forEach) + } else { + return forEach } + } else if (value[conditionalKey] !== undefined) { + const ifExpression = value[conditionalKey] + const thenExpression = value[conditionalThenKey] + const elseExpression = value[conditionalElseKey] - resolvedInput = resolveTemplateStrings({ value: resolvedInput, context, contextOpts, source: undefined }) - } - } - - const filterExpression = value[arrayForEachFilterKey] - - // TODO: maybe there's a more efficient way to do the cloning/extending? - const loopContext = cloneDeep(context) - - const output: unknown[] = [] - - for (const i of Object.keys(resolvedInput)) { - const itemValue = resolvedInput[i] - - loopContext["item"] = new GenericContext({ key: i, value: itemValue }) + if (thenExpression === undefined) { + throw new TemplateError({ + message: `Missing ${conditionalThenKey} field next to ${conditionalKey} field. Got: ${naturalList( + Object.keys(value) + )}`, + source, + }) + } - // Have to override the cache in the parent context here - // TODO: make this a little less hacky :P - const resolvedValues = loopContext["_resolvedValues"] - delete resolvedValues["item.key"] - delete resolvedValues["item.value"] - const subValues = Object.keys(resolvedValues).filter((k) => k.match(/item\.value\.*/)) - subValues.forEach((v) => delete resolvedValues[v]) + const unexpectedKeys = Object.keys(value).filter((k) => !ConditionalLazyValue.allowedConditionalKeys.includes(k)) - // Check $filter clause output, if applicable - if (filterExpression !== undefined) { - const filterResult = resolveTemplateStrings({ - value: value[arrayForEachFilterKey], - context: loopContext, - contextOpts, - source, - path: path && [...path, arrayForEachFilterKey], - }) + if (unexpectedKeys.length > 0) { + const extraKeys = naturalList(unexpectedKeys.map((k) => JSON.stringify(k))) - if (filterResult === false) { - continue - } else if (filterResult !== true) { throw new TemplateError({ - message: `${arrayForEachFilterKey} clause in ${arrayForEachKey} loop must resolve to a boolean value (got ${typeof resolvedInput})`, - path: path && [...path, arrayForEachFilterKey], - value, - resolved: undefined, + message: `Found one or more unexpected keys on ${conditionalKey} object: ${extraKeys}. Allowed: ${naturalList( + ConditionalLazyValue.allowedConditionalKeys + )}`, + source, }) } - } - output.push( - resolveTemplateStrings({ - value: value[arrayForEachReturnKey], - context: loopContext, - contextOpts, - source, - path: path && [...path, arrayForEachKey, i], + return new ConditionalLazyValue(source, { + [conditionalKey]: parseTemplateCollection({ + value: ifExpression, + source: pushYamlPath(conditionalKey, source), + }), + [conditionalThenKey]: parseTemplateCollection({ + value: thenExpression, + source: pushYamlPath(conditionalThenKey, source), + }), + [conditionalElseKey]: + elseExpression === undefined + ? undefined + : parseTemplateCollection({ + value: elseExpression, + source: pushYamlPath(conditionalElseKey, source), + }), }) - ) - } + } else { + const resolved = mapValues(value, (v, k) => { + // if this key is untemplatable, skip parsing this branch of the template tree. + if (untemplatableKeys.includes(k)) { + return templateLeafValueDeepMap(v, (v) => { + return new TemplateLeaf({ + expr: undefined, + value: v, + inputs: {}, + }) + }) + } - // Need to resolve once more to handle e.g. $concat expressions - return resolveTemplateStrings({ value: output, context, contextOpts, source, path }) + return parseTemplateCollection({ value: v, source: pushYamlPath(k, source) }) + }) + if (Object.keys(value).some((k) => k === objectSpreadKey)) { + return new ObjectSpreadLazyValue(source, resolved as ObjectSpreadOperation) + } else { + return resolved + } + } + } else { + throw new InternalError({ + message: `Got unexpected value type: ${typeof value}`, + }) + } } -const expectedConditionalKeys = [conditionalKey, conditionalThenKey, conditionalElseKey] - -function handleConditional({ +// `extends any` here isn't pretty but this function is hard to type correctly +// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-constraint +// TODO: use TemplateCollectionOrValue instead of T; T is lying here, as is any. +export function resolveTemplateStrings({ value, context, contextOpts, - path, - source, + source: _source, }: { - value: any + value: T context: ConfigContext - contextOpts: ContextResolveOpts - path: ObjectPath | undefined - source: ConfigSource | undefined -}) { - // Validate input object - const thenExpression = value[conditionalThenKey] - const elseExpression = value[conditionalElseKey] - - if (thenExpression === undefined) { - throw new TemplateError({ - message: `Missing ${conditionalThenKey} field next to ${conditionalKey} field. Got: ${naturalList( - Object.keys(value) - )}`, - path, - value, - resolved: undefined, - }) - } - - const unexpectedKeys = Object.keys(value).filter((k) => !expectedConditionalKeys.includes(k)) - - if (unexpectedKeys.length > 0) { - const extraKeys = naturalList(unexpectedKeys.map((k) => JSON.stringify(k))) - - throw new TemplateError({ - message: `Found one or more unexpected keys on ${conditionalKey} object: ${extraKeys}. Expected: ${naturalList( - expectedConditionalKeys - )}`, - path, - value, - resolved: undefined, - }) - } - - // Try resolving the value of the $if key - const resolvedConditional = resolveTemplateStrings({ - value: value[conditionalKey], - context, - contextOpts, - source, - path: path && [...path, conditionalKey], - }) - - if (typeof resolvedConditional !== "boolean") { - if (contextOpts.allowPartial) { - return value - } else { - throw new TemplateError({ - message: `Value of ${conditionalKey} key must be (or resolve to) a boolean (got ${typeof resolvedConditional})`, - path: path && [...path, conditionalKey], - value, - resolved: resolvedConditional, - }) - } - } - - // Note: We implicitly default the $else value to undefined - - const resolvedThen = resolveTemplateStrings({ - value: thenExpression, - context, - path: path && [...path, conditionalThenKey], - contextOpts, - source, - }) - const resolvedElse = resolveTemplateStrings({ - value: elseExpression, - context, - path: path && [...path, conditionalElseKey], - contextOpts, - source, - }) - - if (!!resolvedConditional) { - return resolvedThen - } else { - return resolvedElse + contextOpts?: ContextResolveOpts + source?: ConfigSource | undefined +}): T { + const source = { + yamlPath: [], + source: _source, } + const resolved = parseTemplateCollection({ value: value as any, source }) + // First evaluate lazy values deeply, then remove the leaves + return deepEvaluateAndUnwrap({ value: resolved, context, opts: contextOpts || {} }) as any // TODO: The type is a lie! } /** @@ -615,9 +389,10 @@ export function mayContainTemplateString(obj: any): boolean { * Scans for all template strings in the given object and lists the referenced keys. */ export function collectTemplateReferences(obj: T): ContextKeySegment[][] { - const context = new ScanContext() - resolveTemplateStrings({ value: obj, context, contextOpts: { allowPartial: true }, source: undefined }) - return uniq(context.foundKeys.entries()).sort() + throw new NotImplementedError({ message: "TODO: Traverse ast to get references" }) + // const context = new ScanContext() + // resolveTemplateStrings({ value: obj, context, contextOpts: { allowPartial: true, } }) + // return uniq(context.foundKeys.entries()).sort() } export function getRuntimeTemplateReferences(obj: T) { @@ -734,60 +509,6 @@ export function getModuleTemplateReferences(obj: T, context: M return resolveTemplateStrings({ value: moduleNames, context, source: undefined }) } -/** - * Gathers secret references in configs and throws an error if one or more referenced secrets isn't present (or has - * blank values) in the provided secrets map. - * - * Prefix should be e.g. "Module" or "Provider" (used when generating error messages). - * - * TODO: We've disabled this for now. Re-introduce once we've removed get config command call from GE! - */ -export function throwOnMissingSecretKeys(configs: ObjectWithName[], secrets: StringMap, prefix: string, log?: Log) { - const allMissing: [string, ContextKeySegment[]][] = [] // [[key, missing keys]] - for (const config of configs) { - const missing = detectMissingSecretKeys(config, secrets) - if (missing.length > 0) { - allMissing.push([config.name, missing]) - } - } - - if (allMissing.length === 0) { - return - } - - const descriptions = allMissing.map(([key, missing]) => `${prefix} ${key}: ${missing.join(", ")}`) - /** - * Secret keys with empty values should have resulted in an error by this point, but we filter on keys with - * values for good measure. - */ - const loadedKeys = Object.entries(secrets) - .filter(([_key, value]) => value) - .map(([key, _value]) => key) - let footer: string - if (loadedKeys.length === 0) { - footer = deline` - Note: No secrets have been loaded. If you have defined secrets for the current project and environment in Garden - Cloud, this may indicate a problem with your configuration. - ` - } else { - footer = `Secret keys with loaded values: ${loadedKeys.join(", ")}` - } - const errMsg = dedent` - The following secret names were referenced in configuration, but are missing from the secrets loaded remotely: - - ${descriptions.join("\n\n")} - - ${footer} - ` - if (log) { - log.silly(() => errMsg) - } - // throw new ConfigurationError(errMsg, { - // loadedSecretKeys: loadedKeys, - // missingSecretKeys: uniq(flatten(allMissing.map(([_key, missing]) => missing))), - // }) -} - /** * Collects template references to secrets in obj, and returns an array of any secret keys referenced in it that * aren't present (or have blank values) in the provided secrets map. @@ -806,135 +527,3 @@ export function detectMissingSecretKeys(obj: T, secrets: Strin const missingKeys = difference(referencedKeys, keysWithValues) return missingKeys.sort() } - -function buildBinaryExpression(head: any, tail: any) { - return tail.reduce((result: any, element: any) => { - const operator = element[1] - const leftRes = result - const rightRes = element[3] - - // We need to manually handle and propagate errors because the parser doesn't support promises - if (leftRes && leftRes._error) { - return leftRes - } - if (rightRes && rightRes._error) { - return rightRes - } - - const left = getValue(leftRes) - const right = getValue(rightRes) - - // Disallow undefined values for comparisons - if (left === undefined || right === undefined) { - const message = [leftRes, rightRes] - .map((res) => res.message) - .filter(Boolean) - .join(" ") - const err = new TemplateStringError({ - message: message || "Could not resolve one or more keys.", - }) - return { _error: err } - } - - if (operator === "==") { - return left === right - } - if (operator === "!=") { - return left !== right - } - - if (operator === "+") { - if (isNumber(left) && isNumber(right)) { - return left + right - } else if (isString(left) && isString(right)) { - return left + right - } else if (Array.isArray(left) && Array.isArray(right)) { - return left.concat(right) - } else { - const err = new TemplateStringError({ - message: `Both terms need to be either arrays or strings or numbers for + operator (got ${typeof left} and ${typeof right}).`, - }) - return { _error: err } - } - } - - // All other operators require numbers to make sense (we're not gonna allow random JS weirdness) - if (!isNumber(left) || !isNumber(right)) { - const err = new TemplateStringError({ - message: `Both terms need to be numbers for ${operator} operator (got ${typeof left} and ${typeof right}).`, - }) - return { _error: err } - } - - switch (operator) { - case "*": - return left * right - case "/": - return left / right - case "%": - return left % right - case "-": - return left - right - case "<=": - return left <= right - case ">=": - return left >= right - case "<": - return left < right - case ">": - return left > right - default: - const err = new TemplateStringError({ message: "Unrecognized operator: " + operator }) - return { _error: err } - } - }, head) -} - -function buildLogicalExpression(head: any, tail: any, opts: ContextResolveOpts) { - return tail.reduce((result: any, element: any) => { - const operator = element[1] - const leftRes = result - const rightRes = element[3] - - switch (operator) { - case "&&": - if (leftRes && leftRes._error) { - if (!opts.allowPartial && leftRes._error.type === missingKeyExceptionType) { - return false - } - return leftRes - } - - const leftValue = getValue(leftRes) - - if (leftValue === undefined) { - return { resolved: false } - } else if (!leftValue) { - return { resolved: leftValue } - } else { - if (rightRes && rightRes._error) { - if (!opts.allowPartial && rightRes._error.type === missingKeyExceptionType) { - return false - } - return rightRes - } - - const rightValue = getValue(rightRes) - - if (rightValue === undefined) { - return { resolved: false } - } else { - return rightRes - } - } - case "||": - if (leftRes && leftRes._error) { - return leftRes - } - return getValue(leftRes) ? leftRes : rightRes - default: - const err = new TemplateStringError({ message: "Unrecognized operator: " + operator }) - return { _error: err } - } - }, head) -} diff --git a/core/src/template-string/validation.ts b/core/src/template-string/validation.ts new file mode 100644 index 0000000000..3553cc9cdd --- /dev/null +++ b/core/src/template-string/validation.ts @@ -0,0 +1,324 @@ +/* + * Copyright (C) 2018-2023 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { z, infer as inferZodType } from "zod" +import { ConfigContext, ContextResolveOpts } from "../config/template-contexts/base.js" +import { Collection, CollectionOrValue, isArray, isPlainObject } from "../util/objects.js" +import { TemplatePrimitive, TemplateValue } from "./inputs.js" +import { getCollectionSymbol, getLazyConfigProxy } from "./proxy.js" +import Joi from "@hapi/joi" +import { parseTemplateCollection } from "./template-string.js" +import { ConfigSource } from "../config/validation.js" +import { dirname } from "path" + +type Change = { path: (string | number)[]; value: CollectionOrValue } + +// This function gets us all changes and additions from one object to another +// It is not a general purpose method for diffing any two objects. +// We make use of the knowledge that a validated object will only make changes +// by either adding or changing a property on an object, never deleting properties. +// We also know that the object now has been validated so we know that the object will +// afterwards be conforming to the type given during validation, deriving from the base object. +// Thus we only need to track additions or changes, never deletions. +function getChangeset( + base: CollectionOrValue, + compare: CollectionOrValue, + path: (string | number)[] = [], + changeset: Change[] = [] +): Change[] { + if (isArray(base) && isArray(compare)) { + for (let i = 0; i < compare.length; i++) { + getChangeset(base[i], compare[i], [...path, i], changeset) + } + } else if (isPlainObject(base) && isPlainObject(compare)) { + for (const key of Object.keys(compare)) { + getChangeset(base[key], compare[key], [...path, key], changeset) + } + } else if (base !== compare) { + changeset.push({ path, value: compare }) + } + + return changeset +} + +// For overlaying the changesets +function getOverlayProxy( + targetObject: Collection, + changes: Change[], + currentPath: (string | number)[] = [] +): Collection { + // TODO: This needs performance optimization and a proper abstraction to maintain the overlays + const currentPathChanges = changes.filter((change) => change.path.slice(0, -1).join(".") === currentPath.join(".")) + const nextKeys = currentPathChanges + .map((change) => change.path[currentPath.length]) + .filter((key) => typeof key === "string") as string[] + + const proxy = new Proxy(targetObject, { + get(target, prop) { + if (typeof prop === "symbol") { + return target[prop] + } + + const override = changes.find((change) => change.path.join(".") === [...currentPath, prop].join(".")) + + if (override) { + return override.value + } + + if (isArray(target[prop]) || isPlainObject(target[prop])) { + return getOverlayProxy(target[prop], changes, [...currentPath, prop]) + } + + return target[prop] + }, + ownKeys() { + return [...Reflect.ownKeys(targetObject), ...nextKeys] + }, + has(target, key) { + return Reflect.has(target, key) || nextKeys.includes(key as string) + }, + getOwnPropertyDescriptor(target, key) { + return ( + Reflect.getOwnPropertyDescriptor(target, key) || { + configurable: true, + enumerable: true, + writable: false, + } + ) + }, + }) + + return proxy +} + +type GardenConfigParams = { + untemplatableKeys?: string[] + context: ConfigContext + opts: ContextResolveOpts + source: ConfigSource | undefined + configFilePath: string | undefined + metadata: Metadata + overlays?: Change[] + unparsedConfig: CollectionOrValue + parsedConfig?: CollectionOrValue +} + +type TypeAssertion = (object: any) => object is T +export class GardenConfig< + ConfigType extends Collection = Collection, + Metadata = unknown, +> { + private parsedConfig: CollectionOrValue + private unparsedConfig: CollectionOrValue + private context: ConfigContext + private opts: ContextResolveOpts + private overlays: Change[] + + public source: ConfigSource | undefined + public metadata: Metadata + + /** + * The path to the resource's config file, if any. + * + * Configs that are read from a file should always have this set, but generated configs (e.g. from templates + * or `augmentGraph` handlers) don't necessarily have a path on disk. + */ + public configFilePath?: string + + /** + * The path/working directory where commands and operations relating to the config should be executed. This is + * most commonly the directory containing the config file. + * + * Note: When possible, use `action.getSourcePath()` instead, since it factors in remote source paths and source + * overrides (i.e. `BaseActionConfig.source.path`). This is a lower-level field that doesn't contain template strings, + * and can thus be used early in the resolution flow. + */ + public get configFileDirname(): string | undefined { + if (this.configFilePath !== undefined) { + return dirname(this.configFilePath) + } + + return undefined + } + + constructor({ + parsedConfig, + unparsedConfig, + untemplatableKeys = [], + overlays = [], + context, + opts, + source, + configFilePath, + metadata, + }: GardenConfigParams) { + this.metadata = metadata + this.unparsedConfig = unparsedConfig + this.parsedConfig = + parsedConfig || parseTemplateCollection({ value: unparsedConfig, source: { source }, untemplatableKeys }) + this.context = context + this.opts = opts + this.overlays = overlays + this.source = source + this.configFilePath = configFilePath + } + + public transformUnparsedConfig( + transform: ( + unparsedConfig: CollectionOrValue, + context: ConfigContext, + opts: ContextResolveOpts + ) => Collection + ): GardenConfig, Metadata> { + return new GardenConfig({ + unparsedConfig: transform(this.unparsedConfig, this.context, this.opts), + context: this.context, + opts: this.opts, + overlays: [], + source: this.source, + configFilePath: this.configFilePath, + metadata: this.metadata, + }) + } + + public transformParsedConfig( + transform: ( + parsedConfig: CollectionOrValue, + context: ConfigContext, + opts: ContextResolveOpts + ) => CollectionOrValue + ): GardenConfig, Metadata> { + return new GardenConfig({ + parsedConfig: transform(this.parsedConfig, this.context, this.opts), + unparsedConfig: this.unparsedConfig, + context: this.context, + opts: this.opts, + overlays: [], + source: this.source, + configFilePath: this.configFilePath, + metadata: this.metadata, + }) + } + + public withContext( + context: ConfigContext, + opts: ContextResolveOpts = {} + ): GardenConfig, Metadata> { + // we wipe the types, because a new context can result in different results when evaluating template strings + return new GardenConfig({ + parsedConfig: this.parsedConfig, + unparsedConfig: this.unparsedConfig, + context, + opts, + overlays: [], + source: this.source, + configFilePath: this.configFilePath, + metadata: this.metadata, + }) + } + + public assertType>( + assertion: TypeAssertion + ): GardenConfig { + const rawConfig = this.getConfig() + const configIsOfType = assertion(rawConfig) + + if (configIsOfType) { + return new GardenConfig({ + parsedConfig: this.parsedConfig, + unparsedConfig: this.unparsedConfig, + context: this.context, + opts: this.opts, + overlays: this.overlays, + source: this.source, + configFilePath: this.configFilePath, + metadata: this.metadata, + }) + } else { + // TODO: Write a better error message + throw new Error("Config is not of the expected type") + } + } + + public refineWithZod( + validator: Validator + ): GardenConfig, Metadata> { + // merge the schemas + + // instantiate proxy without overlays + const rawConfig = this.getConfig([]) + + // validate config and extract changes + const validated = validator.parse(rawConfig) + const changes = getChangeset(rawConfig as any, validated) + + return new GardenConfig({ + parsedConfig: this.parsedConfig, + unparsedConfig: this.unparsedConfig, + context: this.context, + opts: this.opts, + overlays: [...changes], + source: this.source, + configFilePath: this.configFilePath, + metadata: this.metadata, + }) + } + + // With joi we can't infer the type from the schema + public refineWithJoi>( + validator: Joi.SchemaLike + ): GardenConfig { + // instantiate proxy without overlays + const rawConfig = this.getConfig([]) + + // validate config and extract changes + const validated = Joi.attempt(rawConfig, validator) + const changes = getChangeset(rawConfig as any, validated) + + return new GardenConfig({ + parsedConfig: this.parsedConfig, + unparsedConfig: this.unparsedConfig, + context: this.context, + opts: this.opts, + overlays: [...changes], + source: this.source, + configFilePath: this.configFilePath, + metadata: this.metadata, + }) + } + + // While Readonly would be more accurate, it complicates things when passing the config around + public get config(): ConfigType { + return this.getConfig() + } + + private getConfig(overlays?: Change[]): Readonly { + const configProxy = getLazyConfigProxy({ + parsedConfig: this.parsedConfig, + context: this.context, + opts: this.opts, + }) as ConfigType + + const changes = overlays || this.overlays + if (changes.length > 0) { + return getOverlayProxy(configProxy, changes) as ConfigType + } + + return configProxy + } + + static getTemplateValueTree(proxy: CollectionOrValue): Collection | undefined { + const underlyingTemplateValueTree = proxy?.[getCollectionSymbol] + + if (underlyingTemplateValueTree) { + return underlyingTemplateValueTree as Collection + } + + return undefined + } +} diff --git a/core/src/util/objects.ts b/core/src/util/objects.ts index 82f9029ac4..17c004435a 100644 --- a/core/src/util/objects.ts +++ b/core/src/util/objects.ts @@ -6,23 +6,44 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import { isArray, isPlainObject, mapValues, pickBy } from "lodash-es" +import { isPlainObject as lodashIsPlainObject, mapValues, pickBy } from "lodash-es" +import { ObjectPath } from "../config/template-contexts/base.js" + +export type Collection

= + | CollectionOrValue

[] + | { [key: string]: CollectionOrValue

} + +export type CollectionOrValue

= P | Collection

+ +// adds appropriate type guard to Array.isArray +export function isArray

( + value: CollectionOrValue

+): value is CollectionOrValue

[] { + return Array.isArray(value) +} + +// adds appropriate type guard to lodash isPlainObject +export function isPlainObject

( + value: CollectionOrValue

+): value is { [key: string]: CollectionOrValue

} { + return lodashIsPlainObject(value) +} /** * Recursively process all values in the given input, * walking through all object keys _and array items_. */ -export function deepMap( - value: T | Iterable, - fn: (value: any, key: string | number) => any, - key?: number | string -): U | Iterable { +export function deepMap( + value: CollectionOrValue, + fn: (value: Exclude>, key: string | number, keyPath: ObjectPath) => R, + keyPath: ObjectPath = [] +): CollectionOrValue { if (isArray(value)) { - return value.map((v, k) => deepMap(v, fn, k)) + return value.map((v, k) => deepMap(v, fn, [...keyPath, k])) } else if (isPlainObject(value)) { - return mapValues(value, (v, k) => deepMap((v), fn, k)) + return mapValues(value, (v, k) => deepMap(v, fn, [...keyPath, k])) } else { - return fn(value, key || 0) + return fn(value as Exclude>, keyPath[keyPath.length-1] || 0, keyPath) } } @@ -30,16 +51,16 @@ export function deepMap( * Recursively filter all keys and values in the given input, * walking through all object keys _and array items_. */ -export function deepFilter( - value: T | Iterable, +export function deepFilter( + value: CollectionOrValue, fn: (value: any, key: string | number) => boolean -): U | Iterable { +): CollectionOrValue { if (isArray(value)) { - return >value.filter(fn).map((v) => deepFilter(v, fn)) + return value.filter(fn).map((v) => deepFilter(v, fn)) } else if (isPlainObject(value)) { - return mapValues(pickBy(value, fn), (v) => deepFilter(v, fn)) + return mapValues(pickBy(value, fn), (v) => deepFilter(v, fn)) } else { - return value + return value } } diff --git a/core/src/util/util.ts b/core/src/util/util.ts index 68438f4769..c0fa9a2de0 100644 --- a/core/src/util/util.ts +++ b/core/src/util/util.ts @@ -46,6 +46,8 @@ import split2 from "split2" import type { ExecaError, Options as ExecaOptions } from "execa" import { execa } from "execa" import corePackageJson from "../../package.json" assert { type: "json" } +import { Collection } from "./objects.js" +import { Primitive } from "utility-types" export { apply as jsonMerge } from "json-merge-patch" @@ -695,12 +697,15 @@ export function toEnvVars(vars: PrimitiveMap): { [key: string]: string | undefin * // returns [{ value: 2, duplicateItems: [{ a: 1, b: 2 }, { a: 2, b: 2 }] }] * duplicateKeys(items, "b") */ -export function duplicatesByKey(items: any[], key: string) { - const grouped = groupBy(items, key) +export function duplicatesBy, GroupIdentifier extends Primitive>( + items: TObject[], + valueGetter: (item: TObject) => GroupIdentifier +): { value: GroupIdentifier; duplicateItems: TObject[] }[] { + const grouped = groupBy(items, valueGetter) return Object.entries(grouped) .map(([value, duplicateItems]) => ({ value, duplicateItems })) - .filter(({ duplicateItems }) => duplicateItems.length > 1) + .filter(({ duplicateItems }) => duplicateItems.length > 1) as { value: GroupIdentifier; duplicateItems: TObject[] }[] } export function isNotNull(v: T | null): v is T { diff --git a/core/test/setup.ts b/core/test/setup.ts index a276580088..ae983f37d3 100644 --- a/core/test/setup.ts +++ b/core/test/setup.ts @@ -13,6 +13,8 @@ import { getDefaultProfiler } from "../src/util/profiling.js" import { gardenEnv } from "../src/constants.js" import { testFlags } from "../src/util/util.js" import { initTestLogger, testProjectTempDirs } from "./helpers.js" +import chai from "chai" +chai.config.truncateThreshold = 0 import sourceMapSupport from "source-map-support" sourceMapSupport.install() diff --git a/core/test/unit/src/commands/get/get-config.ts b/core/test/unit/src/commands/get/get-config.ts index f7b280e5f9..037ba37d41 100644 --- a/core/test/unit/src/commands/get/get-config.ts +++ b/core/test/unit/src/commands/get/get-config.ts @@ -701,7 +701,7 @@ describe("GetConfigCommand", () => { opts: withDefaultGlobalOpts({ "exclude-disabled": false, "resolve": "partial" }), }) - expect(res.result!.providers).to.eql(garden.getRawProviderConfigs()) + expect(res.result!.providers).to.eql(garden.getConfiguredProviders()) }) it("should not resolve providers", async () => { diff --git a/core/test/unit/src/config/template-contexts/base.ts b/core/test/unit/src/config/template-contexts/base.ts index 3923abd2e2..6364205afc 100644 --- a/core/test/unit/src/config/template-contexts/base.ts +++ b/core/test/unit/src/config/template-contexts/base.ts @@ -8,7 +8,11 @@ import { expect } from "chai" import stripAnsi from "strip-ansi" -import type { ContextKey, ContextResolveParams } from "../../../../../src/config/template-contexts/base.js" +import type { + ContextKey, + ContextResolveOpts, + ContextResolveParams, +} from "../../../../../src/config/template-contexts/base.js" import { ConfigContext, schema, ScanContext } from "../../../../../src/config/template-contexts/base.js" import { expectError } from "../../../../helpers.js" import { joi } from "../../../../../src/config/common.js" @@ -35,7 +39,7 @@ describe("ConfigContext", () => { describe("resolve", () => { // just a shorthand to aid in testing - function resolveKey(c: ConfigContext, key: ContextKey, opts = {}) { + function resolveKey(c: ConfigContext, key: ContextKey, opts: ContextResolveOpts = {}) { return c.resolve({ key, nodePath: [], opts }) } @@ -46,7 +50,7 @@ describe("ConfigContext", () => { it("should return undefined for missing key", async () => { const c = new TestContext({}) - const { resolved, message } = resolveKey(c, ["basic"]) + const { result: resolved, message } = resolveKey(c, ["basic"]) expect(resolved).to.be.undefined expect(stripAnsi(message!)).to.include("Could not find key basic.") }) @@ -90,7 +94,7 @@ describe("ConfigContext", () => { const c = new TestContext({ nested: new TestContext({ key: "value" }), }) - const { resolved, message } = resolveKey(c, ["basic", "bla"]) + const { result: resolved, message } = resolveKey(c, ["basic", "bla"]) expect(resolved).to.be.undefined expect(stripAnsi(message!)).to.equal("Could not find key basic. Available keys: nested.") }) diff --git a/core/test/unit/src/config/template-contexts/workflow.ts b/core/test/unit/src/config/template-contexts/workflow.ts index 464e97aaff..53c666477b 100644 --- a/core/test/unit/src/config/template-contexts/workflow.ts +++ b/core/test/unit/src/config/template-contexts/workflow.ts @@ -112,7 +112,7 @@ describe("WorkflowStepConfigContext", () => { }, stepName: "step-2", }) - expect(c.resolve({ key: ["steps", "step-1", "outputs", "some"], nodePath: [], opts: {} }).resolved).to.equal( + expect(c.resolve({ key: ["steps", "step-1", "outputs", "some"], nodePath: [], opts: {} }).result).to.equal( "value" ) }) @@ -131,7 +131,7 @@ describe("WorkflowStepConfigContext", () => { }, stepName: "step-2", }) - expect(c.resolve({ key: ["steps", "step-1", "log"], nodePath: [], opts: {} }).resolved).to.equal("bla") + expect(c.resolve({ key: ["steps", "step-1", "log"], nodePath: [], opts: {} }).result).to.equal("bla") }) it("should throw error when attempting to reference a following step", () => { diff --git a/core/test/unit/src/garden.ts b/core/test/unit/src/garden.ts index a8fa2d4817..cec07290bb 100644 --- a/core/test/unit/src/garden.ts +++ b/core/test/unit/src/garden.ts @@ -3047,11 +3047,6 @@ describe("Garden", () => { }) }) - it.skip("should throw an error if references to missing secrets are present in a module config", async () => { - const garden = await makeTestGarden(getDataDir("missing-secrets", "module")) - await expectError(() => garden.scanAndAddConfigs(), { contains: "Module module-a: missing" }) - }) - it("should throw when apiVersion v0 is set in a project with action configs", async () => { const garden = await makeTestGarden(getDataDir("test-projects", "config-action-kind-v0")) diff --git a/core/test/unit/src/tasks/resolve-provider.ts b/core/test/unit/src/tasks/resolve-provider.ts index ad010e965c..97bca0bfb4 100644 --- a/core/test/unit/src/tasks/resolve-provider.ts +++ b/core/test/unit/src/tasks/resolve-provider.ts @@ -45,7 +45,7 @@ describe("ResolveProviderTask", () => { }) const plugin = await garden.getPlugin("test-plugin") - const config = garden.getRawProviderConfigs({ names: ["test-plugin"] })[0] + const config = garden.getConfiguredProviders({ names: ["test-plugin"] })[0] task = new ResolveProviderTask({ garden, diff --git a/core/test/unit/src/template-string.ts b/core/test/unit/src/template-string.ts index e9228ae8f1..4f6424dee6 100644 --- a/core/test/unit/src/template-string.ts +++ b/core/test/unit/src/template-string.ts @@ -11,15 +11,25 @@ import { resolveTemplateString, resolveTemplateStrings, collectTemplateReferences, - throwOnMissingSecretKeys, getActionTemplateReferences, + parseTemplateString, + parseTemplateCollection, } from "../../../src/template-string/template-string.js" -import { ConfigContext } from "../../../src/config/template-contexts/base.js" +import { ConfigContext, GenericContext } from "../../../src/config/template-contexts/base.js" import type { TestGarden } from "../../helpers.js" import { expectError, getDataDir, makeTestGarden } from "../../helpers.js" import { dedent } from "../../../src/util/string.js" import stripAnsi from "strip-ansi" import { TemplateStringError } from "../../../src/exceptions.js" +import { TemplateLeaf } from "../../../src/template-string/inputs.js" +import { + ForEachLazyValue, + OverrideKeyPathLazily, + TemplateStringLazyValue, + deepEvaluate, + deepEvaluateAndUnwrap, + evaluate, +} from "../../../src/template-string/lazy.js" class TestContext extends ConfigContext { constructor(context) { @@ -30,12 +40,18 @@ class TestContext extends ConfigContext { describe("resolveTemplateString", () => { it("should return a non-templated string unchanged", () => { - const res = resolveTemplateString({ string: "somestring", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "somestring", + context: new TestContext({}), + }) expect(res).to.equal("somestring") }) it("should resolve a key with a dash in it", () => { - const res = resolveTemplateString({ string: "${some-key}", context: new TestContext({ "some-key": "value" }) }) + const res = resolveTemplateString({ + string: "${some-key}", + context: new TestContext({ "some-key": "value" }), + }) expect(res).to.equal("value") }) @@ -48,31 +64,34 @@ describe("resolveTemplateString", () => { }) it("should correctly resolve if ? suffix is present but value exists", () => { - const res = resolveTemplateString({ string: "${foo}?", context: new TestContext({ foo: "bar" }) }) + const res = resolveTemplateString({ + string: "${foo}?", + context: new TestContext({ foo: "bar" }), + }) expect(res).to.equal("bar") }) it("should allow undefined values if ? suffix is present", () => { - const res = resolveTemplateString({ string: "${foo}?", context: new TestContext({}) }) - expect(res).to.equal(undefined) - }) - - it("should pass optional string through if allowPartial=true", () => { const res = resolveTemplateString({ string: "${foo}?", context: new TestContext({}), - contextOpts: { allowPartial: true }, }) - expect(res).to.equal("${foo}?") + expect(res).to.equal(undefined) }) it("should support a string literal in a template string as a means to escape it", () => { - const res = resolveTemplateString({ string: "${'$'}{bar}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${'$'}{bar}", + context: new TestContext({}), + }) expect(res).to.equal("${bar}") }) it("should pass through a template string with a double $$ prefix", () => { - const res = resolveTemplateString({ string: "$${bar}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "$${bar}", + context: new TestContext({}), + }) expect(res).to.equal("$${bar}") }) @@ -95,37 +114,58 @@ describe("resolveTemplateString", () => { }) it("should interpolate a format string with a prefix", () => { - const res = resolveTemplateString({ string: "prefix-${some}", context: new TestContext({ some: "value" }) }) + const res = resolveTemplateString({ + string: "prefix-${some}", + context: new TestContext({ some: "value" }), + }) expect(res).to.equal("prefix-value") }) it("should interpolate a format string with a suffix", () => { - const res = resolveTemplateString({ string: "${some}-suffix", context: new TestContext({ some: "value" }) }) + const res = resolveTemplateString({ + string: "${some}-suffix", + context: new TestContext({ some: "value" }), + }) expect(res).to.equal("value-suffix") }) it("should interpolate a format string with a prefix and a suffix", () => { - const res = resolveTemplateString({ string: "prefix-${some}-suffix", context: new TestContext({ some: "value" }) }) + const res = resolveTemplateString({ + string: "prefix-${some}-suffix", + context: new TestContext({ some: "value" }), + }) expect(res).to.equal("prefix-value-suffix") }) it("should interpolate an optional format string with a prefix and a suffix", () => { - const res = resolveTemplateString({ string: "prefix-${some}?-suffix", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "prefix-${some}?-suffix", + context: new TestContext({}), + }) expect(res).to.equal("prefix--suffix") }) it("should interpolate a format string with a prefix with whitespace", () => { - const res = resolveTemplateString({ string: "prefix ${some}", context: new TestContext({ some: "value" }) }) + const res = resolveTemplateString({ + string: "prefix ${some}", + context: new TestContext({ some: "value" }), + }) expect(res).to.equal("prefix value") }) it("should interpolate a format string with a suffix with whitespace", () => { - const res = resolveTemplateString({ string: "${some} suffix", context: new TestContext({ some: "value" }) }) + const res = resolveTemplateString({ + string: "${some} suffix", + context: new TestContext({ some: "value" }), + }) expect(res).to.equal("value suffix") }) it("should correctly interpolate a format string with surrounding whitespace", () => { - const res = resolveTemplateString({ string: "prefix ${some} suffix", context: new TestContext({ some: "value" }) }) + const res = resolveTemplateString({ + string: "prefix ${some} suffix", + context: new TestContext({ some: "value" }), + }) expect(res).to.equal("prefix value suffix") }) @@ -146,7 +186,10 @@ describe("resolveTemplateString", () => { }) it("should handle consecutive format strings", () => { - const res = resolveTemplateString({ string: "${a}${b}", context: new TestContext({ a: "value", b: "other" }) }) + const res = resolveTemplateString({ + string: "${a}${b}", + context: new TestContext({ a: "value", b: "other" }), + }) expect(res).to.equal("valueother") }) @@ -163,13 +206,17 @@ describe("resolveTemplateString", () => { string: "${some} very very very very very long long long long long template string", context: new TestContext({}), }), - { contains: "Invalid template string (${some} very very very very very l…): Could not find key some." } + { contains: "Invalid template string (${some} very very very very very...): Could not find key some." } ) }) it("should replace line breaks in template strings in error messages", () => { void expectError( - () => resolveTemplateString({ string: "${some}\nmulti\nline\nstring", context: new TestContext({}) }), + () => + resolveTemplateString({ + string: "${some}\nmulti\nline\nstring", + context: new TestContext({}), + }), { contains: "Invalid template string (${some}\\nmulti\\nline\\nstring): Could not find key some.", } @@ -177,14 +224,24 @@ describe("resolveTemplateString", () => { }) it("should throw when a nested key is not found", () => { - void expectError(() => resolveTemplateString({ string: "${some.other}", context: new TestContext({ some: {} }) }), { - contains: "Invalid template string (${some.other}): Could not find key other under some.", - }) + void expectError( + () => + resolveTemplateString({ + string: "${some.other}", + context: new TestContext({ some: {} }), + }), + { + contains: "Invalid template string (${some.other}): Could not find key other under some.", + } + ) }) it("should throw with an incomplete template string", () => { try { - resolveTemplateString({ string: "${some", context: new TestContext({ some: {} }) }) + resolveTemplateString({ + string: "${some", + context: new TestContext({ some: {} }), + }) } catch (err) { if (!(err instanceof TemplateStringError)) { expect.fail("Expected TemplateStringError") @@ -199,58 +256,107 @@ describe("resolveTemplateString", () => { }) it("should throw on nested format strings", () => { - void expectError(() => resolveTemplateString({ string: "${resol${part}ed}", context: new TestContext({}) }), { - contains: "Invalid template string (${resol${part}ed}): Unable to parse as valid template string.", - }) + void expectError( + () => + resolveTemplateString({ + string: "${resol${part}ed}", + context: new TestContext({}), + }), + { + contains: "Invalid template string (${resol${part}ed}): Unable to parse as valid template string.", + } + ) }) it("should handle a single-quoted string", () => { - const res = resolveTemplateString({ string: "${'foo'}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${'foo'}", + context: new TestContext({}), + }) expect(res).to.equal("foo") }) it("should handle a numeric literal and return it directly", () => { - const res = resolveTemplateString({ string: "${123}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${123}", + context: new TestContext({}), + }) expect(res).to.equal(123) }) it("should handle a boolean true literal and return it directly", () => { - const res = resolveTemplateString({ string: "${true}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${true}", + context: new TestContext({}), + }) expect(res).to.equal(true) }) it("should handle a boolean false literal and return it directly", () => { - const res = resolveTemplateString({ string: "${false}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${false}", + context: new TestContext({}), + }) expect(res).to.equal(false) }) it("should handle a null literal and return it directly", () => { - const res = resolveTemplateString({ string: "${null}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${null}", + context: new TestContext({}), + }) expect(res).to.equal(null) }) - it("should handle a numeric literal in a logical OR and return it directly", () => { - const res = resolveTemplateString({ string: "${a || 123}", context: new TestContext({}) }) - expect(res).to.equal(123) + it("should handle a numeric literal in a logical OR and return it directly and still track a as input", () => { + const value = parseTemplateString({ + string: "${a || 123}", + }) + + expect(value).to.be.instanceOf(TemplateStringLazyValue) + + const res = evaluate({ value, context: new TestContext({}), opts: {} }) + + expect(res).to.deep.equal( + new TemplateLeaf({ + expr: "${a || 123}", + value: 123, + inputs: { + a: new TemplateLeaf({ expr: undefined, value: undefined, inputs: {} }), + }, + }) + ) }) it("should handle a boolean true literal in a logical OR and return it directly", () => { - const res = resolveTemplateString({ string: "${a || true}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${a || true}", + context: new TestContext({}), + }) expect(res).to.equal(true) }) it("should handle a boolean false literal in a logical OR and return it directly", () => { - const res = resolveTemplateString({ string: "${a || false}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${a || false}", + context: new TestContext({}), + }) expect(res).to.equal(false) }) it("should handle a null literal in a logical OR and return it directly", () => { - const res = resolveTemplateString({ string: "${a || null}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${a || null}", + context: new TestContext({}), + }) expect(res).to.equal(null) }) it("should handle a double-quoted string", () => { - const res = resolveTemplateString({ string: '${"foo"}', context: new TestContext({}) }) + const res = resolveTemplateString({ + string: '${"foo"}', + context: new TestContext({}), + }) expect(res).to.equal("foo") }) @@ -267,7 +373,10 @@ describe("resolveTemplateString", () => { }) it("should handle a logical OR between two identifiers", () => { - const res = resolveTemplateString({ string: "${a || b}", context: new TestContext({ a: undefined, b: "abc" }) }) + const res = resolveTemplateString({ + string: "${a || b}", + context: new TestContext({ a: undefined, b: "abc" }), + }) expect(res).to.equal("abc") }) @@ -294,17 +403,26 @@ describe("resolveTemplateString", () => { }) it("should handle a logical OR between two identifiers without spaces with first value undefined", () => { - const res = resolveTemplateString({ string: "${a||b}", context: new TestContext({ a: undefined, b: "abc" }) }) + const res = resolveTemplateString({ + string: "${a||b}", + context: new TestContext({ a: undefined, b: "abc" }), + }) expect(res).to.equal("abc") }) it("should handle a logical OR between two identifiers with first value undefined and string fallback", () => { - const res = resolveTemplateString({ string: '${a || "foo"}', context: new TestContext({ a: undefined }) }) + const res = resolveTemplateString({ + string: '${a || "foo"}', + context: new TestContext({ a: undefined }), + }) expect(res).to.equal("foo") }) it("should handle a logical OR with undefined nested value and string fallback", () => { - const res = resolveTemplateString({ string: "${a.b || 'foo'}", context: new TestContext({ a: {} }) }) + const res = resolveTemplateString({ + string: "${a.b || 'foo'}", + context: new TestContext({ a: {} }), + }) expect(res).to.equal("foo") }) @@ -317,35 +435,61 @@ describe("resolveTemplateString", () => { }) it("should handle a logical OR between two identifiers without spaces with first value set", () => { - const res = resolveTemplateString({ string: "${a||b}", context: new TestContext({ a: "abc", b: undefined }) }) + const res = resolveTemplateString({ + string: "${a||b}", + context: new TestContext({ a: "abc", b: undefined }), + }) expect(res).to.equal("abc") }) it("should throw if neither key in logical OR is valid", () => { - void expectError(() => resolveTemplateString({ string: "${a || b}", context: new TestContext({}) }), { - contains: "Invalid template string (${a || b}): Could not find key b.", - }) + void expectError( + () => + resolveTemplateString({ + string: "${a || b}", + context: new TestContext({}), + }), + { + contains: "Invalid template string (${a || b}): Could not find key b.", + } + ) }) it("should throw on invalid logical OR string", () => { - void expectError(() => resolveTemplateString({ string: "${a || 'b}", context: new TestContext({}) }), { - contains: "Invalid template string (${a || 'b}): Unable to parse as valid template string.", - }) + void expectError( + () => + resolveTemplateString({ + string: "${a || 'b}", + context: new TestContext({}), + }), + { + contains: "Invalid template string (${a || 'b}): Unable to parse as valid template string.", + } + ) }) it("should handle a logical OR between a string and a string", () => { - const res = resolveTemplateString({ string: "${'a' || 'b'}", context: new TestContext({ a: undefined }) }) + const res = resolveTemplateString({ + string: "${'a' || 'b'}", + context: new TestContext({ a: undefined }), + }) expect(res).to.equal("a") }) it("should handle a logical OR between an empty string and a string", () => { - const res = resolveTemplateString({ string: "${a || 'b'}", context: new TestContext({ a: "" }) }) + const res = resolveTemplateString({ + string: "${a || 'b'}", + context: new TestContext({ a: "" }), + }) expect(res).to.equal("b") }) context("logical AND (&& operator)", () => { it("true literal and true variable reference", () => { - const res = resolveTemplateString({ string: "${true && a}", context: new TestContext({ a: true }) }) + const res = resolveTemplateString({ + string: "${true && a}", + context: new TestContext({ a: true }), + }) expect(res).to.equal(true) }) @@ -359,31 +503,46 @@ describe("resolveTemplateString", () => { it("first part is false but the second part is not resolvable", () => { // i.e. the 2nd clause should not need to be evaluated - const res = resolveTemplateString({ string: "${false && a}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${false && a}", + context: new TestContext({}), + }) expect(res).to.equal(false) }) it("an empty string as the first clause", () => { - const res = resolveTemplateString({ string: "${'' && true}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${'' && true}", + context: new TestContext({}), + }) expect(res).to.equal("") }) it("an empty string as the second clause", () => { - const res = resolveTemplateString({ string: "${true && ''}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${true && ''}", + context: new TestContext({}), + }) expect(res).to.equal("") }) it("a missing reference as the first clause", () => { - const res = resolveTemplateString({ string: "${var.foo && 'a'}", context: new TestContext({ var: {} }) }) + const res = resolveTemplateString({ + string: "${var.foo && 'a'}", + context: new TestContext({ var: {} }), + }) expect(res).to.equal(false) }) it("a missing reference as the second clause", () => { - const res = resolveTemplateString({ string: "${'a' && var.foo}", context: new TestContext({ var: {} }) }) + const res = resolveTemplateString({ + string: "${'a' && foo}", + context: new TestContext({ var: {} }), + }) expect(res).to.equal(false) }) - context("partial resolution", () => { + context.skip("partial resolution", () => { it("a missing reference as the first clause returns the original template", () => { const res = resolveTemplateString({ string: "${var.foo && 'a'}", @@ -405,154 +564,249 @@ describe("resolveTemplateString", () => { }) it("should handle a positive equality comparison between equal resolved values", () => { - const res = resolveTemplateString({ string: "${a == b}", context: new TestContext({ a: "a", b: "a" }) }) + const res = resolveTemplateString({ + string: "${a == b}", + context: new TestContext({ a: "a", b: "a" }), + }) expect(res).to.equal(true) }) it("should handle a positive equality comparison between equal string literals", () => { - const res = resolveTemplateString({ string: "${'a' == 'a'}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${'a' == 'a'}", + context: new TestContext({}), + }) expect(res).to.equal(true) }) it("should handle a positive equality comparison between equal numeric literals", () => { - const res = resolveTemplateString({ string: "${123 == 123}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${123 == 123}", + context: new TestContext({}), + }) expect(res).to.equal(true) }) it("should handle a positive equality comparison between equal boolean literals", () => { - const res = resolveTemplateString({ string: "${true == true}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${true == true}", + context: new TestContext({}), + }) expect(res).to.equal(true) }) it("should handle a positive equality comparison between different resolved values", () => { - const res = resolveTemplateString({ string: "${a == b}", context: new TestContext({ a: "a", b: "b" }) }) + const res = resolveTemplateString({ + string: "${a == b}", + context: new TestContext({ a: "a", b: "b" }), + }) expect(res).to.equal(false) }) it("should handle a positive equality comparison between different string literals", () => { - const res = resolveTemplateString({ string: "${'a' == 'b'}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${'a' == 'b'}", + context: new TestContext({}), + }) expect(res).to.equal(false) }) it("should handle a positive equality comparison between different numeric literals", () => { - const res = resolveTemplateString({ string: "${123 == 456}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${123 == 456}", + context: new TestContext({}), + }) expect(res).to.equal(false) }) it("should handle a positive equality comparison between different boolean literals", () => { - const res = resolveTemplateString({ string: "${true == false}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${true == false}", + context: new TestContext({}), + }) expect(res).to.equal(false) }) it("should handle a negative equality comparison between equal resolved values", () => { - const res = resolveTemplateString({ string: "${a != b}", context: new TestContext({ a: "a", b: "a" }) }) + const res = resolveTemplateString({ + string: "${a != b}", + context: new TestContext({ a: "a", b: "a" }), + }) expect(res).to.equal(false) }) it("should handle a negative equality comparison between equal string literals", () => { - const res = resolveTemplateString({ string: "${'a' != 'a'}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${'a' != 'a'}", + context: new TestContext({}), + }) expect(res).to.equal(false) }) it("should handle a negative equality comparison between equal numeric literals", () => { - const res = resolveTemplateString({ string: "${123 != 123}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${123 != 123}", + context: new TestContext({}), + }) expect(res).to.equal(false) }) it("should handle a negative equality comparison between equal boolean literals", () => { - const res = resolveTemplateString({ string: "${false != false}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${false != false}", + context: new TestContext({}), + }) expect(res).to.equal(false) }) it("should handle a negative equality comparison between different resolved values", () => { - const res = resolveTemplateString({ string: "${a != b}", context: new TestContext({ a: "a", b: "b" }) }) + const res = resolveTemplateString({ + string: "${a != b}", + context: new TestContext({ a: "a", b: "b" }), + }) expect(res).to.equal(true) }) it("should handle a negative equality comparison between different string literals", () => { - const res = resolveTemplateString({ string: "${'a' != 'b'}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${'a' != 'b'}", + context: new TestContext({}), + }) expect(res).to.equal(true) }) it("should handle a negative equality comparison between different numeric literals", () => { - const res = resolveTemplateString({ string: "${123 != 456}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${123 != 456}", + context: new TestContext({}), + }) expect(res).to.equal(true) }) it("should handle a negative equality comparison between different boolean literals", () => { - const res = resolveTemplateString({ string: "${true != false}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${true != false}", + context: new TestContext({}), + }) expect(res).to.equal(true) }) it("should handle a positive equality comparison between different value types", () => { - const res = resolveTemplateString({ string: "${true == 'foo'}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${true == 'foo'}", + context: new TestContext({}), + }) expect(res).to.equal(false) }) it("should handle a negative equality comparison between different value types", () => { - const res = resolveTemplateString({ string: "${123 != false}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${123 != false}", + context: new TestContext({}), + }) expect(res).to.equal(true) }) it("should handle negations on booleans", () => { - const res = resolveTemplateString({ string: "${!true}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${!true}", + context: new TestContext({}), + }) expect(res).to.equal(false) }) it("should handle negations on nulls", () => { - const res = resolveTemplateString({ string: "${!null}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${!null}", + context: new TestContext({}), + }) expect(res).to.equal(true) }) it("should handle negations on empty strings", () => { - const res = resolveTemplateString({ string: "${!''}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${!''}", + context: new TestContext({}), + }) expect(res).to.equal(true) }) it("should handle negations on resolved keys", () => { - const res = resolveTemplateString({ string: "${!a}", context: new TestContext({ a: false }) }) + const res = resolveTemplateString({ + string: "${!a}", + context: new TestContext({ a: false }), + }) expect(res).to.equal(true) }) it("should handle the typeof operator for resolved booleans", () => { - const res = resolveTemplateString({ string: "${typeof a}", context: new TestContext({ a: false }) }) + const res = resolveTemplateString({ + string: "${typeof a}", + context: new TestContext({ a: false }), + }) expect(res).to.equal("boolean") }) it("should handle the typeof operator for resolved numbers", () => { - const res = resolveTemplateString({ string: "${typeof foo}", context: new TestContext({ foo: 1234 }) }) + const res = resolveTemplateString({ + string: "${typeof foo}", + context: new TestContext({ foo: 1234 }), + }) expect(res).to.equal("number") }) it("should handle the typeof operator for strings", () => { - const res = resolveTemplateString({ string: "${typeof 'foo'}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${typeof 'foo'}", + context: new TestContext({}), + }) expect(res).to.equal("string") }) it("should throw when using comparison operators on missing keys", () => { - void expectError(() => resolveTemplateString({ string: "${a >= b}", context: new TestContext({ a: 123 }) }), { - contains: "Invalid template string (${a >= b}): Could not find key b. Available keys: a.", - }) + void expectError( + () => + resolveTemplateString({ + string: "${a >= b}", + context: new TestContext({ a: 123 }), + }), + { + contains: "Invalid template string (${a >= b}): Could not find key b. Available keys: a.", + } + ) }) it("should concatenate two arrays", () => { - const res = resolveTemplateString({ string: "${a + b}", context: new TestContext({ a: [1], b: [2, 3] }) }) + const res = resolveTemplateString({ + string: "${a + b}", + context: new TestContext({ a: [1], b: [2, 3] }), + }) expect(res).to.eql([1, 2, 3]) }) it("should concatenate two strings", () => { - const res = resolveTemplateString({ string: "${a + b}", context: new TestContext({ a: "foo", b: "bar" }) }) + const res = resolveTemplateString({ + string: "${a + b}", + context: new TestContext({ a: "foo", b: "bar" }), + }) expect(res).to.eql("foobar") }) it("should add two numbers together", () => { - const res = resolveTemplateString({ string: "${1 + a}", context: new TestContext({ a: 2 }) }) + const res = resolveTemplateString({ + string: "${1 + a}", + context: new TestContext({ a: 2 }), + }) expect(res).to.equal(3) }) it("should throw when using + on number and array", () => { void expectError( - () => resolveTemplateString({ string: "${a + b}", context: new TestContext({ a: 123, b: ["a"] }) }), + () => + resolveTemplateString({ + string: "${a + b}", + context: new TestContext({ a: 123, b: ["a"] }), + }), { contains: "Invalid template string (${a + b}): Both terms need to be either arrays or strings or numbers for + operator (got number and object).", @@ -561,12 +815,18 @@ describe("resolveTemplateString", () => { }) it("should correctly evaluate clauses in parentheses", () => { - const res = resolveTemplateString({ string: "${(1 + 2) * (3 + 4)}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${(1 + 2) * (3 + 4)}", + context: new TestContext({}), + }) expect(res).to.equal(21) }) it("should handle member lookup with bracket notation", () => { - const res = resolveTemplateString({ string: "${foo['bar']}", context: new TestContext({ foo: { bar: true } }) }) + const res = resolveTemplateString({ + string: "${foo['bar']}", + context: new TestContext({ foo: { bar: true } }), + }) expect(res).to.equal(true) }) @@ -587,7 +847,10 @@ describe("resolveTemplateString", () => { }) it("should handle numeric member lookup with bracket notation", () => { - const res = resolveTemplateString({ string: "${foo[1]}", context: new TestContext({ foo: [false, true] }) }) + const res = resolveTemplateString({ + string: "${foo[1]}", + context: new TestContext({ foo: [false, true] }), + }) expect(res).to.equal(true) }) @@ -642,17 +905,25 @@ describe("resolveTemplateString", () => { it("should throw if bracket expression resolves to a non-primitive", () => { void expectError( - () => resolveTemplateString({ string: "${foo[bar]}", context: new TestContext({ foo: {}, bar: {} }) }), + () => + resolveTemplateString({ + string: "${foo[bar]}", + context: new TestContext({ foo: {}, bar: {} }), + }), { contains: - "Invalid template string (${foo[bar]}): Expression in bracket must resolve to a primitive (got object).", + "Invalid template string (${foo[bar]}): Expression in bracket must resolve to a string or number (got object).", } ) }) it("should throw if attempting to index a primitive with brackets", () => { void expectError( - () => resolveTemplateString({ string: "${foo[bar]}", context: new TestContext({ foo: 123, bar: "baz" }) }), + () => + resolveTemplateString({ + string: "${foo[bar]}", + context: new TestContext({ foo: 123, bar: "baz" }), + }), { contains: 'Invalid template string (${foo[bar]}): Attempted to look up key "baz" on a number.', } @@ -661,7 +932,11 @@ describe("resolveTemplateString", () => { it("should throw when using >= on non-numeric terms", () => { void expectError( - () => resolveTemplateString({ string: "${a >= b}", context: new TestContext({ a: 123, b: "foo" }) }), + () => + resolveTemplateString({ + string: "${a >= b}", + context: new TestContext({ a: 123, b: "foo" }), + }), { contains: "Invalid template string (${a >= b}): Both terms need to be numbers for >= operator (got number and string).", @@ -670,12 +945,18 @@ describe("resolveTemplateString", () => { }) it("should handle a positive ternary expression", () => { - const res = resolveTemplateString({ string: "${foo ? true : false}", context: new TestContext({ foo: true }) }) + const res = resolveTemplateString({ + string: "${foo ? true : false}", + context: new TestContext({ foo: true }), + }) expect(res).to.equal(true) }) it("should handle a negative ternary expression", () => { - const res = resolveTemplateString({ string: "${foo ? true : false}", context: new TestContext({ foo: false }) }) + const res = resolveTemplateString({ + string: "${foo ? true : false}", + context: new TestContext({ foo: false }), + }) expect(res).to.equal(false) }) @@ -712,12 +993,18 @@ describe("resolveTemplateString", () => { }) it("should handle an expression in parentheses", () => { - const res = resolveTemplateString({ string: "${foo || (a > 5)}", context: new TestContext({ foo: false, a: 10 }) }) + const res = resolveTemplateString({ + string: "${foo || (a > 5)}", + context: new TestContext({ foo: false, a: 10 }), + }) expect(res).to.equal(true) }) it("should handle numeric indices on arrays", () => { - const res = resolveTemplateString({ string: "${foo.1}", context: new TestContext({ foo: [false, true] }) }) + const res = resolveTemplateString({ + string: "${foo.1}", + context: new TestContext({ foo: [false, true] }), + }) expect(res).to.equal(true) }) @@ -773,7 +1060,7 @@ describe("resolveTemplateString", () => { }) }) - context("allowPartial=true", () => { + context.skip("allowPartial=true", () => { it("passes through template strings with missing key", () => { const res = resolveTemplateString({ string: "${a}", @@ -804,58 +1091,88 @@ describe("resolveTemplateString", () => { context("when the template string is the full input string", () => { it("should return a resolved number directly", () => { - const res = resolveTemplateString({ string: "${a}", context: new TestContext({ a: 100 }) }) + const res = resolveTemplateString({ + string: "${a}", + context: new TestContext({ a: 100 }), + }) expect(res).to.equal(100) }) it("should return a resolved boolean true directly", () => { - const res = resolveTemplateString({ string: "${a}", context: new TestContext({ a: true }) }) + const res = resolveTemplateString({ + string: "${a}", + context: new TestContext({ a: true }), + }) expect(res).to.equal(true) }) it("should return a resolved boolean false directly", () => { - const res = resolveTemplateString({ string: "${a}", context: new TestContext({ a: false }) }) + const res = resolveTemplateString({ + string: "${a}", + context: new TestContext({ a: false }), + }) expect(res).to.equal(false) }) it("should return a resolved null directly", () => { - const res = resolveTemplateString({ string: "${a}", context: new TestContext({ a: null }) }) + const res = resolveTemplateString({ + string: "${a}", + context: new TestContext({ a: null }), + }) expect(res).to.equal(null) }) it("should return a resolved object directly", () => { - const res = resolveTemplateString({ string: "${a}", context: new TestContext({ a: { b: 123 } }) }) + const res = resolveTemplateString({ + string: "${a}", + context: new TestContext({ a: { b: 123 } }), + }) expect(res).to.eql({ b: 123 }) }) it("should return a resolved array directly", () => { - const res = resolveTemplateString({ string: "${a}", context: new TestContext({ a: [123] }) }) + const res = resolveTemplateString({ + string: "${a}", + context: new TestContext({ a: [123] }), + }) expect(res).to.eql([123]) }) }) context("when the template string is a part of a string", () => { it("should format a resolved number into the string", () => { - const res = resolveTemplateString({ string: "foo-${a}", context: new TestContext({ a: 100 }) }) + const res = resolveTemplateString({ + string: "foo-${a}", + context: new TestContext({ a: 100 }), + }) expect(res).to.equal("foo-100") }) it("should format a resolved boolean true into the string", () => { - const res = resolveTemplateString({ string: "foo-${a}", context: new TestContext({ a: true }) }) + const res = resolveTemplateString({ + string: "foo-${a}", + context: new TestContext({ a: true }), + }) expect(res).to.equal("foo-true") }) it("should format a resolved boolean false into the string", () => { - const res = resolveTemplateString({ string: "foo-${a}", context: new TestContext({ a: false }) }) + const res = resolveTemplateString({ + string: "foo-${a}", + context: new TestContext({ a: false }), + }) expect(res).to.equal("foo-false") }) it("should format a resolved null into the string", () => { - const res = resolveTemplateString({ string: "foo-${a}", context: new TestContext({ a: null }) }) + const res = resolveTemplateString({ + string: "foo-${a}", + context: new TestContext({ a: null }), + }) expect(res).to.equal("foo-null") }) - context("allowPartial=true", () => { + context.skip("allowPartial=true", () => { it("passes through template strings with missing key", () => { const res = resolveTemplateString({ string: "${a}-${b}", @@ -898,27 +1215,42 @@ describe("resolveTemplateString", () => { }) it("positive string literal contains string literal", () => { - const res = resolveTemplateString({ string: "${'foobar' contains 'foo'}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${'foobar' contains 'foo'}", + context: new TestContext({}), + }) expect(res).to.equal(true) }) it("string literal contains string literal (negative)", () => { - const res = resolveTemplateString({ string: "${'blorg' contains 'blarg'}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${'blorg' contains 'blarg'}", + context: new TestContext({}), + }) expect(res).to.equal(false) }) it("string literal contains string reference", () => { - const res = resolveTemplateString({ string: "${a contains 'foo'}", context: new TestContext({ a: "foobar" }) }) + const res = resolveTemplateString({ + string: "${a contains 'foo'}", + context: new TestContext({ a: "foobar" }), + }) expect(res).to.equal(true) }) it("string reference contains string literal (negative)", () => { - const res = resolveTemplateString({ string: "${a contains 'blarg'}", context: new TestContext({ a: "foobar" }) }) + const res = resolveTemplateString({ + string: "${a contains 'blarg'}", + context: new TestContext({ a: "foobar" }), + }) expect(res).to.equal(false) }) it("string contains number", () => { - const res = resolveTemplateString({ string: "${a contains 0}", context: new TestContext({ a: "hmm-0" }) }) + const res = resolveTemplateString({ + string: "${a contains 0}", + context: new TestContext({ a: "hmm-0" }), + }) expect(res).to.equal(true) }) @@ -955,7 +1287,10 @@ describe("resolveTemplateString", () => { }) it("object contains number literal", () => { - const res = resolveTemplateString({ string: "${a contains 123}", context: new TestContext({ a: { 123: 456 } }) }) + const res = resolveTemplateString({ + string: "${a contains 123}", + context: new TestContext({ a: { 123: 456 } }), + }) expect(res).to.equal(true) }) @@ -976,17 +1311,26 @@ describe("resolveTemplateString", () => { }) it("array contains string literal", () => { - const res = resolveTemplateString({ string: "${a contains 'foo'}", context: new TestContext({ a: ["foo"] }) }) + const res = resolveTemplateString({ + string: "${a contains 'foo'}", + context: new TestContext({ a: ["foo"] }), + }) expect(res).to.equal(true) }) it("array contains number", () => { - const res = resolveTemplateString({ string: "${a contains 1}", context: new TestContext({ a: [0, 1] }) }) + const res = resolveTemplateString({ + string: "${a contains 1}", + context: new TestContext({ a: [0, 1] }), + }) expect(res).to.equal(true) }) it("array contains numeric index (negative)", () => { - const res = resolveTemplateString({ string: "${a contains 1}", context: new TestContext({ a: [0] }) }) + const res = resolveTemplateString({ + string: "${a contains 1}", + context: new TestContext({ a: [0] }), + }) expect(res).to.equal(false) }) }) @@ -1111,7 +1455,10 @@ describe("resolveTemplateString", () => { it("throws if an if block has an optional suffix", () => { void expectError( () => - resolveTemplateString({ string: "prefix ${if a}?content ${endif}", context: new TestContext({ a: true }) }), + resolveTemplateString({ + string: "prefix ${if a}?content ${endif}", + context: new TestContext({ a: true }), + }), { contains: "Invalid template string (prefix ${if a}?content ${endif}): Cannot specify optional suffix in if-block.", @@ -1121,7 +1468,11 @@ describe("resolveTemplateString", () => { it("throws if an if block doesn't have a matching endif", () => { void expectError( - () => resolveTemplateString({ string: "prefix ${if a}content", context: new TestContext({ a: true }) }), + () => + resolveTemplateString({ + string: "prefix ${if a}content", + context: new TestContext({ a: true }), + }), { contains: "Invalid template string (prefix ${if a}content): Missing ${endif} after ${if ...} block.", } @@ -1130,7 +1481,11 @@ describe("resolveTemplateString", () => { it("throws if an endif block doesn't have a matching if", () => { void expectError( - () => resolveTemplateString({ string: "prefix content ${endif}", context: new TestContext({ a: true }) }), + () => + resolveTemplateString({ + string: "prefix content ${endif}", + context: new TestContext({ a: true }), + }), { contains: "Invalid template string (prefix content ${endif}): Found ${endif} block without a preceding ${if...} block.", @@ -1141,40 +1496,66 @@ describe("resolveTemplateString", () => { context("helper functions", () => { it("resolves a helper function with a string literal", () => { - const res = resolveTemplateString({ string: "${base64Encode('foo')}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${base64Encode('foo')}", + context: new TestContext({}), + }) expect(res).to.equal("Zm9v") }) it("resolves a template string in a helper argument", () => { - const res = resolveTemplateString({ string: "${base64Encode('${a}')}", context: new TestContext({ a: "foo" }) }) + const res = resolveTemplateString({ + string: "${base64Encode('${a}')}", + context: new TestContext({ a: "foo" }), + }) expect(res).to.equal("Zm9v") }) it("resolves a helper function with multiple arguments", () => { - const res = resolveTemplateString({ string: "${split('a,b,c', ',')}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${split('a,b,c', ',')}", + context: new TestContext({}), + }) expect(res).to.eql(["a", "b", "c"]) }) it("resolves a helper function with a template key reference", () => { - const res = resolveTemplateString({ string: "${base64Encode(a)}", context: new TestContext({ a: "foo" }) }) + const res = resolveTemplateString({ + string: "${base64Encode(a)}", + context: new TestContext({ a: "foo" }), + }) expect(res).to.equal("Zm9v") }) it("generates a correct hash with a string literal from the sha256 helper function", () => { - const res = resolveTemplateString({ string: "${sha256('This Is A Test String')}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${sha256('This Is A Test String')}", + context: new TestContext({}), + }) expect(res).to.equal("9a058284378d1cc6b4348aacb6ba847918376054b094bbe06eb5302defc52685") }) it("throws if an argument is missing", () => { - void expectError(() => resolveTemplateString({ string: "${base64Decode()}", context: new TestContext({}) }), { - contains: - "Invalid template string (${base64Decode()}): Missing argument 'string' (at index 0) for base64Decode helper function.", - }) + void expectError( + () => + resolveTemplateString({ + string: "${base64Decode()}", + context: new TestContext({}), + }), + { + contains: + "Invalid template string (${base64Decode()}): Missing argument 'string' (at index 0) for base64Decode helper function.", + } + ) }) it("throws if a wrong argument type is passed", () => { void expectError( - () => resolveTemplateString({ string: "${base64Decode(a)}", context: new TestContext({ a: 1234 }) }), + () => + resolveTemplateString({ + string: "${base64Decode(a)}", + context: new TestContext({ a: 1234 }), + }), { contains: "Invalid template string (${base64Decode(a)}): Error validating argument 'string' for base64Decode helper function:\nvalue must be a string", @@ -1183,20 +1564,34 @@ describe("resolveTemplateString", () => { }) it("throws if the function can't be found", () => { - void expectError(() => resolveTemplateString({ string: "${floop('blop')}", context: new TestContext({}) }), { - contains: - "Invalid template string (${floop('blop')}): Could not find helper function 'floop'. Available helper functions:", - }) + void expectError( + () => + resolveTemplateString({ + string: "${floop('blop')}", + context: new TestContext({}), + }), + { + contains: + "Invalid template string (${floop('blop')}): Could not find helper function 'floop'. Available helper functions:", + } + ) }) it("throws if the function fails", () => { - void expectError(() => resolveTemplateString({ string: "${jsonDecode('{]}')}", context: new TestContext({}) }), { - contains: - "Invalid template string (${jsonDecode('{]}')}): Error from helper function jsonDecode: SyntaxError: Expected property name or '}' in JSON at position 1 (line 1 column 2)", - }) + void expectError( + () => + resolveTemplateString({ + string: "${jsonDecode('{]}')}", + context: new TestContext({}), + }), + { + contains: + "Invalid template string (${jsonDecode('{]}')}): Error from helper function jsonDecode: SyntaxError: Expected property name or '}' in JSON at position 1 (line 1 column 2)", + } + ) }) - it("does not apply helper function on unresolved template string and returns string as-is, when allowPartial=true", () => { + it.skip("does not apply helper function on unresolved template string and returns string as-is, when allowPartial=true", () => { const res = resolveTemplateString({ string: "${base64Encode('${environment.namespace}')}", context: new TestContext({}), @@ -1209,7 +1604,10 @@ describe("resolveTemplateString", () => { context("concat", () => { it("allows empty strings", () => { - const res = resolveTemplateString({ string: "${concat('', '')}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${concat('', '')}", + context: new TestContext({}), + }) expect(res).to.equal("") }) @@ -1224,7 +1622,11 @@ describe("resolveTemplateString", () => { errorMessage: string }) { void expectError( - () => resolveTemplateString({ string: template, context: new TestContext(testContextVars) }), + () => + resolveTemplateString({ + string: template, + context: new TestContext(testContextVars), + }), { contains: `Invalid template string (\${concat(a, b)}): ${errorMessage}`, } @@ -1260,24 +1662,36 @@ describe("resolveTemplateString", () => { context("isEmpty", () => { context("allows nulls", () => { it("resolves null as 'true'", () => { - const res = resolveTemplateString({ string: "${isEmpty(null)}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${isEmpty(null)}", + context: new TestContext({}), + }) expect(res).to.be.true }) it("resolves references to null as 'true'", () => { - const res = resolveTemplateString({ string: "${isEmpty(a)}", context: new TestContext({ a: null }) }) + const res = resolveTemplateString({ + string: "${isEmpty(a)}", + context: new TestContext({ a: null }), + }) expect(res).to.be.true }) }) context("allows empty strings", () => { it("resolves an empty string as 'true'", () => { - const res = resolveTemplateString({ string: "${isEmpty('')}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${isEmpty('')}", + context: new TestContext({}), + }) expect(res).to.be.true }) it("resolves a reference to an empty string as 'true'", () => { - const res = resolveTemplateString({ string: "${isEmpty(a)}", context: new TestContext({ a: "" }) }) + const res = resolveTemplateString({ + string: "${isEmpty(a)}", + context: new TestContext({ a: "" }), + }) expect(res).to.be.true }) }) @@ -1302,7 +1716,11 @@ describe("resolveTemplateString", () => { it("throws on invalid string in the start index", () => { void expectError( - () => resolveTemplateString({ string: "${slice(foo, 'a', 3)}", context: new TestContext({ foo: "abcdef" }) }), + () => + resolveTemplateString({ + string: "${slice(foo, 'a', 3)}", + context: new TestContext({ foo: "abcdef" }), + }), { contains: `Invalid template string (\${slice(foo, 'a', 3)}): Error from helper function slice: Error: start index must be a number or a numeric string (got "a")`, } @@ -1311,7 +1729,11 @@ describe("resolveTemplateString", () => { it("throws on invalid string in the end index", () => { void expectError( - () => resolveTemplateString({ string: "${slice(foo, 0, 'b')}", context: new TestContext({ foo: "abcdef" }) }), + () => + resolveTemplateString({ + string: "${slice(foo, 0, 'b')}", + context: new TestContext({ foo: "abcdef" }), + }), { contains: `Invalid template string (\${slice(foo, 0, 'b')}): Error from helper function slice: Error: end index must be a number or a numeric string (got "b")`, } @@ -1322,7 +1744,10 @@ describe("resolveTemplateString", () => { context("array literals", () => { it("returns an empty array literal back", () => { - const res = resolveTemplateString({ string: "${[]}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${[]}", + context: new TestContext({}), + }) expect(res).to.eql([]) }) @@ -1335,12 +1760,18 @@ describe("resolveTemplateString", () => { }) it("resolves a key in an array literal", () => { - const res = resolveTemplateString({ string: "${[foo]}", context: new TestContext({ foo: "bar" }) }) + const res = resolveTemplateString({ + string: "${[foo]}", + context: new TestContext({ foo: "bar" }), + }) expect(res).to.eql(["bar"]) }) it("resolves a nested key in an array literal", () => { - const res = resolveTemplateString({ string: "${[foo.bar]}", context: new TestContext({ foo: { bar: "baz" } }) }) + const res = resolveTemplateString({ + string: "${[foo.bar]}", + context: new TestContext({ foo: { bar: "baz" } }), + }) expect(res).to.eql(["baz"]) }) @@ -1353,12 +1784,18 @@ describe("resolveTemplateString", () => { }) it("calls a helper with an array literal argument", () => { - const res = resolveTemplateString({ string: "${join(['foo', 'bar'], ',')}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${join(['foo', 'bar'], ',')}", + context: new TestContext({}), + }) expect(res).to.eql("foo,bar") }) it("allows empty string separator in join helper function", () => { - const res = resolveTemplateString({ string: "${join(['foo', 'bar'], '')}", context: new TestContext({}) }) + const res = resolveTemplateString({ + string: "${join(['foo', 'bar'], '')}", + context: new TestContext({}), + }) expect(res).to.eql("foobar") }) }) @@ -1378,7 +1815,7 @@ describe("resolveTemplateStrings", () => { something: "else", }) - const result = resolveTemplateStrings({ source: undefined, value: obj, context: templateContext }) + const result = resolveTemplateStrings({ value: obj, context: templateContext }) expect(result).to.eql({ some: "value", @@ -1398,7 +1835,7 @@ describe("resolveTemplateStrings", () => { key: "value", }) - const result = resolveTemplateStrings({ source: undefined, value: obj, context: templateContext }) + const result = resolveTemplateStrings({ value: obj, context: templateContext }) expect(result).to.eql({ some: "value", @@ -1414,7 +1851,7 @@ describe("resolveTemplateStrings", () => { } const templateContext = new TestContext({}) - const result = resolveTemplateStrings({ source: undefined, value: obj, context: templateContext }) + const result = resolveTemplateStrings({ value: obj, context: templateContext }) expect(result).to.eql({ a: "a", @@ -1431,7 +1868,7 @@ describe("resolveTemplateStrings", () => { } const templateContext = new TestContext({}) - const result = resolveTemplateStrings({ source: undefined, value: obj, context: templateContext }) + const result = resolveTemplateStrings({ value: obj, context: templateContext }) expect(result).to.eql({ a: "a", @@ -1448,7 +1885,7 @@ describe("resolveTemplateStrings", () => { } const templateContext = new TestContext({ obj: { a: "a", b: "b" } }) - const result = resolveTemplateStrings({ source: undefined, value: obj, context: templateContext }) + const result = resolveTemplateStrings({ value: obj, context: templateContext }) expect(result).to.eql({ a: "a", @@ -1468,7 +1905,7 @@ describe("resolveTemplateStrings", () => { } const templateContext = new TestContext({ obj: { b: "b" } }) - const result = resolveTemplateStrings({ source: undefined, value: obj, context: templateContext }) + const result = resolveTemplateStrings({ value: obj, context: templateContext }) expect(result).to.eql({ a: "a", @@ -1484,7 +1921,7 @@ describe("resolveTemplateStrings", () => { } const templateContext = new TestContext({ var: { obj: { a: "a", b: "b" } } }) - const result = resolveTemplateStrings({ value: obj, context: templateContext, source: undefined }) + const result = resolveTemplateStrings({ value: obj, context: templateContext }) expect(result).to.eql({ a: "a", @@ -1493,7 +1930,7 @@ describe("resolveTemplateStrings", () => { }) }) - it("should partially resolve $merge keys if a dependency cannot be resolved yet in partial mode", () => { + it.skip("should partially resolve $merge keys if a dependency cannot be resolved yet in partial mode", () => { const obj = { "key-value-array": { $forEach: "${inputs.merged-object || []}", @@ -1521,7 +1958,6 @@ describe("resolveTemplateStrings", () => { value: obj, context: templateContext, contextOpts: { allowPartial: true }, - source: undefined, }) expect(result).to.eql({ @@ -1546,12 +1982,17 @@ describe("resolveTemplateStrings", () => { }, } const templateContext = new TestContext({ - inputs: { - "merged-object": { - $merge: "${var.empty || var.input-object}", - INTERNAL_VAR_1: "INTERNAL_VAR_1", + inputs: parseTemplateCollection({ + value: { + "merged-object": { + $merge: "${var.empty || var.input-object}", + INTERNAL_VAR_1: "INTERNAL_VAR_1", + }, }, - }, + source: { + source: undefined, + }, + }), var: { "input-object": { EXTERNAL_VAR_1: "EXTERNAL_VAR_1", @@ -1559,7 +2000,7 @@ describe("resolveTemplateStrings", () => { }, }) - const result = resolveTemplateStrings({ value: obj, context: templateContext, source: undefined }) + const result = resolveTemplateStrings({ value: obj, context: templateContext }) expect(result).to.eql({ "key-value-array": [ @@ -1576,9 +2017,7 @@ describe("resolveTemplateStrings", () => { } const templateContext = new TestContext({ var: { obj: { a: "a", b: "b" } } }) - expect(() => resolveTemplateStrings({ value: obj, context: templateContext, source: undefined })).to.throw( - "Invalid template string" - ) + expect(() => resolveTemplateStrings({ value: obj, context: templateContext })).to.throw("Invalid template string") }) context("$concat", () => { @@ -1586,7 +2025,10 @@ describe("resolveTemplateStrings", () => { const obj = { foo: ["a", { $concat: ["b", "c"] }, "d"], } - const res = resolveTemplateStrings({ source: undefined, value: obj, context: new TestContext({}) }) + const res = resolveTemplateStrings({ + value: obj, + context: new TestContext({}), + }) expect(res).to.eql({ foo: ["a", "b", "c", "d"], }) @@ -1597,7 +2039,6 @@ describe("resolveTemplateStrings", () => { foo: ["a", { $concat: "${foo}" }, "d"], } const res = resolveTemplateStrings({ - source: undefined, value: obj, context: new TestContext({ foo: ["b", "c"] }), }) @@ -1611,7 +2052,6 @@ describe("resolveTemplateStrings", () => { foo: ["a", { $concat: { $forEach: ["B", "C"], $return: "${lower(item.value)}" } }, "d"], } const res = resolveTemplateStrings({ - source: undefined, value: obj, context: new TestContext({ foo: ["b", "c"] }), }) @@ -1620,12 +2060,12 @@ describe("resolveTemplateStrings", () => { }) }) - it("throws if $concat value is not an array and allowPartial=false", () => { + it.skip("throws if $concat value is not an array and allowPartial=false", () => { const obj = { foo: ["a", { $concat: "b" }, "d"], } - void expectError(() => resolveTemplateStrings({ source: undefined, value: obj, context: new TestContext({}) }), { + void expectError(() => resolveTemplateStrings({ value: obj, context: new TestContext({}) }), { contains: "Value of $concat key must be (or resolve to) an array (got string)", }) }) @@ -1635,17 +2075,16 @@ describe("resolveTemplateStrings", () => { foo: ["a", { $concat: "b", nope: "nay", oops: "derp" }, "d"], } - void expectError(() => resolveTemplateStrings({ source: undefined, value: obj, context: new TestContext({}) }), { + void expectError(() => resolveTemplateStrings({ value: obj, context: new TestContext({}) }), { contains: 'A list item with a $concat key cannot have any other keys (found "nope" and "oops")', }) }) - it("ignores if $concat value is not an array and allowPartial=true", () => { + it.skip("ignores if $concat value is not an array and allowPartial=true", () => { const obj = { foo: ["a", { $concat: "${foo}" }, "d"], } const res = resolveTemplateStrings({ - source: undefined, value: obj, context: new TestContext({}), contextOpts: { allowPartial: true }, @@ -1665,7 +2104,10 @@ describe("resolveTemplateStrings", () => { $else: 456, }, } - const res = resolveTemplateStrings({ source: undefined, value: obj, context: new TestContext({ foo: 1 }) }) + const res = resolveTemplateStrings({ + value: obj, + context: new TestContext({ foo: 1 }), + }) expect(res).to.eql({ bar: 123 }) }) @@ -1677,7 +2119,10 @@ describe("resolveTemplateStrings", () => { $else: 456, }, } - const res = resolveTemplateStrings({ source: undefined, value: obj, context: new TestContext({ foo: 2 }) }) + const res = resolveTemplateStrings({ + value: obj, + context: new TestContext({ foo: 2 }), + }) expect(res).to.eql({ bar: 456 }) }) @@ -1688,11 +2133,14 @@ describe("resolveTemplateStrings", () => { $then: 123, }, } - const res = resolveTemplateStrings({ source: undefined, value: obj, context: new TestContext({ foo: 2 }) }) + const res = resolveTemplateStrings({ + value: obj, + context: new TestContext({ foo: 2 }), + }) expect(res).to.eql({ bar: undefined }) }) - it("returns object as-is if $if doesn't resolve to boolean and allowPartial=true", () => { + it.skip("returns object as-is if $if doesn't resolve to boolean and allowPartial=true", () => { const obj = { bar: { $if: "${foo}", @@ -1701,7 +2149,6 @@ describe("resolveTemplateStrings", () => { }, } const res = resolveTemplateStrings({ - source: undefined, value: obj, context: new TestContext({ foo: 2 }), contextOpts: { allowPartial: true }, @@ -1718,7 +2165,11 @@ describe("resolveTemplateStrings", () => { } void expectError( - () => resolveTemplateStrings({ source: undefined, value: obj, context: new TestContext({ foo: "bla" }) }), + () => + resolveTemplateStrings({ + value: obj, + context: new TestContext({ foo: "bla" }), + }), { contains: "Value of $if key must be (or resolve to) a boolean (got string)", } @@ -1733,7 +2184,11 @@ describe("resolveTemplateStrings", () => { } void expectError( - () => resolveTemplateStrings({ source: undefined, value: obj, context: new TestContext({ foo: 1 }) }), + () => + resolveTemplateStrings({ + value: obj, + context: new TestContext({ foo: 1 }), + }), { contains: "Missing $then field next to $if field", } @@ -1750,7 +2205,11 @@ describe("resolveTemplateStrings", () => { } void expectError( - () => resolveTemplateStrings({ source: undefined, value: obj, context: new TestContext({ foo: 1 }) }), + () => + resolveTemplateStrings({ + value: obj, + context: new TestContext({ foo: 1 }), + }), { contains: 'Found one or more unexpected keys on $if object: "foo"', } @@ -1766,7 +2225,10 @@ describe("resolveTemplateStrings", () => { $return: "foo", }, } - const res = resolveTemplateStrings({ source: undefined, value: obj, context: new TestContext({}) }) + const res = resolveTemplateStrings({ + value: obj, + context: new TestContext({}), + }) expect(res).to.eql({ foo: ["foo", "foo", "foo"], }) @@ -1783,7 +2245,10 @@ describe("resolveTemplateStrings", () => { $return: "${item.key}: ${item.value}", }, } - const res = resolveTemplateStrings({ source: undefined, value: obj, context: new TestContext({}) }) + const res = resolveTemplateStrings({ + value: obj, + context: new TestContext({}), + }) expect(res).to.eql({ foo: ["a: 1", "b: 2", "c: 3"], }) @@ -1797,12 +2262,12 @@ describe("resolveTemplateStrings", () => { }, } - void expectError(() => resolveTemplateStrings({ source: undefined, value: obj, context: new TestContext({}) }), { + void expectError(() => resolveTemplateStrings({ value: obj, context: new TestContext({}) }), { contains: "Value of $forEach key must be (or resolve to) an array or mapping object (got string)", }) }) - it("ignores the loop if the input isn't a list or object and allowPartial=true", () => { + it.skip("ignores the loop if the input isn't a list or object and allowPartial=true", () => { const obj = { foo: { $forEach: "${foo}", @@ -1810,7 +2275,6 @@ describe("resolveTemplateStrings", () => { }, } const res = resolveTemplateStrings({ - source: undefined, value: obj, context: new TestContext({}), contextOpts: { allowPartial: true }, @@ -1825,7 +2289,7 @@ describe("resolveTemplateStrings", () => { }, } - void expectError(() => resolveTemplateStrings({ source: undefined, value: obj, context: new TestContext({}) }), { + void expectError(() => resolveTemplateStrings({ value: obj, context: new TestContext({}) }), { contains: "Missing $return field next to $forEach field.", }) }) @@ -1840,7 +2304,7 @@ describe("resolveTemplateStrings", () => { }, } - void expectError(() => resolveTemplateStrings({ source: undefined, value: obj, context: new TestContext({}) }), { + void expectError(() => resolveTemplateStrings({ value: obj, context: new TestContext({}) }), { contains: 'Found one or more unexpected keys on $forEach object: "$concat" and "foo"', }) }) @@ -1853,7 +2317,6 @@ describe("resolveTemplateStrings", () => { }, } const res = resolveTemplateStrings({ - source: undefined, value: obj, context: new TestContext({ foo: ["a", "b", "c"] }), }) @@ -1870,7 +2333,6 @@ describe("resolveTemplateStrings", () => { }, } const res = resolveTemplateStrings({ - source: undefined, value: obj, context: new TestContext({ foo: ["a", "b", "c"] }), }) @@ -1888,7 +2350,6 @@ describe("resolveTemplateStrings", () => { }, } const res = resolveTemplateStrings({ - source: undefined, value: obj, context: new TestContext({ foo: ["a", "b", "c"] }), }) @@ -1906,8 +2367,8 @@ describe("resolveTemplateStrings", () => { }, } - void expectError(() => resolveTemplateStrings({ source: undefined, value: obj, context: new TestContext({}) }), { - contains: "$filter clause in $forEach loop must resolve to a boolean value (got object)", + void expectError(() => resolveTemplateStrings({ value: obj, context: new TestContext({}) }), { + contains: "$filter clause in $forEach loop must resolve to a boolean value (got string)", }) }) @@ -1920,7 +2381,10 @@ describe("resolveTemplateStrings", () => { }, }, } - const res = resolveTemplateStrings({ source: undefined, value: obj, context: new TestContext({}) }) + const res = resolveTemplateStrings({ + value: obj, + context: new TestContext({}), + }) expect(res).to.eql({ foo: ["a-1", "a-2", "b-1", "b-2", "c-1", "c-2"], }) @@ -1939,7 +2403,10 @@ describe("resolveTemplateStrings", () => { }, }, } - const res = resolveTemplateStrings({ source: undefined, value: obj, context: new TestContext({}) }) + const res = resolveTemplateStrings({ + value: obj, + context: new TestContext({}), + }) expect(res).to.eql({ foo: [ ["A1", "A2"], @@ -1955,7 +2422,10 @@ describe("resolveTemplateStrings", () => { $return: "foo", }, } - const res = resolveTemplateStrings({ source: undefined, value: obj, context: new TestContext({}) }) + const res = resolveTemplateStrings({ + value: obj, + context: new TestContext({}), + }) expect(res).to.eql({ foo: [], }) @@ -1993,7 +2463,10 @@ describe("resolveTemplateStrings", () => { }, } - const res = resolveTemplateStrings({ source: undefined, value: obj, context: new TestContext({ services }) }) + const res = resolveTemplateStrings({ + value: obj, + context: new TestContext({ services }), + }) expect(res).to.eql({ services: [ { @@ -2017,6 +2490,803 @@ describe("resolveTemplateStrings", () => { }) }) +describe("lazy override", () => { + it("allows wrapping a lazy value to override something deeply", () => { + const context = new GenericContext({ + var: { + world: { + one: "one", + two: "two", + }, + }, + }) + + const obj = { + hello: "${var.world}", + } + + const parsed = parseTemplateCollection({ value: obj, source: { source: undefined } }) + + const overridden = new OverrideKeyPathLazily( + parsed, + ["hello", "three"], + new TemplateLeaf({ + value: "three", + expr: undefined, + inputs: {}, + }) + ) + + const evaluatedOriginal = deepEvaluateAndUnwrap({ value: parsed, context, opts: {} }) + + expect(evaluatedOriginal).to.eql({ + hello: { + one: "one", + two: "two", + }, + }) + + const evaluatedWithOverride = deepEvaluateAndUnwrap({ value: overridden, context, opts: {} }) + + expect(evaluatedWithOverride).to.eql({ + hello: { + one: "one", + two: "two", + three: "three", + }, + }) + }) +}) + +describe("input tracking", () => { + describe("resolveTemplateString", () => { + it("records references for every collection item in the result of the template expression, array", () => { + const context = new TestContext({ + var: { + elements: ["hydrogen", "caesium"], + }, + }) + + const value = parseTemplateString({ + string: "${var.elements}", + }) + + const res = evaluate({ value, context, opts: {} }) + + expect(res).to.deep.equal([ + { + value: "hydrogen", + expr: "${var.elements}", + inputs: { + "var.elements.0": { + expr: undefined, + inputs: {}, + value: "hydrogen", + }, + }, + }, + { + value: "caesium", + expr: "${var.elements}", + inputs: { + "var.elements.1": { + expr: undefined, + inputs: {}, + value: "caesium", + }, + }, + }, + ]) + }) + + it("records references for every collection item in the result of the template expression, array with ternary expression", () => { + const context = new TestContext({ + var: { + elements: ["hydrogen", "caesium"], + fruits: ["banana", "apple"], + colors: ["red", "green"], + }, + }) + + const value = parseTemplateString({ + string: "${ var.elements ? var.fruits : var.colors }", + }) + + const res = evaluate({ value, context, opts: {} }) + + expect(res).to.eql([ + new TemplateLeaf({ + value: "banana", + expr: "${ var.elements ? var.fruits : var.colors }", + inputs: { + "var.fruits.0": new TemplateLeaf({ + expr: undefined, + inputs: {}, + value: "banana", + }), + "var.elements.0": new TemplateLeaf({ + expr: undefined, + inputs: {}, + value: "hydrogen", + }), + "var.elements.1": new TemplateLeaf({ + expr: undefined, + inputs: {}, + value: "caesium", + }), + }, + }), + new TemplateLeaf({ + value: "apple", + expr: "${ var.elements ? var.fruits : var.colors }", + inputs: { + "var.fruits.1": new TemplateLeaf({ + expr: undefined, + inputs: {}, + value: "apple", + }), + "var.elements.0": new TemplateLeaf({ + expr: undefined, + inputs: {}, + value: "hydrogen", + }), + "var.elements.1": new TemplateLeaf({ + expr: undefined, + inputs: {}, + value: "caesium", + }), + }, + }), + ]) + }) + + it("records references for every collection item in the result of the template expression, foo bar ternary", () => { + const context = new TestContext({ + var: { + array: [], + }, + }) + + const value = parseTemplateString({ + string: "${ var.array ? 'foo' : 'bar' }", + }) + + const res = evaluate({ value, context, opts: {} }) + + // empty arrays are truthy + expect(res).to.deep.include({ + expr: "${ var.array ? 'foo' : 'bar' }", + value: "foo", + inputs: { + "var.array": { + expr: undefined, + inputs: {}, + value: [], + }, + }, + }) + }) + + it("records references for every collection item in the result of the template expression, object", () => { + const context = new TestContext({ + var: { + foo: { + bar: 1, + baz: parseTemplateString({ string: "${local.env.FRUIT}" }), + }, + }, + local: { + env: { + FRUIT: "banana", + }, + }, + }) + + const value = parseTemplateString({ + string: "${var.foo}", + }) + + const res = deepEvaluate({ value, context, opts: {} }) + + expect(res).to.eql({ + bar: new TemplateLeaf({ + value: 1, + expr: "${var.foo}", + inputs: { + "var.foo.bar": new TemplateLeaf({ + value: 1, + expr: undefined, + inputs: {}, + }), + }, + }), + baz: new TemplateLeaf({ + value: "banana", + expr: "${var.foo}", + inputs: { + "var.foo.baz": new TemplateLeaf({ + value: "banana", + expr: "${local.env.FRUIT}", + inputs: { + "local.env.FRUIT": new TemplateLeaf({ + value: "banana", + expr: undefined, + inputs: {}, + }), + }, + }), + }, + }), + }) + }) + }) + describe("resolveTemplateStrings", () => { + it("records template references (array, direct)", () => { + const context = new TestContext({ + var: { + array: ["element"], + }, + }) + const obj = { + spec: { + elements: "${var.array}", + }, + } + + const value = parseTemplateCollection({ value: obj, source: { source: undefined } }) + + const res = deepEvaluate({ value, context, opts: {} }) + + expect(res).to.eql({ + spec: { + elements: [ + new TemplateLeaf({ + expr: "${var.array}", + inputs: { + "var.array.0": new TemplateLeaf({ + expr: undefined, + inputs: {}, + value: "element", + }), + }, + value: "element", + }), + ], + }, + }) + }) + + it("records template references (forEach, no references)", () => { + const context = new TestContext({}) + const obj = { + foo: { + $forEach: [], + $return: "foo", + }, + } + + const res = resolveTemplateStrings({ value: obj, context }) + expect(res).to.eql({ + foo: [], + }) + }) + + it("merges input for every single item of the foreach separately", () => { + const context = new TestContext({ + var: { + colors: ["red", "green", "blue"], + }, + }) + const obj = { + hello: { + world: { + $forEach: "${var.colors}", + $return: "My favourite color!", + }, + }, + } + + const value = parseTemplateCollection({ value: obj, source: { source: undefined } }) + + const res = deepEvaluate({ value, context, opts: {} }) + + expect(res).to.deep.equal({ + hello: { + world: [ + { + expr: undefined, + value: "My favourite color!", + inputs: { + "var.colors.0": { + expr: undefined, + value: "red", + inputs: {}, + }, + }, + }, + { + expr: undefined, + value: "My favourite color!", + inputs: { + "var.colors.1": { + expr: undefined, + value: "green", + inputs: {}, + }, + }, + }, + { + expr: undefined, + value: "My favourite color!", + inputs: { + "var.colors.2": { + expr: undefined, + value: "blue", + inputs: {}, + }, + }, + }, + ], + }, + }) + }) + + it("tracks inputs correctly in foreach when using secrets.db_password instead of item.value", () => { + const context = new TestContext({ + var: { + array: ["element"], + }, + secrets: { + db_password: "secure", + }, + }) + const obj = { + spec: { + passwords: { + $forEach: "${var.array}", + $return: "${secrets.db_password}", + }, + }, + } + const value = parseTemplateCollection({ value: obj, source: { source: undefined } }) + const res = deepEvaluate({ value, context, opts: {} }) + expect(res).to.deep.equal({ + spec: { + passwords: [ + { + expr: "${secrets.db_password}", + value: "secure", + inputs: { + "secrets.db_password": { + value: "secure", + expr: undefined, + inputs: {}, + }, + "var.array.0": { + value: "element", + expr: undefined, + inputs: {}, + }, + }, + }, + ], + }, + }) + }) + + it("tracks inputs correctly with forEach item.value", () => { + const context = new TestContext({ + var: { + array: ["element"], + }, + }) + const obj = { + spec: { + elements: { + $forEach: "${var.array}", + $return: "${item.value}", + }, + }, + } + const value = parseTemplateCollection({ value: obj, source: { source: undefined } }) + + const res = deepEvaluate({ value, context, opts: {} }) + + expect(res).to.eql({ + spec: { + elements: [ + new TemplateLeaf({ + expr: "${item.value}", + value: "element", + inputs: { + "item.value": new TemplateLeaf({ + expr: "${var.array}", + value: "element", + inputs: { + "var.array.0": new TemplateLeaf({ + expr: undefined, + value: "element", + inputs: {}, + }), + }, + }), + }, + }), + ], + }, + }) + }) + + it("tracks inputs correctly with forEach item.value.xyz", () => { + const context = new TestContext({ + var: { + foo: [ + { + xyz: "xyz_value_0", + abc: "abc_value_0", + }, + { + xyz: "xyz_value_1", + abc: "abc_value_1", + }, + ], + }, + }) + const obj = { + foo: { + $forEach: "${var.foo}", + $return: "${item.value.xyz}", + }, + } + const value = parseTemplateCollection({ value: obj, source: { source: undefined } }) + + const res = evaluate({ value, context, opts: {} }) + + expect(res["foo"]).to.be.instanceOf(ForEachLazyValue) + + expect(deepEvaluate({ value: res, context, opts: {} })).to.deep.include({ + foo: [ + new TemplateLeaf({ + expr: "${item.value.xyz}", + value: "xyz_value_0", + inputs: { + "item.value.xyz": new TemplateLeaf({ + expr: "${var.foo}", + value: "xyz_value_0", + inputs: { + "var.foo.0.xyz": new TemplateLeaf({ + expr: undefined, + inputs: {}, + value: "xyz_value_0", + }), + }, + }), + }, + }), + new TemplateLeaf({ + expr: "${item.value.xyz}", + value: "xyz_value_1", + inputs: { + "item.value.xyz": new TemplateLeaf({ + value: "xyz_value_1", + expr: "${var.foo}", + inputs: { + "var.foo.1.xyz": new TemplateLeaf({ + expr: undefined, + inputs: {}, + value: "xyz_value_1", + }), + }, + }), + }, + }), + ], + }) + }) + + it("tracks input when using concat", () => { + const context = new TestContext({ + var: { + left: [ + { + xyz: "xyz_value_0", + abc: "abc_value_0", + }, + { + xyz: "xyz_value_1", + abc: "abc_value_1", + }, + ], + right: [{ xyz: "xyz_value_concatenated" }], + }, + }) + const obj = { + foo: { + $forEach: "${concat(var.left, var.right)}", + $return: "${item.value.xyz}", + }, + } + + const value = parseTemplateCollection({ value: obj, source: { source: undefined } }) + const res = deepEvaluate({ value, context, opts: {} }) + + expect(res).to.eql({ + foo: [ + new TemplateLeaf({ + expr: "${item.value.xyz}", + value: "xyz_value_0", + inputs: { + "item.value.xyz": new TemplateLeaf({ + expr: "${concat(var.left, var.right)}", + value: "xyz_value_0", + inputs: { + "var.left.0.xyz": new TemplateLeaf({ + expr: undefined, + inputs: {}, + value: "xyz_value_0", + }), + }, + }), + }, + }), + new TemplateLeaf({ + expr: "${item.value.xyz}", + value: "xyz_value_1", + inputs: { + "item.value.xyz": new TemplateLeaf({ + value: "xyz_value_1", + expr: "${concat(var.left, var.right)}", + inputs: { + "var.left.1.xyz": new TemplateLeaf({ + expr: undefined, + value: "xyz_value_1", + inputs: {}, + }), + }, + }), + }, + }), + new TemplateLeaf({ + expr: "${item.value.xyz}", + value: "xyz_value_concatenated", + inputs: { + "item.value.xyz": new TemplateLeaf({ + value: "xyz_value_concatenated", + expr: "${concat(var.left, var.right)}", + inputs: { + "var.right.0.xyz": new TemplateLeaf({ + value: "xyz_value_concatenated", + expr: undefined, + inputs: {}, + }), + }, + }), + }, + }), + ], + }) + }) + + it("correctly evaluates static template expressions", () => { + const context = new TestContext({}) + const obj = { + foo: "bar", + } + + const value = parseTemplateCollection({ value: obj, source: { source: undefined } }) + const res = evaluate({ value, context, opts: {} }) + + expect(res).to.eql({ + foo: new TemplateLeaf({ + expr: undefined, + value: "bar", + inputs: {}, + }), + }) + }) + + it("correctly evaluates previously resolved TemplateValues in Context with secrets.PASSWORD", () => { + const context = new TestContext({ + var: { + sharedPassword: new TemplateLeaf({ + expr: "${secrets.PASSWORD}", + value: "secure", + inputs: { + "secrets.PASSWORD": new TemplateLeaf({ + expr: undefined, + value: "secure", + inputs: {}, + }), + }, + }), + }, + }) + + const value = parseTemplateCollection({ + value: { + spec: { + password: "${var.sharedPassword}", + }, + }, + source: { source: undefined }, + }) + + const res = deepEvaluate({ value, context, opts: {} }) + + expect(res).to.eql({ + spec: { + password: new TemplateLeaf({ + expr: "${var.sharedPassword}", + value: "secure", + inputs: { + "var.sharedPassword": new TemplateLeaf({ + expr: "${secrets.PASSWORD}", + value: "secure", + inputs: { + "secrets.PASSWORD": new TemplateLeaf({ + expr: undefined, + value: "secure", + inputs: {}, + }), + }, + }), + }, + }), + }, + }) + }) + + it("correctly resolves local.env.IMAGE", () => { + const context = new TestContext({ + local: { + env: { + IMAGE: "image:latest", + }, + }, + }) + + const value = parseTemplateCollection({ + value: { + spec: { + image: "${local.env.IMAGE}", + }, + }, + source: { source: undefined }, + }) + + const res = deepEvaluate({ value, context, opts: {} }) + + expect(res).to.eql({ + spec: { + image: new TemplateLeaf({ + expr: "${local.env.IMAGE}", + value: "image:latest", + inputs: { + "local.env.IMAGE": new TemplateLeaf({ + expr: undefined, + inputs: {}, + value: "image:latest", + }), + }, + }), + }, + }) + }) + + it("correctly evaluates array with 1 reference", () => { + const context = new TestContext({ + var: { + foo: { + bar: "baz", + }, + }, + }) + + const value = parseTemplateCollection({ + value: { + foo: { + bar: ["${var.foo.bar}"], + }, + }, + source: { source: undefined }, + }) + + const res = deepEvaluate({ value, context, opts: {} }) + + expect(res).to.eql({ + foo: { + bar: [ + new TemplateLeaf({ + expr: "${var.foo.bar}", + value: "baz", + inputs: { + "var.foo.bar": new TemplateLeaf({ + expr: undefined, + inputs: {}, + value: "baz", + }), + }, + }), + ], + }, + }) + }) + + it("correctly evaluates abc-$merge", () => { + const context = new TestContext({ + var: { + foo: { + bar: "baz", + fruit: new TemplateLeaf({ + expr: "${local.env.FRUIT}", + value: "banana", + inputs: { + "local.env.FRUIT": new TemplateLeaf({ + expr: undefined, + value: "banana", + inputs: {}, + }), + }, + }), + }, + }, + }) + + const value = parseTemplateCollection({ + value: { + a: { + b: { + c: { + $merge: "${var.foo}", + }, + }, + }, + }, + source: { source: undefined }, + }) + + const res = deepEvaluate({ value, context, opts: {} }) + + expect(res).to.eql({ + a: { + b: { + c: { + bar: new TemplateLeaf({ + expr: "${var.foo}", + value: "baz", + inputs: { + "var.foo.bar": new TemplateLeaf({ + expr: undefined, + value: "baz", + inputs: {}, + }), + }, + }), + fruit: new TemplateLeaf({ + expr: "${var.foo}", + value: "banana", + inputs: { + "var.foo.fruit": new TemplateLeaf({ + expr: "${local.env.FRUIT}", + value: "banana", + inputs: { + "local.env.FRUIT": new TemplateLeaf({ + expr: undefined, + value: "banana", + inputs: {}, + }), + }, + }), + }, + }), + }, + }, + }, + }) + }) + }) +}) + describe("collectTemplateReferences", () => { it("should return and sort all template string references in an object", () => { const obj = { @@ -2198,55 +3468,6 @@ describe("getActionTemplateReferences", () => { }) }) -describe.skip("throwOnMissingSecretKeys", () => { - it("should not throw an error if no secrets are referenced", () => { - const configs = [ - { - name: "foo", - foo: "${banana.llama}", - nested: { boo: "${moo}" }, - }, - ] - - throwOnMissingSecretKeys(configs, {}, "Module") - throwOnMissingSecretKeys(configs, { someSecret: "123" }, "Module") - }) - - it("should throw an error if one or more secrets is missing", () => { - const configs = [ - { - name: "moduleA", - foo: "${secrets.a}", - nested: { boo: "${secrets.b}" }, - }, - { - name: "moduleB", - bar: "${secrets.a}", - nested: { boo: "${secrets.b}" }, - baz: "${secrets.c}", - }, - ] - - void expectError( - () => throwOnMissingSecretKeys(configs, { b: "123" }, "Module"), - (err) => { - expect(err.message).to.match(/Module moduleA: a/) - expect(err.message).to.match(/Module moduleB: a, c/) - expect(err.message).to.match(/Secret keys with loaded values: b/) - } - ) - - void expectError( - () => throwOnMissingSecretKeys(configs, {}, "Module"), - (err) => { - expect(err.message).to.match(/Module moduleA: a, b/) - expect(err.message).to.match(/Module moduleB: a, b, c/) - expect(err.message).to.match(/Note: No secrets have been loaded./) - } - ) - }) -}) - describe("functional tests", () => { context("cross-context variable references", () => { let dataDir: string diff --git a/core/test/unit/src/template-string/proxy.ts b/core/test/unit/src/template-string/proxy.ts new file mode 100644 index 0000000000..02190b5080 --- /dev/null +++ b/core/test/unit/src/template-string/proxy.ts @@ -0,0 +1,440 @@ +/* + * Copyright (C) 2018-2023 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { expect } from "chai" +import { GenericContext } from "../../../../src/config/template-contexts/base.js" +import { getCollectionSymbol, getLazyConfigProxy } from "../../../../src/template-string/proxy.js" +import { parseTemplateString, parseTemplateCollection } from "../../../../src/template-string/template-string.js" +import type { CollectionOrValue } from "../../../../src/util/objects.js" +import { isArray } from "../../../../src/util/objects.js" +import { type TemplatePrimitive } from "../../../../src/template-string/inputs.js" + +describe("getLazyConfigProxy", () => { + it("makes it easier to access template values", () => { + const obj = { + fruits: ["apple", "banana"], + } + + const proxy = getLazyConfigProxy({ + parsedConfig: parseTemplateCollection({ value: obj, source: { source: undefined } }), + context: new GenericContext({}), + opts: {}, + }) + + expect(proxy["fruits"][0]).to.equal("apple") + expect(proxy["fruits"][1]).to.equal("banana") + }) + + it("only supports instantiating the proxy with collection values, not primitives, even if they are hidden behind lazy values", () => { + const proxy = getLazyConfigProxy({ + parsedConfig: parseTemplateString({ string: "${1234}" }), + context: new GenericContext({}), + opts: {}, + }) + + expect(() => { + proxy["foo"] + }).to.throw() + }) + + it("should unwrap a leaf value", () => { + const parsedConfig = parseTemplateCollection({ + value: { + my: { + deep: { + structure: { + withAnArray: [{ containing: "different" }, "stuff"], + }, + }, + }, + }, + source: { source: undefined }, + }) + + const proxy = getLazyConfigProxy({ + parsedConfig, + context: new GenericContext({}), + opts: {}, + }) + + expect(proxy).to.deep.equal({ + my: { + deep: { + structure: { + withAnArray: [{ containing: "different" }, "stuff"], + }, + }, + }, + }) + }) + + it("should have array methods and properties", () => { + const parsedConfig = parseTemplateCollection({ + value: { + myArray: [1, 2, 3], + myDeeperArray: [{ foo: "${[10,9,8,7]}" }, { foo: "baz" }], + }, + source: { source: undefined }, + }) + + const proxy = getLazyConfigProxy({ + parsedConfig, + context: new GenericContext({}), + opts: {}, + }) + + expect(proxy["myArray"].length).to.equal(3) + expect(proxy["myDeeperArray"].length).to.equal(2) + expect(proxy["myDeeperArray"]).to.be.an("array") + expect(proxy["myDeeperArray"][0].foo.length).to.equal(4) + expect(proxy["myDeeperArray"][0].foo).to.be.an("array") + expect(isArray(proxy)).to.equal(false) + }) + + it("should support iteration over proxied array values", () => { + const parsedConfig = parseTemplateCollection({ + value: ["Hello 1", "Hello 2", "Hello 3"], + source: { source: undefined }, + }) + + const proxy = getLazyConfigProxy({ + parsedConfig, + expectedCollectionType: "array", + context: new GenericContext({}), + opts: {}, + }) as string[] + + const iterated1: string[] = [] + for (const item of proxy) { + iterated1.push(item) + } + const iterated2 = proxy.map((item) => item) + const iterated3: string[] = [] + proxy.forEach((item) => iterated3.push(item)) + + expect(proxy.length).to.equal(3) + expect(proxy).to.be.an("array") + expect(iterated1).to.deep.equal(["Hello 1", "Hello 2", "Hello 3"]) + expect(iterated2).to.deep.equal(["Hello 1", "Hello 2", "Hello 3"]) + expect(iterated3).to.deep.equal(["Hello 1", "Hello 2", "Hello 3"]) + }) + + it("should support iteration over proxied Object.entries", () => { + const parsedConfig = parseTemplateCollection({ + value: { + foo: "bar", + baz: "qux", + }, + source: { source: undefined }, + }) + + const proxy = getLazyConfigProxy({ + parsedConfig, + context: new GenericContext({}), + opts: {}, + }) + + const iterated1: string[] = [] + for (const [k, v] of Object.entries(proxy)) { + iterated1.push(`${k}=${v}`) + } + const iterated2 = Object.entries(proxy).map(([k, v]) => `${k}=${v}`) + const iterated3: string[] = [] + Object.entries(proxy).forEach(([k, v]) => iterated3.push(`${k}=${v}`)) + + expect(iterated1).to.deep.equal(["foo=bar", "baz=qux"]) + expect(iterated2).to.deep.equal(["foo=bar", "baz=qux"]) + expect(iterated3).to.deep.equal(["foo=bar", "baz=qux"]) + }) + + it("should work with forEach", () => { + const parsedConfig = parseTemplateCollection({ + value: { + $forEach: "${[1,2,3]}", + $return: "Hello ${item.value}", + }, + source: { source: undefined }, + }) + + const proxy = getLazyConfigProxy({ + parsedConfig, + context: new GenericContext({}), + expectedCollectionType: "array", + opts: {}, + }) + + expect(proxy).to.deep.equal(["Hello 1", "Hello 2", "Hello 3"]) + expect(proxy).to.be.an("array") + expect(proxy.length).to.equal(3) + }) + + it("it only lazily evaluates even the outermost lazy value", () => { + const context = new GenericContext({}) + const parsedConfig = parseTemplateCollection({ + value: { + $merge: "${var.willExistLater}", + }, + source: { source: undefined }, + }) + + let proxy: CollectionOrValue + expect(() => { + proxy = getLazyConfigProxy({ + parsedConfig, + context, + opts: {}, + }) + }).to.not.throw() + + context["var"] = { + willExistLater: { + foo: "bar", + }, + } + + expect(proxy).to.deep.equal({ foo: "bar" }) + }) + + it("evaluates lazily when values are accessed", () => { + const context = new GenericContext({}) + const parsedConfig = parseTemplateCollection({ + value: { + spec: "${var.willExistLater}", + }, + source: { source: undefined }, + }) + + let proxy: CollectionOrValue + expect(() => { + proxy = getLazyConfigProxy({ + parsedConfig, + context, + opts: {}, + }) + }).to.not.throw() + + context["var"] = { + willExistLater: { + foo: "bar", + }, + } + + expect(proxy).to.deep.equal({ spec: { foo: "bar" } }) + }) + + it("evaluates lazily when values are accessed", () => { + const context = new GenericContext({ + var: { + alreadyExists: "I am here", + }, + }) + const parsedConfig = parseTemplateCollection({ + value: { + cannotResolveYet: "${var.willExistLater}", + alreadyResolvable: "${var.alreadyExists}", + }, + source: { source: undefined }, + }) + + const proxy = getLazyConfigProxy({ + parsedConfig, + context, + opts: {}, + }) + + expect(Object.keys(proxy)).to.eql(["cannotResolveYet", "alreadyResolvable"]) + + expect(proxy["alreadyResolvable"]).to.equal("I am here") + + expect(() => { + proxy["cannotResolveYet"] + }).to.throw() + + context["var"]["willExistLater"] = { + foo: "bar", + } + + expect(proxy["cannotResolveYet"]).to.deep.equal({ foo: "bar" }) + }) + + it("allows variables referencing other variables even when they are declared together in the same scope", () => { + const projectConfig = parseTemplateCollection({ + value: { + variables: { + variable_one: "${var.variable_two}", + variable_two: '${join(["H", "e", "l", "l", "o"], "")}', + }, + }, + source: { source: undefined }, + }) + + const context = new GenericContext({ + var: { + ...projectConfig["variables"], + }, + }) + + const actionConfig = parseTemplateCollection({ + value: { + kind: "Build", + name: "my-action", + spec: { + image: "${var.variable_one}", + }, + }, + source: { source: undefined }, + }) + + const proxy = getLazyConfigProxy({ + parsedConfig: actionConfig, + context, + opts: {}, + }) + + expect(proxy).to.deep.equal({ + kind: "Build", + name: "my-action", + spec: { + image: "Hello", + }, + }) + }) + + it("allows optional proxies, that do not throw but return undefined, if something can't be resolved", () => { + const context = new GenericContext({}) + const parsedConfig = parseTemplateCollection({ + value: { + myOptionalValue: "${var.willExistLater}", + }, + source: { source: undefined }, + }) + + const proxy = getLazyConfigProxy({ + parsedConfig, + context, + opts: { + // TODO: rename this to optional + allowPartial: true, + }, + }) + + expect(proxy["myOptionalValue"]).to.equal(undefined) + + context["var"] = { + willExistLater: { + foo: "bar", + }, + } + + expect(proxy).to.deep.equal({ myOptionalValue: { foo: "bar" } }) + }) + + it("partial returns undefined even for && and || clauses", () => { + const context = new GenericContext({}) + const parsedConfig = parseTemplateCollection({ + value: { + orClause: "${var.willExistLater || 'defaultValue'}", + andClause: "${var.willExistLater && 'conclusionValue'}", + }, + source: { source: undefined }, + }) + + const partialProxy = getLazyConfigProxy({ + parsedConfig, + context, + opts: { + // TODO: rename this to optional + allowPartial: true, + }, + }) + + const strictProxy = getLazyConfigProxy({ + parsedConfig, + context, + opts: {}, + }) + + expect(partialProxy["orClause"]).to.equal(undefined) + expect(partialProxy["andClause"]).to.equal(undefined) + + expect(strictProxy["orClause"]).to.equal("defaultValue") + expect(strictProxy["andClause"]).to.equal(false) + + context["var"] = { + willExistLater: { + foo: "bar", + }, + } + + expect(partialProxy["orClause"]).to.deep.equal({ foo: "bar" }) + expect(strictProxy["orClause"]).to.deep.equal({ foo: "bar" }) + expect(partialProxy["andClause"]).to.deep.equal("conclusionValue") + expect(strictProxy["andClause"]).to.deep.equal("conclusionValue") + }) + + it("allows getting the underlying collection back", () => { + const context = new GenericContext({}) + const parsedConfig = parseTemplateCollection({ + value: { + myOptionalValue: "${var.willExistLater}", + }, + source: { source: undefined }, + }) + + const proxy = getLazyConfigProxy({ + parsedConfig, + context, + opts: {}, + }) + + const underlyingConfig = proxy[getCollectionSymbol] + + expect(underlyingConfig).to.deep.equal(parsedConfig) + }) + + it("allows getting the underlying collection back for arrays", () => { + const context = new GenericContext({}) + const parsedConfig = parseTemplateCollection({ + value: ["Hello 1", "Hello 2", "Hello 3"], + source: { source: undefined }, + }) + + const proxy = getLazyConfigProxy({ + parsedConfig, + expectedCollectionType: "array", + context, + opts: {}, + }) + + const underlyingConfig = proxy[getCollectionSymbol] + + expect(underlyingConfig).to.deep.equal(parsedConfig) + }) + + it("forbids mutating the proxy", () => { + const context = new GenericContext({}) + const parsedConfig = parseTemplateCollection({ + value: { + luckyNumber: "${3}", + }, + source: { source: undefined }, + }) + + const proxy = getLazyConfigProxy({ + parsedConfig, + context, + opts: {}, + }) + + expect(() => { + proxy["luckyNumber"] = 13 + }).to.throw() + + expect(proxy["luckyNumber"]).to.equal(3) + }) +}) diff --git a/core/test/unit/src/template-string/validation.ts b/core/test/unit/src/template-string/validation.ts new file mode 100644 index 0000000000..2cda009c7a --- /dev/null +++ b/core/test/unit/src/template-string/validation.ts @@ -0,0 +1,237 @@ +/* + * Copyright (C) 2018-2023 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { z } from "zod" +import { parseTemplateCollection } from "../../../../src/template-string/template-string.js" +import { expect } from "chai" +import { GenericContext } from "../../../../src/config/template-contexts/base.js" +import { GardenConfig } from "../../../../src/template-string/validation.js" +import Joi from "@hapi/joi" + +// In the future we might +// const varsFromFirstConfig = firstConfig.atPath("var") // can contain lazy values +// const actionConfigWithVars = actionConfig.merge(firstConfig) + +describe("GardenConfig", () => { + it("takes parsed config collection, and offers a validate() method that returns a lazy config proxy with the correct type information", () => { + const parsedConfig = parseTemplateCollection({ + value: { + kind: "Deployment", + type: "kubernetes", + spec: { + files: ["manifests/deployment.yaml"], + }, + }, + source: { source: undefined }, + }) + + const unrefinedConfig = new GardenConfig({ + parsedConfig, + source: undefined, + context: new GenericContext({}), + opts: {}, + }) + + const config = unrefinedConfig.refineWithZod( + z.object({ + kind: z.literal("Deployment"), + type: z.literal("kubernetes"), + spec: z.object({ + files: z.array(z.string()), + }), + }) + ) + + const proxy = config.config + + // proxy has type hints, no need to use bracket notation + expect(proxy.spec.files[0]).to.equal("manifests/deployment.yaml") + }) + + it("if schema validation mutates the data structures, e.g. it has defaults, the original config object does not change", () => { + const parsedConfig = parseTemplateCollection({ + value: { + kind: "Deployment", + type: "kubernetes", + spec: { + // replicas defaults to 1, but isn't specified here + files: ["manifests/deployment.yaml"], + }, + }, + source: { source: undefined }, + }) + + const unrefinedConfig = new GardenConfig({ + parsedConfig, + context: new GenericContext({}), + source: undefined, + opts: {}, + }) + + const config = unrefinedConfig.refineWithZod( + z.object({ + kind: z.literal("Deployment"), + type: z.literal("kubernetes"), + spec: z.object({ + // replicas defaults to 1 + replicas: z.number().default(1), + files: z.array(z.string()), + }), + }) + ) + + const proxy = config.config + + // const spec = proxy.spec + + // spec.replicas = 2 + + // expect(proxy.spec.replicas).to.equal(2) + + // proxy has type hints, no need to use bracket notation + expect(proxy.spec.replicas).to.equal(1) + + expect(proxy).to.deep.equal({ + kind: "Deployment", + type: "kubernetes", + spec: { + files: ["manifests/deployment.yaml"], + replicas: 1, + }, + }) + + const unrefinedProxy = unrefinedConfig.config + + // the unrefined config has not been mutated + expect(unrefinedProxy).to.deep.equal({ + kind: "Deployment", + type: "kubernetes", + spec: { + files: ["manifests/deployment.yaml"], + }, + }) + }) + + it("does not keep overrides when the context changes", () => { + const parsedConfig = parseTemplateCollection({ + value: { + replicas: "${var.replicas}", + }, + source: { source: undefined }, + }) + + const config1 = new GardenConfig({ + parsedConfig, + context: new GenericContext({}), + source: undefined, + opts: { + allowPartial: true, + }, + }).refineWithZod( + z.object({ + // if replicas is not specified, it defaults to 1 + replicas: z.number().default(1), + }) + ) + + const proxy1 = config1.config + + // replicas is specified, but it's using a variable that's not defined yet and the proxy is in `allowPartial` mode + expect(proxy1.replicas).to.equal(1) + + // Now var.replicas is defined and the default from spec.replicas should not be used anymore. + const config2 = config1 + .withContext(new GenericContext({ var: { replicas: 7 } })) + .refineWithZod( + z.object({ + // if replicas is not specified, it defaults to 1 + replicas: z.number().default(1), + }) + ) + + // You can even refine multiple times, and the types will be merged together. + .refineWithZod( + z.object({ + foobar: z.string().default("foobar"), + }) + ) + + const proxy2 = config2.config + + proxy2 satisfies { replicas: number; foobar: string } + + expect(proxy2.replicas).to.equal(7) + expect(proxy2.foobar).to.equal("foobar") + }) + + it("can be used with any type assertion", () => { + const context = new GenericContext({ var: { fruits: ["apple", "banana"] } }) + + const isFruits = (value: any): value is { fruits: string[] } => { + if (Array.isArray(value.fruits)) { + return value.fruits.every((item) => { + return typeof item === "string" + }) + } + return false + } + + const parsedConfig = parseTemplateCollection({ + value: { + fruits: "${var.fruits}", + }, + source: { source: undefined }, + }) + + const config = new GardenConfig({ + parsedConfig, + context, + source: undefined, + opts: { + allowPartial: true, + }, + }).assertType(isFruits) + + const proxy = config.config + + proxy satisfies { fruits: string[] } + + expect(proxy.fruits).to.deep.equal(["apple", "banana"]) + }) + + it("can be used with joi validators", () => { + const fruitsSchema = Joi.object({ fruits: Joi.array().items(Joi.string()) }) + type Fruits = { + fruits: string[] + } + + const context = new GenericContext({ var: { fruits: ["apple", "banana"] } }) + + const parsedConfig = parseTemplateCollection({ + value: { + fruits: "${var.fruits}", + }, + source: { source: undefined }, + }) + + const config = new GardenConfig({ + parsedConfig, + context, + source: undefined, + opts: { + allowPartial: true, + }, + }).refineWithJoi(fruitsSchema) + + const proxy = config.config + + proxy satisfies Fruits + + expect(proxy.fruits).to.deep.equal(["apple", "banana"]) + }) +})