From e6a215277473cbae5bbf1a5927fce7e255c072b4 Mon Sep 17 00:00:00 2001 From: Jon Edvald Date: Tue, 9 Nov 2021 09:27:03 +0100 Subject: [PATCH] feat(config): support for-loops for lists You can now map through a list of values by using the special `$forEach/$return` object. You specify an object with two keys, `$forEach: ` and `$return: `. You can also optionally add a `$filter: ` key, which if evaluates to `false` for a particular value, it will be omitted. See the added docs for more details and examples. --- core/src/config/common.ts | 10 +- core/src/config/template-contexts/base.ts | 2 +- core/src/template-string/template-string.ts | 165 ++++++++++--- core/test/unit/src/template-string.ts | 224 ++++++++++++++++++ docs/using-garden/variables-and-templating.md | 101 ++++++++ examples/demo-project/backend/garden.yml | 7 +- 6 files changed, 474 insertions(+), 35 deletions(-) diff --git a/core/src/config/common.ts b/core/src/config/common.ts index dfe7d18d10..9f600f2880 100644 --- a/core/src/config/common.ts +++ b/core/src/config/common.ts @@ -16,6 +16,9 @@ import { DEFAULT_API_VERSION } from "../constants" export const objectSpreadKey = "$merge" export const arrayConcatKey = "$concat" +export const arrayForEachKey = "$forEach" +export const arrayForEachReturnKey = "$return" +export const arrayForEachFilterKey = "$filter" const ajv = new Ajv({ allErrors: true, useDefaults: true }) @@ -315,9 +318,14 @@ joi = joi.extend({ // return { value } // }, args(schema: any, keys: any) { - // Always allow the $merge key, which we resolve and collapse in resolveTemplateStrings() + // Always allow the special $merge, $forEach etc. keys, which we resolve and collapse in resolveTemplateStrings() + // Note: we allow both the expected schema and strings, since they may be templates resolving to the expected type. return schema.keys({ [objectSpreadKey]: joi.alternatives(joi.object(), joi.string()), + [arrayConcatKey]: joi.alternatives(joi.array(), joi.string()), + [arrayForEachKey]: joi.alternatives(joi.array(), joi.string()), + [arrayForEachFilterKey]: joi.any(), + [arrayForEachReturnKey]: joi.any(), ...(keys || {}), }) }, diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index 8f5c2f431a..7944e9b314 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -53,7 +53,7 @@ export function schema(joiSchema: Joi.Schema) { // Note: we're using classes here to be able to use decorators to describe each context node and key export abstract class ConfigContext { private readonly _rootContext: ConfigContext - private readonly _resolvedValues: { [path: string]: string } + private readonly _resolvedValues: { [path: string]: any } // This is used for special-casing e.g. runtime.* resolution protected _alwaysAllowPartial: boolean diff --git a/core/src/template-string/template-string.ts b/core/src/template-string/template-string.ts index 2855d85ca9..63d895c15a 100644 --- a/core/src/template-string/template-string.ts +++ b/core/src/template-string/template-string.ts @@ -13,11 +13,21 @@ import { ScanContext, ContextResolveOutput, ContextKeySegment, + GenericContext, } from "../config/template-contexts/base" -import { difference, uniq, isPlainObject, isNumber } from "lodash" -import { Primitive, StringMap, isPrimitive, objectSpreadKey, arrayConcatKey } from "../config/common" +import { difference, uniq, isPlainObject, isNumber, cloneDeep } from "lodash" +import { + Primitive, + StringMap, + isPrimitive, + objectSpreadKey, + arrayConcatKey, + arrayForEachKey, + arrayForEachReturnKey, + arrayForEachFilterKey, +} from "../config/common" import { profile } from "../util/profiling" -import { dedent, deline, truncate } from "../util/string" +import { dedent, deline, naturalList, truncate } from "../util/string" import { ObjectWithName } from "../util/util" import { LogEntry } from "../logger/log-entry" import { ModuleConfigContext } from "../config/template-contexts/module" @@ -206,9 +216,23 @@ export const resolveTemplateStrings = profile(function $resolveTemplateStrings 1) { + const extraKeys = naturalList( + Object.keys(v) + .filter((k) => k !== arrayConcatKey) + .map((k) => JSON.stringify(k)) + ) + throw new ConfigurationError( + `A list item with a ${arrayConcatKey} key cannot have any other keys (found ${extraKeys})`, + { + value: v, + } + ) + } + // Handle array concatenation via $concat - const resolved = resolveTemplateStrings(v.$concat, context, opts) + const resolved = resolveTemplateStrings(v[arrayConcatKey], context, opts) if (Array.isArray(resolved)) { output.push(...resolved) @@ -230,37 +254,124 @@ export const resolveTemplateStrings = profile(function $resolveTemplateStrings(output) } else if (isPlainObject(value)) { - // Resolve $merge keys, depth-first, leaves-first - let output = {} - - for (const [k, v] of Object.entries(value)) { - const resolved = resolveTemplateStrings(v, context, opts) - - if (k === objectSpreadKey) { - if (isPlainObject(resolved)) { - output = { ...output, ...resolved } - } else if (opts.allowPartial) { - output[k] = resolved + if (value[arrayForEachKey] !== undefined) { + // Handle $forEach loop + return handleForEachObject(value, context, opts) + } else { + // Resolve $merge keys, depth-first, leaves-first + let output = {} + + for (const [k, v] of Object.entries(value)) { + const resolved = resolveTemplateStrings(v, context, opts) + + if (k === objectSpreadKey) { + if (isPlainObject(resolved)) { + output = { ...output, ...resolved } + } else if (opts.allowPartial) { + output[k] = resolved + } else { + throw new ConfigurationError( + `Value of ${objectSpreadKey} key must be (or resolve to) a mapping object (got ${typeof resolved})`, + { + value, + resolved, + } + ) + } } else { - throw new ConfigurationError( - `Value of ${objectSpreadKey} key must be (or resolve to) a mapping object (got ${typeof resolved})`, - { - value, - resolved, - } - ) + output[k] = resolved } - } else { - output[k] = resolved } - } - return output + return output + } } else { return value } }) +const expectedKeys = [arrayForEachKey, arrayForEachReturnKey, arrayForEachFilterKey] + +function handleForEachObject(value: any, context: ConfigContext, opts: ContextResolveOpts) { + // Validate input object + if (value[arrayForEachReturnKey] === undefined) { + throw new ConfigurationError(`Missing ${arrayForEachReturnKey} field next to ${arrayForEachKey} field.`, { + value, + }) + } + + const unexpectedKeys = Object.keys(value).filter((k) => !expectedKeys.includes(k)) + + if (unexpectedKeys.length > 0) { + const extraKeys = naturalList(unexpectedKeys.map((k) => JSON.stringify(k))) + + throw new ConfigurationError(`Found one or more unexpected keys on $forEach object: ${extraKeys}`, { + value, + expectedKeys, + unexpectedKeys, + }) + } + + // Try resolving the value of the $forEach key + let resolvedInput = resolveTemplateStrings(value[arrayForEachKey], context, opts) + const isObject = isPlainObject(resolvedInput) + + if (!Array.isArray(resolvedInput) && !isObject) { + if (opts.allowPartial) { + return value + } else { + throw new ConfigurationError( + `Value of ${arrayForEachKey} key must be (or resolve to) an array or mapping object (got ${typeof resolvedInput})`, + { + value, + resolved: resolvedInput, + } + ) + } + } + + const filterExpression = value[arrayForEachFilterKey] + + // TODO: maybe there's a more efficient way to do the cloning/extending? + const loopContext = cloneDeep(context) + + const output: unknown[] = [] + + for (const i of Object.keys(resolvedInput)) { + const itemValue = resolvedInput[i] + + loopContext["item"] = new GenericContext({ key: i, value: itemValue }) + + // Have to override the cache in the parent context here + // TODO: make this a little less hacky :P + delete loopContext["_resolvedValues"]["item.key"] + delete loopContext["_resolvedValues"]["item.value"] + + // Check $filter clause output, if applicable + if (filterExpression !== undefined) { + const filterResult = resolveTemplateStrings(value[arrayForEachFilterKey], loopContext, opts) + + if (filterResult === false) { + continue + } else if (filterResult !== true) { + throw new ConfigurationError( + `${arrayForEachFilterKey} clause in ${arrayForEachKey} loop must resolve to a boolean value (got ${typeof resolvedInput})`, + { + itemValue, + filterExpression, + filterResult, + } + ) + } + } + + output.push(resolveTemplateStrings(value[arrayForEachReturnKey], loopContext, opts)) + } + + // Need to resolve once more to handle e.g. $concat expressions + return resolveTemplateStrings(output, context, opts) +} + /** * Scans for all template strings in the given object and lists the referenced keys. */ diff --git a/core/test/unit/src/template-string.ts b/core/test/unit/src/template-string.ts index 1a9322d9e7..fd88c4a361 100644 --- a/core/test/unit/src/template-string.ts +++ b/core/test/unit/src/template-string.ts @@ -1224,6 +1224,16 @@ describe("resolveTemplateStrings", () => { }) }) + it("resolves a $forEach in the $concat clause", () => { + const obj = { + foo: ["a", { $concat: { $forEach: ["B", "C"], $return: "${lower(item.value)}" } }, "d"], + } + const res = resolveTemplateStrings(obj, new TestContext({ foo: ["b", "c"] })) + expect(res).to.eql({ + foo: ["a", "b", "c", "d"], + }) + }) + it("throws if $concat value is not an array and allowPartial=false", () => { const obj = { foo: ["a", { $concat: "b" }, "d"], @@ -1236,6 +1246,20 @@ describe("resolveTemplateStrings", () => { ) }) + it("throws if object with $concat key contains other keys as well", () => { + const obj = { + foo: ["a", { $concat: "b", nope: "nay", oops: "derp" }, "d"], + } + + expectError( + () => resolveTemplateStrings(obj, new TestContext({})), + (err) => + expect(stripAnsi(err.message)).to.equal( + 'A list item with a $concat key cannot have any other keys (found "nope" and "oops")' + ) + ) + }) + it("ignores if $concat value is not an array and allowPartial=true", () => { const obj = { foo: ["a", { $concat: "${foo}" }, "d"], @@ -1246,6 +1270,206 @@ describe("resolveTemplateStrings", () => { }) }) }) + + context("$forEach", () => { + it("loops through an array", () => { + const obj = { + foo: { + $forEach: ["a", "b", "c"], + $return: "foo", + }, + } + const res = resolveTemplateStrings(obj, new TestContext({})) + expect(res).to.eql({ + foo: ["foo", "foo", "foo"], + }) + }) + + it("loops through an object", () => { + const obj = { + foo: { + $forEach: { + a: 1, + b: 2, + c: 3, + }, + $return: "${item.key}: ${item.value}", + }, + } + const res = resolveTemplateStrings(obj, new TestContext({})) + expect(res).to.eql({ + foo: ["a: 1", "b: 2", "c: 3"], + }) + }) + + it("throws if the input isn't a list or object and allowPartial=false", () => { + const obj = { + foo: { + $forEach: "foo", + $return: "foo", + }, + } + + expectError( + () => resolveTemplateStrings(obj, new TestContext({})), + (err) => + expect(stripAnsi(err.message)).to.equal( + "Value of $forEach key must be (or resolve to) an array or mapping object (got string)" + ) + ) + }) + + it("ignores the loop if the input isn't a list or object and allowPartial=true", () => { + const obj = { + foo: { + $forEach: "${foo}", + $return: "foo", + }, + } + const res = resolveTemplateStrings(obj, new TestContext({}), { allowPartial: true }) + expect(res).to.eql(obj) + }) + + it("throws if there's no $return clause", () => { + const obj = { + foo: { + $forEach: [1, 2, 3], + }, + } + + expectError( + () => resolveTemplateStrings(obj, new TestContext({})), + (err) => expect(stripAnsi(err.message)).to.equal("Missing $return field next to $forEach field.") + ) + }) + + it("throws if there are superfluous keys on the object", () => { + const obj = { + foo: { + $forEach: [1, 2, 3], + $return: "foo", + $concat: [4, 5, 6], + foo: "bla", + }, + } + + expectError( + () => resolveTemplateStrings(obj, new TestContext({})), + (err) => + expect(stripAnsi(err.message)).to.equal( + 'Found one or more unexpected keys on $forEach object: "$concat" and "foo"' + ) + ) + }) + + it("exposes item.value and item.key when resolving the $return clause", () => { + const obj = { + foo: { + $forEach: "${foo}", + $return: "${item.key}: ${item.value}", + }, + } + const res = resolveTemplateStrings(obj, new TestContext({ foo: ["a", "b", "c"] })) + expect(res).to.eql({ + foo: ["0: a", "1: b", "2: c"], + }) + }) + + it("resolves the input before processing", () => { + const obj = { + foo: { + $forEach: "${foo}", + $return: "${item.value}", + }, + } + const res = resolveTemplateStrings(obj, new TestContext({ foo: ["a", "b", "c"] })) + expect(res).to.eql({ + foo: ["a", "b", "c"], + }) + }) + + it("filters out items if $filter resolves to false", () => { + const obj = { + foo: { + $forEach: "${foo}", + $filter: "${item.value != 'b'}", + $return: "${item.value}", + }, + } + const res = resolveTemplateStrings(obj, new TestContext({ foo: ["a", "b", "c"] })) + expect(res).to.eql({ + foo: ["a", "c"], + }) + }) + + it("throws if $filter doesn't resolve to a boolean", () => { + const obj = { + foo: { + $forEach: ["a", "b", "c"], + $filter: "foo", + $return: "${item.value}", + }, + } + + expectError( + () => resolveTemplateStrings(obj, new TestContext({})), + (err) => + expect(stripAnsi(err.message)).to.equal( + "$filter clause in $forEach loop must resolve to a boolean value (got object)" + ) + ) + }) + + it("handles $concat clauses in $return", () => { + const obj = { + foo: { + $forEach: ["a", "b", "c"], + $return: { + $concat: ["${item.value}-1", "${item.value}-2"], + }, + }, + } + const res = resolveTemplateStrings(obj, new TestContext({})) + expect(res).to.eql({ + foo: ["a-1", "a-2", "b-1", "b-2", "c-1", "c-2"], + }) + }) + + it("handles $forEach clauses in $return", () => { + const obj = { + foo: { + $forEach: [ + ["a1", "a2"], + ["b1", "b2"], + ], + $return: { + $forEach: "${item.value}", + $return: "${upper(item.value)}", + }, + }, + } + const res = resolveTemplateStrings(obj, new TestContext({})) + expect(res).to.eql({ + foo: [ + ["A1", "A2"], + ["B1", "B2"], + ], + }) + }) + + it("resolves to empty list for empty list input", () => { + const obj = { + foo: { + $forEach: [], + $return: "foo", + }, + } + const res = resolveTemplateStrings(obj, new TestContext({})) + expect(res).to.eql({ + foo: [], + }) + }) + }) }) describe("collectTemplateReferences", () => { diff --git a/docs/using-garden/variables-and-templating.md b/docs/using-garden/variables-and-templating.md index 3e161249ee..f5deb93f94 100644 --- a/docs/using-garden/variables-and-templating.md +++ b/docs/using-garden/variables-and-templating.md @@ -230,6 +230,107 @@ tests: - suite-b ``` +### For loops + +You can map through a list of values by using the special `$forEach/$return` object. + +You specify an object with two keys, `$forEach: ` and `$return: `. You can also optionally add a `$filter: ` key, which if evaluates to `false` for a particular value, it will be omitted. + +Template strings in the `$return` and `$filter` fields are resolved with the same template context as what's available when resolving the for-loop, in addition to `${item.value}` which resolves to the list item being processed, and `${item.key}`. + +You can loop over lists as well as mapping objects. When looping over lists, `${item.key}` resolves to the index number (starting with 0) of the item in the list. When looping over mapping objects, `${item.key}` is simply the key name of the key value pair. + +Here's an example where we kebab-case a list of string values: + +```yaml +kind: Module +... +variables: + values: + - some_name + - AnotherName + - __YET_ANOTHER_NAME__ +tasks: + - name: my-task + # resolves to [some-name, another-name, yet-another-name] + args: + $forEach: ${var.values} + $return: ${kebabCase(item.value)} +``` + +Here's another example, where we create an object for each value in a list and skip certain values: + +```yaml +kind: Module +... +variables: + ports: + - 80 + - 8000 + - 8100 + - 8200 +services: + - name: my-service + ports: + # loop through the ports list declared above + $forEach: ${var.ports} + # only use values higher than 1000 + $filter: ${item.value > 1000} + # for each port number, create an object with a name and a port key + $return: + name: port-${item.key} # item.key is the array index, starting with 0 + containerPort: ${item.value} +``` + +And here we loop over a mapping object instead of a list: + +```yaml +kind: Module +... +variables: + ports: + http: 8000 + admin: 8100 + debug: 8200 +services: + - name: my-service + ports: + # loop through the ports map declared above + $forEach: ${var.ports} + # for each port number, create an object with a name and a port key + $return: + name: ${item.key} + containerPort: ${item.value} +``` + +And lastly, here we have an arbitrary object for each value instead of a simple numeric value: + +```yaml +kind: Module +... +variables: + ports: + http: + container: 8000 + service: 80 + admin: + container: 8100 + debug: + container: 8200 +services: + - name: my-service + ports: + # loop through the ports map declared above + $forEach: ${var.ports} + # for each port number, create an object with a name and a port key + $return: + name: ${item.key} + # see how we can reference nested keys on item.value + containerPort: ${item.value.container} + # resolve to the service key if it's set, otherwise the container key + servicePort: ${item.value.service || item.value.container} +``` + ### Merging maps Any object or mapping field supports a special `$merge` key, which allows you to merge two objects together. This can be used to avoid repeating a set of commonly repeated values. diff --git a/examples/demo-project/backend/garden.yml b/examples/demo-project/backend/garden.yml index 6e793e34c2..22b96313e2 100644 --- a/examples/demo-project/backend/garden.yml +++ b/examples/demo-project/backend/garden.yml @@ -6,9 +6,6 @@ type: container # You can specify variables here at the module level variables: ingressPath: /hello-backend - commandPrefix: - - sh - - -c services: - name: backend @@ -27,6 +24,4 @@ services: tasks: - name: test - command: - - $concat: ${var.commandPrefix} - - "echo task output" \ No newline at end of file + command: ["sh", "-c", "echo task output"] \ No newline at end of file