From 6eea05bb485431d29045f2aa7d7c356f85fe4782 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Thu, 13 May 2021 17:14:19 -0700 Subject: [PATCH] lang: String templates as values This is a design prototype of one possible way to meet the need of defining a template in a different location from where it will ultimately be evaluated. For example, a generic module might accept a template as an input variable and then evaluate that template using values that are private to the module, thus separating the concern of providing the data from the concern of assembling the data into a particular string shape. This is only just enough to try out the behavior and see how it feels and how intuitive it seems. If we were to move forward with something like this then a real implementation would need some extra care to some other concerns, including but not limited to: - Generating good error messages when templates don't make valid use of their arguments. - Properly blocking template values from being used in places where they don't make sense, such as as input to jsonencode(...), or as a root module output value. - Possibly a function similar to templatefile which can create a template value from a file on disk, to allow factoring out larger templates while still being compatible with modules that expect template values. --- internal/typeexpr/get_type.go | 48 ++++++++++ internal/typeexpr/public.go | 35 ++++++++ lang/functions.go | 3 + lang/functions_test.go | 14 +++ lang/references.go | 10 +++ lang/templatevals/doc.go | 5 ++ lang/templatevals/funcs.go | 163 ++++++++++++++++++++++++++++++++++ lang/templatevals/types.go | 160 +++++++++++++++++++++++++++++++++ 8 files changed, 438 insertions(+) create mode 100644 lang/templatevals/doc.go create mode 100644 lang/templatevals/funcs.go create mode 100644 lang/templatevals/types.go diff --git a/internal/typeexpr/get_type.go b/internal/typeexpr/get_type.go index de5465b997ba..9e8838188579 100644 --- a/internal/typeexpr/get_type.go +++ b/internal/typeexpr/get_type.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/terraform/lang/templatevals" "github.com/zclconf/go-cty/cty" ) @@ -46,6 +47,13 @@ func getType(expr hcl.Expression, constraint bool) (cty.Type, hcl.Diagnostics) { Detail: "The object type constructor requires one argument specifying the attribute types and values as a map.", Subject: expr.Range().Ptr(), }} + case "template": + return cty.DynamicPseudoType, hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: invalidTypeSummary, + Detail: "The template type constructor requires one argument specifying the argument names and types as a map.", + Subject: expr.Range().Ptr(), + }} case "tuple": return cty.DynamicPseudoType, hcl.Diagnostics{{ Severity: hcl.DiagError, @@ -113,6 +121,14 @@ func getType(expr hcl.Expression, constraint bool) (cty.Type, hcl.Diagnostics) { Subject: &subjectRange, Context: &contextRange, }} + case "template": + return cty.DynamicPseudoType, hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: invalidTypeSummary, + Detail: "The template type constructor requires one argument specifying the argument names and types as a map.", + Subject: &subjectRange, + Context: &contextRange, + }} case "tuple": return cty.DynamicPseudoType, hcl.Diagnostics{{ Severity: hcl.DiagError, @@ -212,6 +228,38 @@ func getType(expr hcl.Expression, constraint bool) (cty.Type, hcl.Diagnostics) { // minor versions of cty. We're accepting that because Terraform // itself is considering optional attributes as experimental right now. return cty.ObjectWithOptionalAttrs(atys, optAttrs), diags + case "template": + attrDefs, diags := hcl.ExprMap(call.Arguments[0]) + if diags.HasErrors() { + return cty.DynamicPseudoType, hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: invalidTypeSummary, + Detail: "Template type constructor requires a map whose keys are argument names and whose values are the corresponding argument types.", + Subject: call.Arguments[0].Range().Ptr(), + Context: expr.Range().Ptr(), + }} + } + + atys := make(map[string]cty.Type) + for _, attrDef := range attrDefs { + attrName := hcl.ExprAsKeyword(attrDef.Key) + if attrName == "" { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: invalidTypeSummary, + Detail: "Template type constructor map keys must be argument names.", + Subject: attrDef.Key.Range().Ptr(), + Context: expr.Range().Ptr(), + }) + continue + } + atyExpr := attrDef.Value + + aty, attrDiags := getType(atyExpr, constraint) + diags = append(diags, attrDiags...) + atys[attrName] = aty + } + return templatevals.Type(atys), diags case "tuple": elemDefs, diags := hcl.ExprList(call.Arguments[0]) if diags.HasErrors() { diff --git a/internal/typeexpr/public.go b/internal/typeexpr/public.go index 3b8f618fbcd1..b6e8169d1769 100644 --- a/internal/typeexpr/public.go +++ b/internal/typeexpr/public.go @@ -6,6 +6,7 @@ import ( "sort" "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/hashicorp/terraform/lang/templatevals" "github.com/hashicorp/hcl/v2" "github.com/zclconf/go-cty/cty" @@ -124,6 +125,40 @@ func TypeString(ty cty.Type) string { return buf.String() } + if templatevals.IsTemplateType(ty) { + var buf bytes.Buffer + buf.WriteString("template({") + atys := templatevals.TypeArgs(ty) + names := make([]string, 0, len(atys)) + for name := range atys { + names = append(names, name) + } + sort.Strings(names) + first := true + for _, name := range names { + aty := atys[name] + if !first { + buf.WriteByte(',') + } + if !hclsyntax.ValidIdentifier(name) { + // Should never happen for any type produced by this package, + // but we'll do something reasonable here just so we don't + // produce garbage if someone gives us a hand-assembled object + // type that has weird attribute names. + // Using Go-style quoting here isn't perfect, since it doesn't + // exactly match HCL syntax, but it's fine for an edge-case. + buf.WriteString(fmt.Sprintf("%q", name)) + } else { + buf.WriteString(name) + } + buf.WriteByte('=') + buf.WriteString(TypeString(aty)) + first = false + } + buf.WriteString("})") + return buf.String() + } + // Should never happen because we covered all cases above. panic(fmt.Errorf("unsupported type %#v", ty)) } diff --git a/lang/functions.go b/lang/functions.go index 21be9a7f1013..619d54850cc2 100644 --- a/lang/functions.go +++ b/lang/functions.go @@ -11,6 +11,7 @@ import ( "github.com/hashicorp/terraform/experiments" "github.com/hashicorp/terraform/lang/funcs" + "github.com/hashicorp/terraform/lang/templatevals" ) var impureFunctions = []string{ @@ -60,6 +61,7 @@ func (s *Scope) Functions() map[string]function.Function { "dirname": funcs.DirnameFunc, "distinct": stdlib.DistinctFunc, "element": stdlib.ElementFunc, + "evaltemplate": templatevals.EvalTemplateFunc, "chunklist": stdlib.ChunklistFunc, "file": funcs.MakeFileFunc(s.BaseDir, false), "fileexists": funcs.MakeFileExistsFunc(s.BaseDir), @@ -87,6 +89,7 @@ func (s *Scope) Functions() map[string]function.Function { "log": stdlib.LogFunc, "lookup": funcs.LookupFunc, "lower": stdlib.LowerFunc, + "maketemplate": templatevals.MakeTemplateFunc, "map": funcs.MapFunc, "matchkeys": funcs.MatchkeysFunc, "max": stdlib.MaxFunc, diff --git a/lang/functions_test.go b/lang/functions_test.go index c33bc3672a10..fd9b7736e051 100644 --- a/lang/functions_test.go +++ b/lang/functions_test.go @@ -325,6 +325,13 @@ func TestFunctions(t *testing.T) { }, }, + "evaltemplate": { + { + `evaltemplate(maketemplate("Hello ${template.a}!"), {a = "world"})`, + cty.StringVal("Hello world!"), + }, + }, + "file": { { `file("hello.txt")`, @@ -550,6 +557,13 @@ func TestFunctions(t *testing.T) { }, }, + "maketemplate": { + { + `evaltemplate(maketemplate("Hello ${template.a}!"), {a = "world"})`, + cty.StringVal("Hello world!"), + }, + }, + "map": { // There are intentionally no test cases for "map" because // it is a stub that always returns an error. diff --git a/lang/references.go b/lang/references.go index 569251cb8dfb..65e09b2e3444 100644 --- a/lang/references.go +++ b/lang/references.go @@ -30,6 +30,16 @@ func References(traversals []hcl.Traversal) ([]*addrs.Reference, tfdiags.Diagnos refs := make([]*addrs.Reference, 0, len(traversals)) for _, traversal := range traversals { + if traversal.RootName() == "template" { + // "template" traversals don't create references. Instead, they + // are just part of the template values syntax within the + // special "maketemplate" function. We'll ignore references + // here to allow maketemplate calls to work, although any use + // of these outside of maketemplate will ultimately fail during + // evaluation, because we don't define this symbol there. + continue + } + ref, refDiags := addrs.ParseRef(traversal) diags = diags.Append(refDiags) if ref == nil { diff --git a/lang/templatevals/doc.go b/lang/templatevals/doc.go new file mode 100644 index 000000000000..080a0949a7b5 --- /dev/null +++ b/lang/templatevals/doc.go @@ -0,0 +1,5 @@ +// Package templatevals deals with the idea of "template values" in the +// Terraform language, which allow passing around not-yet-evaluated string +// templates in a structured way that allows for type checking and avoids +// confusing additional template escaping. +package templatevals diff --git a/lang/templatevals/funcs.go b/lang/templatevals/funcs.go new file mode 100644 index 000000000000..5ab9a2778e54 --- /dev/null +++ b/lang/templatevals/funcs.go @@ -0,0 +1,163 @@ +package templatevals + +import ( + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/ext/customdecode" + "github.com/hashicorp/hcl/v2/hclsyntax" + + "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-cty/cty/convert" + "github.com/zclconf/go-cty/cty/function" +) + +var MakeTemplateFunc = function.New(&function.Spec{ + Params: []function.Parameter{ + { + Name: "source", + Type: customdecode.ExpressionClosureType, + }, + }, + Type: func(args []cty.Value) (cty.Type, error) { + // Because our parameter is constrained with the special + // customdecode.ExpressionClosureType constraint, it's + // guaranteed to be a known value of a capsule type + // wrapping customdecode.ExpressionClosure. + closure := args[0].EncapsulatedValue().(*customdecode.ExpressionClosure) + expr := closure.Expression + parentCtx := closure.EvalContext + + // Although in principle this lazy evaluation mechanism could + // apply to any sort of expression, we intentionally constrain + // it only to template expressions here to reinforce that this + // is not intended as a general lambda function mechanism. + switch expr.(type) { + case *hclsyntax.TemplateExpr, *hclsyntax.TemplateWrapExpr: + // ok + default: + return cty.DynamicPseudoType, function.NewArgErrorf(0, "must be a string template expression") + } + + // Our initial template type has arguments derived from the references + // that have traversals starting with "template". + atys := make(map[string]cty.Type) + for _, traversal := range expr.Variables() { + if traversal.RootName() != "template" { + // We don't care about any other traversals + continue + } + var step1 hcl.TraverseAttr + if len(traversal) >= 2 { + if ta, ok := traversal[1].(hcl.TraverseAttr); ok { + step1 = ta + } + } + name := step1.Name + if name == "" { // The conditions above didn't match, then + return cty.DynamicPseudoType, function.NewArgErrorf(0, "template argument reference at %s must include an attribute lookup representing the argument name", traversal.SourceRange()) + } + // All of our arguments start off with unconstrained types because + // we can't walk backwards from an expression to all of the types + // that could succeed with it. However, the type conversion + // behavior for template values includes a more specific type check + // if the destination type has more constrained arguments. + atys[name] = cty.DynamicPseudoType + } + + // Before we return we'll check to make sure the expression is + // evaluable _at all_ (even before we know the argument values) + // because that'll help users catch totally-invalid templates early, + // even before they try to pass them to another module to be evaluated. + ctx := parentCtx.NewChild() + ctx.Variables = map[string]cty.Value{ + "template": cty.UnknownVal(cty.Object(atys)), + } + v, diags := expr.Value(ctx) + if diags.HasErrors() { + // It would be nice to have a way to report these diags + // directly out to HCL, but unfortunately we're sending them + // out through cty and it doesn't understand HCL diagnostics. + return cty.DynamicPseudoType, function.NewArgErrorf(0, "invalid template: %s", diags.Error()) + } + if _, err := convert.Convert(v, cty.String); err != nil { + // We'll catch this early where possible. It won't always be + // possible, because the return type might vary depending on + // the input, so we must re-check this in evaltemplate too. + return cty.DynamicPseudoType, function.NewArgErrorf(0, "invalid template: must produce a string result") + } + + // If all of the above was successful then this template seems valid + // and we can determine which type we're returning. (The actual + // _value_ of that type will come in Impl.) + return Type(atys), nil + }, + Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { + // We already did all of our checking inside Type, so our only remaining + // job now is to wrap the expression closure up inside a value of + // our capsule type. + closure := args[0].EncapsulatedValue().(*customdecode.ExpressionClosure) + tv := &templateVal{ + expr: closure.Expression, + ctx: closure.EvalContext, + } + return cty.CapsuleVal(retType, tv), nil + }, +}) + +// TODO: Consider also a "templatefromfile" function that compiles a separate +// file as a template in a similar way that "templatefile" does, but which +// returns a template value rather than immediately evaluating the template. +// This would then be more convenient for situations where the expected +// template is quite large in itself and thus worth factoring out into a +// separate file. + +var EvalTemplateFunc = function.New(&function.Spec{ + Params: []function.Parameter{ + { + Name: "template", + // We need to type-check this dynamically because there is an + // infinite number of possible template types. + Type: cty.DynamicPseudoType, + }, + { + Name: "args", + // We also need to type-check _this_ dynamically, because + // we expect an object type whose attributes depend on the + // template type. + Type: cty.DynamicPseudoType, + AllowMarked: true, + }, + }, + Type: function.StaticReturnType(cty.String), + Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { + template := args[0] + rawArgsObj, retMarks := args[1].Unmark() + + tv := template.EncapsulatedValue().(*templateVal) + atys := TypeArgs(template.Type()) + + // The given arguments object must be compatible with the expected + // argument types. This'll catch if the call lacks any arguments + // that the template requires, or if any of them are of an unsuitable + // type. + argsObj, err := convert.Convert(rawArgsObj, cty.Object(atys)) + if err != nil { + return cty.NilVal, function.NewArgError(1, err) + } + + ctx := tv.ctx.NewChild() + ctx.Variables = map[string]cty.Value{ + "template": argsObj, + } + v, diags := tv.expr.Value(ctx) + if diags.HasErrors() { + return cty.NilVal, function.NewArgErrorf(0, "incompatible template: %s", diags.Error()) + } + + v, err = convert.Convert(v, retType) + if err != nil { + return cty.NilVal, function.NewArgErrorf(0, "template must produce a string result") + } + + return v.WithMarks(retMarks), nil + }, +}) diff --git a/lang/templatevals/types.go b/lang/templatevals/types.go new file mode 100644 index 000000000000..b0a7f729a5b6 --- /dev/null +++ b/lang/templatevals/types.go @@ -0,0 +1,160 @@ +package templatevals + +import ( + "fmt" + "reflect" + + "github.com/hashicorp/hcl/v2" + "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-cty/cty/convert" +) + +// Type constructs a cty type representing a lazily-evaluated template. +// +// Template types are parameterized by their set of required argument names and +// associated type constraints. +func Type(atys map[string]cty.Type) cty.Type { + var ret cty.Type + ops := &cty.CapsuleOps{ + TypeGoString: func(goTy reflect.Type) string { + return fmt.Sprintf("templatevals.Type(%#v)", atys) + }, + GoString: func(rv interface{}) string { + tv := rv.(*templateVal) + return fmt.Sprintf("templatevals.Val(%#v, %#v, %#v)", atys, tv.expr, tv.ctx) + }, + ConversionFrom: func(dest cty.Type) func(interface{}, cty.Path) (cty.Value, error) { + if !IsTemplateType(dest) { + // Can only convert between template types + return nil + } + + // There are some other constraints on successful conversion but + // we'll wait until inside the conversion function to deal with + // those, so we can return specialized errors. + dstAtys := TypeArgs(dest) + return func(rv interface{}, path cty.Path) (cty.Value, error) { + // Conversion is allowed only if the destination arguments are + // all assignable to the source arguments, thus making the + // result potentially _more_ constrained in what arguments + // the template can accept. To check that we make some + // temporary object types to borrow the object type conversion + // behavior. + srcObjTy := cty.Object(atys) + dstObjTy := cty.Object(dstAtys) + if srcObjTy.Equals(dstObjTy) { + // Easy case then: the two types are equivalent. + } else if conv := convert.GetConversionUnsafe(dstObjTy, srcObjTy); conv != nil { + // Also valid, from an interface-conformance perspective + // (note that dst and src are intentionally inverted above + // because we're effectively testing if an object of the + // destination type (the final template arguments) would be + // assignable to the source type (the arguments that the + // template actually expects.) + } else { + // TODO: A better error message, saying something about + // what is wrong. + return cty.NilVal, path.NewErrorf("incompatible template arguments") + } + + // Even if the static type information suggests compatibility, + // our template argument constraints start off very broad + // at the point of definition (everything is "any") and + // constraining further requires that the template can pass + // type checking when given arguments of the destination + // types. + tv := rv.(*templateVal) + expr := tv.expr + parentCtx := tv.ctx + ctx := parentCtx.NewChild() + ctx.Variables = map[string]cty.Value{ + "template": cty.UnknownVal(dstObjTy), + } + _, diags := expr.Value(ctx) + if diags.HasErrors() { + // TODO: Again, a better error message. Doing better here + // probably in practice means trying more surgical type + // checks with only one argument at a time set to a + // specific type constraint, to see which ones fail and + // which ones succeed. Although even that wouldn't be + // 100% because it might be the combination of two + // arguments that makes it invalid! + return cty.NilVal, path.NewErrorf("incompatible usage of template arguments") + } + + // If we get down here without returning an error then this + // conversion seems acceptable from a type-checking standpoint, + // and so we can wrap our same templateVal value up in the + // destination type. + // Note that the resulting template might still fail for + // dynamic reasons, e.g. if it's expecting valid JSON but + // doesn't _get_ valid JSON, but we'll catch that sort of + // problem at evaluation time. + return cty.CapsuleVal(dest, tv), nil + } + }, + ExtensionData: func(key interface{}) interface{} { + switch key { + case templateTypeAtys: + return atys + default: + return nil + } + }, + } + ret = cty.CapsuleWithOps("template", templateValReflect, ops) + return ret +} + +// Val constructs a new value of a template type with the given expression and +// evaluation context. +// +// The given type must be a template type, or this function will panic. +// +// This function does no validation of whether the given expression and context +// are compatible with one another or whether the the expression can support +// the given argument types. The caller must guarantee such compatibility. +func Val(ty cty.Type, expr hcl.Expression, ctx *hcl.EvalContext) cty.Value { + if !IsTemplateType(ty) { + panic(fmt.Sprintf("can't construct template value of non-template type %#v", ty)) + } + rv := &templateVal{ + expr: expr, + ctx: ctx, + } + return cty.CapsuleVal(ty, rv) +} + +func IsTemplateType(ty cty.Type) bool { + if !ty.IsCapsuleType() { + return false + } + return ty.EncapsulatedType() == templateValReflect +} + +// TypeArgs returns the arguments and their associated types for the given +// type, which must be a template type or this function will panic. +// +// Do not modify the returned array. It is part of the internal state of +// the template type. +func TypeArgs(ty cty.Type) map[string]cty.Type { + if !IsTemplateType(ty) { + panic("templatevals.TypeArgs on non-template type") + } + return ty.CapsuleExtensionData(templateTypeAtys).(map[string]cty.Type) +} + +func IsTemplateVal(v cty.Value) bool { + return IsTemplateType(v.Type()) +} + +type templateVal struct { + expr hcl.Expression + ctx *hcl.EvalContext +} + +var templateValReflect = reflect.TypeOf(templateVal{}) + +type templateTypeAtysKey int + +var templateTypeAtys templateTypeAtysKey = 0