feat: track file() usage in Terragrunt change detection.

Signed-off-by: i4k <[email protected]>

go.mod

@@ -194,7 +194,7 @@ require (
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-uuid v1.0.3
-	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mitchellh/go-homedir v1.1.0
 	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
 	github.com/rs/zerolog v1.28.0
 	github.com/zclconf/go-cty-yaml v1.0.3 // indirect

stack/manager_test.go

@@ -193,6 +193,17 @@ func TestListChangedStacks(t *testing.T) {
 				changed: []string{"/tg-stack"},
 			},
 		},
+		{
+			name: "opening files to read mark stack as changed",
+			repobuilder: func(t *testing.T) repository {
+				t.Helper()
+				return singleTerragruntStackWithDependentOnFileChangedRepo(t, "force")
+			},
+			want: listTestResult{
+				list:    []string{"/stack1", "/stack2"},
+				changed: []string{"/stack1"},
+			},
+		},
 		{
 			name: "single Terragrunt stack with single local Terraform module changed referenced with abspath",
 			repobuilder: func(t *testing.T) repository {
@@ -776,6 +787,65 @@ func singleTerragruntStackWithSingleTerraformModuleChangedRepo(t *testing.T, ena
 	return repo
 }
 
+func singleTerragruntStackWithDependentOnFileChangedRepo(t *testing.T, enabledOption string) repository {
+	repo := singleMergeCommitRepoNoStack(t)
+	test.WriteFile(t, repo.Dir, "terramate.tm.hcl", Doc(
+		Block("terramate",
+			Block("config",
+				Block("change_detection",
+					Block("terragrunt",
+						Str("enabled", enabledOption),
+					),
+				),
+			),
+		),
+	).String())
+
+	test.WriteFile(t, repo.Dir, "hello.txt", "good world")
+
+	stack1 := test.Mkdir(t, repo.Dir, "stack1")
+	stack2 := test.Mkdir(t, repo.Dir, "stack2")
+
+	modules := test.Mkdir(t, repo.Dir, "modules")
+	module1 := test.Mkdir(t, modules, "module1")
+
+	repo.modules = append(repo.modules, module1)
+
+	root, err := config.LoadRoot(repo.Dir)
+	assert.NoError(t, err)
+
+	createStack(t, root, stack1)
+	createStack(t, root, stack2)
+
+	test.WriteFile(t, stack1, "terragrunt.hcl", Doc(
+		Block("terraform",
+			Str("source", "../modules/module1"),
+		),
+		Block("locals",
+			Expr("hello", `file("../hello.txt")`),
+		),
+	).String())
+
+	test.WriteFile(t, module1, "main.tf", `# empty file`)
+
+	g := test.NewGitWrapper(t, repo.Dir, []string{})
+	assert.NoError(t, g.Checkout("testbranch", true), "create branch failed")
+
+	assert.NoError(t, g.Add(repo.Dir), "add files")
+	assert.NoError(t, g.Commit("files"), "commit files")
+
+	addMergeCommit(t, repo.Dir, "testbranch")
+	assert.NoError(t, g.DeleteBranch("testbranch"), "delete testbranch")
+
+	// now we branch again and modify the hello.txt
+	assert.NoError(t, g.Checkout("testbranch2", true), "create branch testbranch2 failed")
+
+	test.WriteFile(t, repo.Dir, "hello.txt", `evil world`)
+	assert.NoError(t, g.Add(repo.Dir), "add files")
+	assert.NoError(t, g.Commit("hello changed"), "commit files")
+	return repo
+}
+
 func singleTerragruntStackWithSingleTerraformModuleChangedFromAbsPathSourceRepo(t *testing.T, enabledOption string) repository {
 	repo := singleMergeCommitRepoNoStack(t)
 	test.WriteFile(t, repo.Dir, "terramate.tm.hcl", Doc(

tg/funcs.go

@@ -23,22 +23,25 @@ package tg
 import (
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
+	"unicode/utf8"
 
 	"github.com/gruntwork-io/go-commons/errors"
 	tgconfig "github.com/gruntwork-io/terragrunt/config"
 	"github.com/gruntwork-io/terragrunt/util"
+	"github.com/mitchellh/go-homedir"
 	"github.com/terramate-io/terramate/project"
 	"github.com/zclconf/go-cty/cty"
 	"github.com/zclconf/go-cty/cty/function"
 )
 
 type tgFunction func(ctx *tgconfig.ParsingContext, rootdir string, mod *Module, args []string) (string, error)
 
-// findInParentFoldersFunc implements the Terragrunt `find_in_parent_folders` function.
-func findInParentFoldersFunc(pctx *tgconfig.ParsingContext, rootdir string, mod *Module) function.Function {
+// tgFindInParentFoldersFuncImpl implements the Terragrunt `find_in_parent_folders` function.
+func tgFindInParentFoldersFuncImpl(pctx *tgconfig.ParsingContext, rootdir string, mod *Module) function.Function {
 	return wrapStringSliceToStringAsFuncImpl(pctx, rootdir, mod, findInParentFoldersImpl)
 }
 
@@ -113,8 +116,8 @@ func findInParentFoldersImpl(
 	}
 }
 
-// readTerragruntConfigFunc implements the Terragrunt `read_terragrunt_config` function.
-func readTerragruntConfigFunc(ctx *tgconfig.ParsingContext, rootdir string, mod *Module) function.Function {
+// tgReadTerragruntConfigFuncImpl implements the Terragrunt `read_terragrunt_config` function.
+func tgReadTerragruntConfigFuncImpl(ctx *tgconfig.ParsingContext, rootdir string, mod *Module) function.Function {
 	return function.New(&function.Spec{
 		// Takes one required string param
 		Params: []function.Parameter{
@@ -173,8 +176,8 @@ func readTerragruntConfigImpl(ctx *tgconfig.ParsingContext, configPath string, d
 	return tgconfig.TerragruntConfigAsCty(cfg)
 }
 
-// readTFVarsFile reads a *.tfvars or *.tfvars.json file and returns the contents as a JSON encoded string
-func readTFVarsFile(ctx *tgconfig.ParsingContext, rootdir string, mod *Module, args []string) (string, error) {
+// tgReadTFVarsFileFuncImpl reads a *.tfvars or *.tfvars.json file and returns the contents as a JSON encoded string
+func tgReadTFVarsFileFuncImpl(ctx *tgconfig.ParsingContext, rootdir string, mod *Module, args []string) (string, error) {
 	if len(args) != 1 {
 		return "", errors.WithStackTrace(tgconfig.WrongNumberOfParamsError{Func: "read_tfvars_file", Expected: "1", Actual: len(args)})
 	}
@@ -218,6 +221,62 @@ func readTFVarsFile(ctx *tgconfig.ParsingContext, rootdir string, mod *Module, a
 	return string(data), nil
 }
 
+func tgFileFuncImpl(_ *tgconfig.ParsingContext, rootdir string, mod *Module) function.Function {
+	return function.New(&function.Spec{
+		Params: []function.Parameter{
+			{
+				Name: "path",
+				Type: cty.String,
+			},
+		},
+		Type: function.StaticReturnType(cty.String),
+		Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) {
+			basedir := filepath.Join(rootdir, filepath.FromSlash(mod.Path.String()))
+			path := args[0].AsString()
+			if !filepath.IsAbs(path) {
+				path = filepath.Join(basedir, path)
+			}
+			mod.DependsOn = append(mod.DependsOn, project.PrjAbsPath(rootdir, path))
+			src, err := readFileBytes(basedir, path)
+			if err != nil {
+				err = function.NewArgError(0, err)
+				return cty.UnknownVal(cty.String), err
+			}
+
+			if !utf8.Valid(src) {
+				return cty.UnknownVal(cty.String), fmt.Errorf("contents of %s are not valid UTF-8; use the filebase64 function to obtain the Base64 encoded contents or the other file functions (e.g. filemd5, filesha256) to obtain file hashing results instead", path)
+			}
+			return cty.StringVal(string(src)), nil
+		},
+	})
+}
+
+func readFileBytes(baseDir, path string) ([]byte, error) {
+	path, err := homedir.Expand(path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to expand ~: %s", err)
+	}
+
+	if !filepath.IsAbs(path) {
+		path = filepath.Join(baseDir, path)
+	}
+
+	// Ensure that the path is canonical for the host OS
+	path = filepath.Clean(path)
+
+	src, err := ioutil.ReadFile(path)
+	if err != nil {
+		// ReadFile does not return Terraform-user-friendly error
+		// messages, so we'll provide our own.
+		if os.IsNotExist(err) {
+			return nil, fmt.Errorf("no file exists at %s; this function works only with files that are distributed as part of the configuration source code, so if this file will be created by a resource in this configuration you must instead obtain this result from an attribute of that resource", path)
+		}
+		return nil, fmt.Errorf("failed to read %s", path)
+	}
+
+	return src, nil
+}
+
 // getCleanedTargetConfigPath returns a cleaned path to the target config (the `terragrunt.hcl` or
 // `terragrunt.hcl.json` file), handling relative paths correctly. This will automatically append
 // `terragrunt.hcl` or `terragrunt.hcl.json` to the path if the target path is a directory.

tg/tg_module.go

@@ -98,9 +98,12 @@ func ScanModules(rootdir string, dir project.Path, trackDependencies bool) (Modu
 
 		// Override the predefined functions to intercept the function calls that process paths.
 		pctx.PredefinedFunctions = make(map[string]function.Function)
-		pctx.PredefinedFunctions[config.FuncNameFindInParentFolders] = findInParentFoldersFunc(pctx, rootdir, mod)
-		pctx.PredefinedFunctions[config.FuncNameReadTerragruntConfig] = readTerragruntConfigFunc(pctx, rootdir, mod)
-		pctx.PredefinedFunctions[config.FuncNameReadTfvarsFile] = wrapStringSliceToStringAsFuncImpl(pctx, rootdir, mod, readTFVarsFile)
+		pctx.PredefinedFunctions[config.FuncNameFindInParentFolders] = tgFindInParentFoldersFuncImpl(pctx, rootdir, mod)
+		pctx.PredefinedFunctions[config.FuncNameReadTerragruntConfig] = tgReadTerragruntConfigFuncImpl(pctx, rootdir, mod)
+		pctx.PredefinedFunctions[config.FuncNameReadTfvarsFile] = wrapStringSliceToStringAsFuncImpl(pctx, rootdir, mod, tgReadTFVarsFileFuncImpl)
+
+		// override Terraform function
+		pctx.PredefinedFunctions["file"] = tgFileFuncImpl(pctx, rootdir, mod)
 
 		// Here we parse the Terragrunt file which calls into our overrided functions.
 		// After this returns, the module's DependsOn will be populated.

tg/tg_module_test.go

@@ -450,6 +450,57 @@ func TestTerragruntScanModules(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "module reading file",
+			layout: []string{
+				`f:abc/terragrunt.hcl:` + Doc(
+					Block("terraform",
+						Str("source", "https://some.etc/prj"),
+					),
+					Block("locals",
+						Expr("hello", `file("../hello.txt")`),
+					),
+				).String(),
+				`f:hello.txt:world`,
+			},
+			want: want{
+				modules: tg.Modules{
+					{
+						Path:       project.NewPath("/abc"),
+						Source:     "https://some.etc/prj",
+						ConfigFile: project.NewPath("/abc/terragrunt.hcl"),
+						DependsOn: project.Paths{
+							project.NewPath("/hello.txt"),
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "module reading non-existent file",
+			layout: []string{
+				`f:abc/terragrunt.hcl:` + Doc(
+					Block("terraform",
+						Str("source", "https://some.etc/prj"),
+					),
+					Block("locals",
+						Expr("hello", `try(file("../hello.txt"), "whatever")`),
+					),
+				).String(),
+			},
+			want: want{
+				modules: tg.Modules{
+					{
+						Path:       project.NewPath("/abc"),
+						Source:     "https://some.etc/prj",
+						ConfigFile: project.NewPath("/abc/terragrunt.hcl"),
+						DependsOn: project.Paths{
+							project.NewPath("/hello.txt"),
+						},
+					},
+				},
+			},
+		},
 		{
 			name: "local module directory also tracked as dependency",
 			layout: []string{