diff --git a/.gitignore b/.gitignore index 11d25e00f..7e95ba7fc 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ .terraform terraform.tfstate terraform.tfvars +terraform.tfvars.json *.tfstate* .terragrunt .terragrunt-cache diff --git a/go.mod b/go.mod index 731224d57..e5069e92b 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a github.com/jstemmer/go-junit-report v0.9.1 github.com/magiconair/properties v1.8.0 - github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326 // indirect + github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326 github.com/miekg/dns v1.1.31 github.com/mitchellh/go-homedir v1.1.0 github.com/oracle/oci-go-sdk v7.1.0+incompatible diff --git a/go.sum b/go.sum index 43ca0c598..fd815b4dc 100644 --- a/go.sum +++ b/go.sum @@ -265,8 +265,6 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl/v2 v2.8.2 h1:wmFle3D1vu0okesm8BTLVDyJ6/OL9DCLUwn0b2OptiY= github.com/hashicorp/hcl/v2 v2.8.2/go.mod h1:bQTN5mpo+jewjJgh8jr0JUguIi7qPHUF6yIfAEN3jqY= -github.com/hashicorp/terraform-json v0.9.0 h1:WE7+Wt93W93feOiCligElSyS0tlDzwZUtJuDGIBr8zg= -github.com/hashicorp/terraform-json v0.9.0/go.mod h1:3defM4kkMfttwiE7VakJDwCd4R+umhSQnvJwORXbprE= github.com/hashicorp/terraform-json v0.12.0 h1:8czPgEEWWPROStjkWPUnTQDXmpmZPlkQAwYYLETaTvw= github.com/hashicorp/terraform-json v0.12.0/go.mod h1:pmbq9o4EuL43db5+0ogX10Yofv1nozM+wskr/bGFJpI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -281,6 +279,7 @@ github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5i github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= diff --git a/modules/files/files.go b/modules/files/files.go index 2c1b6169d..239f15f81 100644 --- a/modules/files/files.go +++ b/modules/files/files.go @@ -150,10 +150,10 @@ func CopyFolderContentsWithFilter(source string, destination string, filter func return nil } -// PathContainsTerraformStateOrVars returns true if the path corresponds to a Terraform state file or .tfvars file. +// PathContainsTerraformStateOrVars returns true if the path corresponds to a Terraform state file or .tfvars/.tfvars.json file. func PathContainsTerraformStateOrVars(path string) bool { filename := filepath.Base(path) - return filename == "terraform.tfstate" || filename == "terraform.tfstate.backup" || filename == "terraform.tfvars" + return filename == "terraform.tfstate" || filename == "terraform.tfstate.backup" || filename == "terraform.tfvars" || filename == "terraform.tfvars.json" } // PathContainsTerraformState returns true if the path corresponds to a Terraform state file. diff --git a/modules/files/files_test.go b/modules/files/files_test.go index c859dedfc..520567479 100644 --- a/modules/files/files_test.go +++ b/modules/files/files_test.go @@ -173,6 +173,34 @@ func TestCopyTerragruntFolderToTemp(t *testing.T) { requireDirectoriesEqual(t, expectedDir, tmpDir) } +func TestPathContainsTerraformStateOrVars(t *testing.T) { + var data = []struct { + desc string + path string + contains bool + }{ + {"contains tfvars", "./folder/terraform.tfvars", true}, + {"contains tfvars.json", "./folder/hello/terraform.tfvars.json", true}, + {"contains state", "./folder/hello/helloagain/terraform.tfstate", true}, + {"contains state backup", "./folder/hey/terraform.tfstate.backup", true}, + {"does not contain any", "./folder/salut/terraform.json", false}, + } + + for _, tt := range data { + tt := tt + t.Run(tt.desc, func(t *testing.T) { + result := PathContainsTerraformStateOrVars(tt.path) + if result != tt.contains { + if tt.contains { + t.Errorf("Expected %s to contain Terraform related file", tt.path) + } else { + t.Errorf("Expected %s to not contain Terraform related file", tt.path) + } + } + }) + } +} + // Diffing two directories to ensure they have the exact same files, contents, etc and showing exactly what's different // takes a lot of code. Why waste time on that when this functionality is already nicely implemented in the Unix/Linux // "diff" command? We shell out to that command at test time. diff --git a/modules/terraform/format.go b/modules/terraform/format.go index 4edd1b16d..f50072d7c 100644 --- a/modules/terraform/format.go +++ b/modules/terraform/format.go @@ -119,8 +119,8 @@ func FormatTerraformPluginDirAsArgs(pluginDir string) []string { return pluginArgs } -// FormatTerraformArgs will format multiple args with the arg name (e.g. "-var-file", []string{"foo.tfvars", "bar.tfvars"}) -// returns "-var-file foo.tfvars -var-file bar.tfvars" +// FormatTerraformArgs will format multiple args with the arg name (e.g. "-var-file", []string{"foo.tfvars", "bar.tfvars", "baz.tfvars.json"}) +// returns "-var-file foo.tfvars -var-file bar.tfvars -var-file baz.tfvars.json" func FormatTerraformArgs(argName string, args []string) []string { argsList := []string{} for _, argValue := range args { diff --git a/modules/terraform/var-file.go b/modules/terraform/var-file.go index a81ec00a5..09f3a2d3b 100644 --- a/modules/terraform/var-file.go +++ b/modules/terraform/var-file.go @@ -5,9 +5,11 @@ import ( "fmt" "io/ioutil" "reflect" + "strings" "github.com/gruntwork-io/terratest/modules/testing" + "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hclparse" "github.com/stretchr/testify/require" "github.com/zclconf/go-cty/cty" @@ -133,9 +135,9 @@ func GetAllVariablesFromVarFileE(t testing.TestingT, fileName string, out interf return parseAndDecodeVarFile(string(fileContents), fileName, out) } -// parseAndDecodeVarFile uses the HCL2 parser to parse the given varfile string into an HCL file body, and then decode it +// parseAndDecodeVarFile uses the HCL2 parser to parse the given varfile string into an HCL or HCL JSON file body, and then decode it // into a map that maps var names to values. -func parseAndDecodeVarFile(hclContents string, filename string, out interface{}) (err error) { +func parseAndDecodeVarFile(fileContents string, filename string, out interface{}) (err error) { // The HCL2 parser and especially cty conversions will panic in many types of errors, so we have to recover from // those panics here and convert them to normal errors defer func() { @@ -146,7 +148,16 @@ func parseAndDecodeVarFile(hclContents string, filename string, out interface{}) parser := hclparse.NewParser() - file, parseDiagnostics := parser.ParseHCL([]byte(hclContents), filename) + var file *hcl.File + var parseDiagnostics hcl.Diagnostics + + // determine if a JSON variables file is submitted and parse accordingly + if strings.HasSuffix(filename, ".json") { + file, parseDiagnostics = parser.ParseJSON([]byte(fileContents), filename) + } else { + file, parseDiagnostics = parser.ParseHCL([]byte(fileContents), filename) + } + if parseDiagnostics != nil && parseDiagnostics.HasErrors() { return parseDiagnostics } diff --git a/modules/terraform/var-file_test.go b/modules/terraform/var-file_test.go index b5c0aa76d..3fca42147 100644 --- a/modules/terraform/var-file_test.go +++ b/modules/terraform/var-file_test.go @@ -228,7 +228,243 @@ func TestGetAllVariablesFromVarFileStructOut(t *testing.T) { err := GetAllVariablesFromVarFileE(t, randomFileName, ®ion) require.NoError(t, err) require.Equal(t, "us-east-2", region.AwsRegion) +} + +func TestGetVariablesFromVarFilesAsStringJSON(t *testing.T) { + randomFileName := fmt.Sprintf("./%s.tfvars.json", random.UniqueId()) + + testJSON := []byte(` + { + "aws_region": "us-east-2", + "aws_account_id": "111111111111", + "number_type": 2, + "boolean_type": true, + "tags": { + "foo": "bar" + }, + "list": ["item1"] + }`) + + WriteFile(t, randomFileName, testJSON) + defer os.Remove(randomFileName) + + stringVal := GetVariableAsStringFromVarFile(t, randomFileName, "aws_region") + + boolString := GetVariableAsStringFromVarFile(t, randomFileName, "boolean_type") + + numString := GetVariableAsStringFromVarFile(t, randomFileName, "number_type") + + require.Equal(t, "us-east-2", stringVal) + require.Equal(t, "true", boolString) + require.Equal(t, "2", numString) + +} + +func TestGetVariablesFromVarFilesAsStringKeyDoesNotExistJSON(t *testing.T) { + randomFileName := fmt.Sprintf("./%s.tfvars.json", random.UniqueId()) + + testJSON := []byte(` + { + "aws_region": "us-east-2", + "aws_account_id": "111111111111", + "tags": { + "foo": "bar" + }, + "list": ["item1"] + }`) + + WriteFile(t, randomFileName, testJSON) + defer os.Remove(randomFileName) + + _, err := GetVariableAsStringFromVarFileE(t, randomFileName, "badkey") + + require.Error(t, err) +} + +func TestGetVariableAsMapFromVarFileJSON(t *testing.T) { + randomFileName := fmt.Sprintf("./%s.tfvars.json", random.UniqueId()) + expected := make(map[string]string) + expected["foo"] = "bar" + + testJSON := []byte(` + { + "aws_region": "us-east-2", + "aws_account_id": "111111111111", + "tags": { + "foo": "bar" + }, + "list": ["item1"] + }`) + + WriteFile(t, randomFileName, testJSON) + defer os.Remove(randomFileName) + + val := GetVariableAsMapFromVarFile(t, randomFileName, "tags") + require.Equal(t, expected, val) +} + +func TestGetVariableAsMapFromVarFileNotMapJSON(t *testing.T) { + randomFileName := fmt.Sprintf("./%s.tfvars.json", random.UniqueId()) + + testJSON := []byte(` + { + "aws_region": "us-east-2", + "aws_account_id": "111111111111", + "tags": { + "foo": "bar" + }, + "list": ["item1"] + }`) + + WriteFile(t, randomFileName, testJSON) + defer os.Remove(randomFileName) + + _, err := GetVariableAsMapFromVarFileE(t, randomFileName, "aws_region") + + require.Error(t, err) +} + +func TestGetVariableAsMapFromVarFileKeyDoesNotExistJSON(t *testing.T) { + randomFileName := fmt.Sprintf("./%s.tfvars.json", random.UniqueId()) + + testJSON := []byte(` + { + "aws_region": "us-east-2", + "aws_account_id": "111111111111", + "tags": { + "foo": "bar" + }, + "list": ["item1"] + }`) + + WriteFile(t, randomFileName, testJSON) + defer os.Remove(randomFileName) + + _, err := GetVariableAsMapFromVarFileE(t, randomFileName, "badkey") + + require.Error(t, err) +} + +func TestGetVariableAsListFromVarFileJSON(t *testing.T) { + randomFileName := fmt.Sprintf("./%s.tfvars.json", random.UniqueId()) + expected := []string{"item1"} + + testJSON := []byte(` + { + "aws_region": "us-east-2", + "aws_account_id": "111111111111", + "tags": { + "foo": "bar" + }, + "list": ["item1"] + }`) + + WriteFile(t, randomFileName, testJSON) + defer os.Remove(randomFileName) + + val := GetVariableAsListFromVarFile(t, randomFileName, "list") + require.Equal(t, expected, val) +} + +func TestGetVariableAsListNotListJSON(t *testing.T) { + randomFileName := fmt.Sprintf("./%s.tfvars.json", random.UniqueId()) + + testJSON := []byte(` + { + "aws_region": "us-east-2", + "aws_account_id": "111111111111", + "tags": { + "foo": "bar" + }, + "list": ["item1"] + }`) + + WriteFile(t, randomFileName, testJSON) + defer os.Remove(randomFileName) + + _, err := GetVariableAsListFromVarFileE(t, randomFileName, "tags") + + require.Error(t, err) +} + +func TestGetVariableAsListKeyDoesNotExistJSON(t *testing.T) { + randomFileName := fmt.Sprintf("./%s.tfvars.json", random.UniqueId()) + + testJSON := []byte(` + { + "aws_region": "us-east-2", + "aws_account_id": "111111111111", + "tags": { + "foo": "bar" + }, + "list": ["item1"] + }`) + + WriteFile(t, randomFileName, testJSON) + defer os.Remove(randomFileName) + + _, err := GetVariableAsListFromVarFileE(t, randomFileName, "badkey") + + require.Error(t, err) +} + +func TestGetAllVariablesFromVarFileBadFileJSON(t *testing.T) { + randomFileName := fmt.Sprintf("./%s.tfvars.json", random.UniqueId()) + testJSON := []byte(` + { + thiswillnotwork + }`) + + WriteFile(t, randomFileName, testJSON) + defer os.Remove(randomFileName) + + var variables map[string]interface{} + err := GetAllVariablesFromVarFileE(t, randomFileName, &variables) + require.Error(t, err) + + // HCL library could change their error string, so we are only testing the error string contains what we add to it + require.Regexp(t, fmt.Sprintf("^%s:3,7-22: ", randomFileName), err.Error()) +} + +func TestGetAllVariablesFromVarFileJSON(t *testing.T) { + randomFileName := fmt.Sprintf("./%s.tfvars.json", random.UniqueId()) + testJSON := []byte(` + { + "aws_region": "us-east-2" + } + `) + + WriteFile(t, randomFileName, testJSON) + defer os.Remove(randomFileName) + + var variables map[string]interface{} + err := GetAllVariablesFromVarFileE(t, randomFileName, &variables) + require.NoError(t, err) + + expected := make(map[string]interface{}) + expected["aws_region"] = "us-east-2" + + require.Equal(t, expected, variables) +} + +func TestGetAllVariablesFromVarFileStructOutJSON(t *testing.T) { + randomFileName := fmt.Sprintf("./%s.tfvars.json", random.UniqueId()) + testJSON := []byte(` + { + "aws_region": "us-east-2" + } + `) + + WriteFile(t, randomFileName, testJSON) + defer os.Remove(randomFileName) + + var region struct { + AwsRegion string `cty:"aws_region"` + } + err := GetAllVariablesFromVarFileE(t, randomFileName, ®ion) + require.NoError(t, err) + require.Equal(t, "us-east-2", region.AwsRegion) } // Helper function to write a file to the filesystem