Merge pull request #959 from rescDev/feature/enableJSONParsing

Feature: Enable JSON parsing for getting TF config variables

.gitignore

@@ -2,6 +2,7 @@
 .terraform
 terraform.tfstate
 terraform.tfvars
+terraform.tfvars.json
 *.tfstate*
 .terragrunt
 .terragrunt-cache

go.sum

@@ -267,8 +267,6 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hashicorp/hcl/v2 v2.8.2 h1:wmFle3D1vu0okesm8BTLVDyJ6/OL9DCLUwn0b2OptiY=
 github.com/hashicorp/hcl/v2 v2.8.2/go.mod h1:bQTN5mpo+jewjJgh8jr0JUguIi7qPHUF6yIfAEN3jqY=
-github.com/hashicorp/terraform-json v0.9.0 h1:WE7+Wt93W93feOiCligElSyS0tlDzwZUtJuDGIBr8zg=
-github.com/hashicorp/terraform-json v0.9.0/go.mod h1:3defM4kkMfttwiE7VakJDwCd4R+umhSQnvJwORXbprE=
 github.com/hashicorp/terraform-json v0.12.0 h1:8czPgEEWWPROStjkWPUnTQDXmpmZPlkQAwYYLETaTvw=
 github.com/hashicorp/terraform-json v0.12.0/go.mod h1:pmbq9o4EuL43db5+0ogX10Yofv1nozM+wskr/bGFJpI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
@@ -283,6 +281,7 @@ github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5i
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=

modules/files/files.go

@@ -150,10 +150,10 @@ func CopyFolderContentsWithFilter(source string, destination string, filter func
 	return nil
 }
 
-// PathContainsTerraformStateOrVars returns true if the path corresponds to a Terraform state file or .tfvars file.
+// PathContainsTerraformStateOrVars returns true if the path corresponds to a Terraform state file or .tfvars/.tfvars.json file.
 func PathContainsTerraformStateOrVars(path string) bool {
 	filename := filepath.Base(path)
-	return filename == "terraform.tfstate" || filename == "terraform.tfstate.backup" || filename == "terraform.tfvars"
+	return filename == "terraform.tfstate" || filename == "terraform.tfstate.backup" || filename == "terraform.tfvars" || filename == "terraform.tfvars.json"
 }
 
 // PathContainsTerraformState returns true if the path corresponds to a Terraform state file.

modules/files/files_test.go

@@ -173,6 +173,34 @@ func TestCopyTerragruntFolderToTemp(t *testing.T) {
 	requireDirectoriesEqual(t, expectedDir, tmpDir)
 }
 
+func TestPathContainsTerraformStateOrVars(t *testing.T) {
+	var data = []struct {
+		desc     string
+		path     string
+		contains bool
+	}{
+		{"contains tfvars", "./folder/terraform.tfvars", true},
+		{"contains tfvars.json", "./folder/hello/terraform.tfvars.json", true},
+		{"contains state", "./folder/hello/helloagain/terraform.tfstate", true},
+		{"contains state backup", "./folder/hey/terraform.tfstate.backup", true},
+		{"does not contain any", "./folder/salut/terraform.json", false},
+	}
+
+	for _, tt := range data {
+		tt := tt
+		t.Run(tt.desc, func(t *testing.T) {
+			result := PathContainsTerraformStateOrVars(tt.path)
+			if result != tt.contains {
+				if tt.contains {
+					t.Errorf("Expected %s to contain Terraform related file", tt.path)
+				} else {
+					t.Errorf("Expected %s to not contain Terraform related file", tt.path)
+				}
+			}
+		})
+	}
+}
+
 // Diffing two directories to ensure they have the exact same files, contents, etc and showing exactly what's different
 // takes a lot of code. Why waste time on that when this functionality is already nicely implemented in the Unix/Linux
 // "diff" command? We shell out to that command at test time.

modules/terraform/format.go

@@ -119,8 +119,8 @@ func FormatTerraformPluginDirAsArgs(pluginDir string) []string {
 	return pluginArgs
 }
 
-// FormatTerraformArgs will format multiple args with the arg name (e.g. "-var-file", []string{"foo.tfvars", "bar.tfvars"})
-// returns "-var-file foo.tfvars -var-file bar.tfvars"
+// FormatTerraformArgs will format multiple args with the arg name (e.g. "-var-file", []string{"foo.tfvars", "bar.tfvars", "baz.tfvars.json"})
+// returns "-var-file foo.tfvars -var-file bar.tfvars -var-file baz.tfvars.json"
 func FormatTerraformArgs(argName string, args []string) []string {
 	argsList := []string{}
 	for _, argValue := range args {

modules/terraform/var-file.go

@@ -5,9 +5,11 @@ import (
 	"fmt"
 	"io/ioutil"
 	"reflect"
+	"strings"
 
 	"github.com/gruntwork-io/terratest/modules/testing"
 
+	"github.com/hashicorp/hcl/v2"
 	"github.com/hashicorp/hcl/v2/hclparse"
 	"github.com/stretchr/testify/require"
 	"github.com/zclconf/go-cty/cty"
@@ -133,9 +135,9 @@ func GetAllVariablesFromVarFileE(t testing.TestingT, fileName string, out interf
 	return parseAndDecodeVarFile(string(fileContents), fileName, out)
 }
 
-// parseAndDecodeVarFile uses the HCL2 parser to parse the given varfile string into an HCL file body, and then decode it
+// parseAndDecodeVarFile uses the HCL2 parser to parse the given varfile string into an HCL or HCL JSON file body, and then decode it
 // into a map that maps var names to values.
-func parseAndDecodeVarFile(hclContents string, filename string, out interface{}) (err error) {
+func parseAndDecodeVarFile(fileContents string, filename string, out interface{}) (err error) {
 	// The HCL2 parser and especially cty conversions will panic in many types of errors, so we have to recover from
 	// those panics here and convert them to normal errors
 	defer func() {
@@ -146,7 +148,16 @@ func parseAndDecodeVarFile(hclContents string, filename string, out interface{})
 
 	parser := hclparse.NewParser()
 
-	file, parseDiagnostics := parser.ParseHCL([]byte(hclContents), filename)
+	var file *hcl.File
+	var parseDiagnostics hcl.Diagnostics
+
+	// determine if a JSON variables file is submitted and parse accordingly
+	if strings.HasSuffix(filename, ".json") {
+		file, parseDiagnostics = parser.ParseJSON([]byte(fileContents), filename)
+	} else {
+		file, parseDiagnostics = parser.ParseHCL([]byte(fileContents), filename)
+	}
+
 	if parseDiagnostics != nil && parseDiagnostics.HasErrors() {
 		return parseDiagnostics
 	}

modules/terraform/var-file_test.go

@@ -228,7 +228,243 @@ func TestGetAllVariablesFromVarFileStructOut(t *testing.T) {
 	err := GetAllVariablesFromVarFileE(t, randomFileName, &region)
 	require.NoError(t, err)
 	require.Equal(t, "us-east-2", region.AwsRegion)
+}
+
+func TestGetVariablesFromVarFilesAsStringJSON(t *testing.T) {
+	randomFileName := fmt.Sprintf("./%s.tfvars.json", random.UniqueId())
+
+	testJSON := []byte(`
+		{
+			"aws_region": "us-east-2",
+			"aws_account_id": "111111111111",
+			"number_type": 2,
+			"boolean_type": true,
+			"tags": {
+				"foo": "bar"
+			},
+			"list": ["item1"]
+		}`)
+
+	WriteFile(t, randomFileName, testJSON)
+	defer os.Remove(randomFileName)
+
+	stringVal := GetVariableAsStringFromVarFile(t, randomFileName, "aws_region")
+
+	boolString := GetVariableAsStringFromVarFile(t, randomFileName, "boolean_type")
+
+	numString := GetVariableAsStringFromVarFile(t, randomFileName, "number_type")
+
+	require.Equal(t, "us-east-2", stringVal)
+	require.Equal(t, "true", boolString)
+	require.Equal(t, "2", numString)
+
+}
+
+func TestGetVariablesFromVarFilesAsStringKeyDoesNotExistJSON(t *testing.T) {
+	randomFileName := fmt.Sprintf("./%s.tfvars.json", random.UniqueId())
+
+	testJSON := []byte(`
+		{
+			"aws_region": "us-east-2",
+			"aws_account_id": "111111111111",
+			"tags": {
+				"foo": "bar"
+			},
+			"list": ["item1"]
+		}`)
+
+	WriteFile(t, randomFileName, testJSON)
+	defer os.Remove(randomFileName)
+
+	_, err := GetVariableAsStringFromVarFileE(t, randomFileName, "badkey")
+
+	require.Error(t, err)
+}
+
+func TestGetVariableAsMapFromVarFileJSON(t *testing.T) {
+	randomFileName := fmt.Sprintf("./%s.tfvars.json", random.UniqueId())
+	expected := make(map[string]string)
+	expected["foo"] = "bar"
+
+	testJSON := []byte(`
+		{
+			"aws_region": "us-east-2",
+			"aws_account_id": "111111111111",
+			"tags": {
+				"foo": "bar"
+			},
+			"list": ["item1"]
+		}`)
+
+	WriteFile(t, randomFileName, testJSON)
+	defer os.Remove(randomFileName)
+
+	val := GetVariableAsMapFromVarFile(t, randomFileName, "tags")
+	require.Equal(t, expected, val)
+}
+
+func TestGetVariableAsMapFromVarFileNotMapJSON(t *testing.T) {
+	randomFileName := fmt.Sprintf("./%s.tfvars.json", random.UniqueId())
+
+	testJSON := []byte(`
+		{
+			"aws_region": "us-east-2",
+			"aws_account_id": "111111111111",
+			"tags": {
+				"foo": "bar"
+			},
+			"list": ["item1"]
+		}`)
+
+	WriteFile(t, randomFileName, testJSON)
+	defer os.Remove(randomFileName)
+
+	_, err := GetVariableAsMapFromVarFileE(t, randomFileName, "aws_region")
+
+	require.Error(t, err)
+}
+
+func TestGetVariableAsMapFromVarFileKeyDoesNotExistJSON(t *testing.T) {
+	randomFileName := fmt.Sprintf("./%s.tfvars.json", random.UniqueId())
+
+	testJSON := []byte(`
+		{
+			"aws_region": "us-east-2",
+			"aws_account_id": "111111111111",
+			"tags": {
+				"foo": "bar"
+			},
+			"list": ["item1"]
+		}`)
+
+	WriteFile(t, randomFileName, testJSON)
+	defer os.Remove(randomFileName)
+
+	_, err := GetVariableAsMapFromVarFileE(t, randomFileName, "badkey")
+
+	require.Error(t, err)
+}
+
+func TestGetVariableAsListFromVarFileJSON(t *testing.T) {
+	randomFileName := fmt.Sprintf("./%s.tfvars.json", random.UniqueId())
+	expected := []string{"item1"}
+
+	testJSON := []byte(`
+		{
+			"aws_region": "us-east-2",
+			"aws_account_id": "111111111111",
+			"tags": {
+				"foo": "bar"
+			},
+			"list": ["item1"]
+		}`)
+
+	WriteFile(t, randomFileName, testJSON)
+	defer os.Remove(randomFileName)
+
+	val := GetVariableAsListFromVarFile(t, randomFileName, "list")
 
+	require.Equal(t, expected, val)
+}
+
+func TestGetVariableAsListNotListJSON(t *testing.T) {
+	randomFileName := fmt.Sprintf("./%s.tfvars.json", random.UniqueId())
+
+	testJSON := []byte(`
+		{
+			"aws_region": "us-east-2",
+			"aws_account_id": "111111111111",
+			"tags": {
+				"foo": "bar"
+			},
+			"list": ["item1"]
+		}`)
+
+	WriteFile(t, randomFileName, testJSON)
+	defer os.Remove(randomFileName)
+
+	_, err := GetVariableAsListFromVarFileE(t, randomFileName, "tags")
+
+	require.Error(t, err)
+}
+
+func TestGetVariableAsListKeyDoesNotExistJSON(t *testing.T) {
+	randomFileName := fmt.Sprintf("./%s.tfvars.json", random.UniqueId())
+
+	testJSON := []byte(`
+		{
+			"aws_region": "us-east-2",
+			"aws_account_id": "111111111111",
+			"tags": {
+				"foo": "bar"
+			},
+			"list": ["item1"]
+		}`)
+
+	WriteFile(t, randomFileName, testJSON)
+	defer os.Remove(randomFileName)
+
+	_, err := GetVariableAsListFromVarFileE(t, randomFileName, "badkey")
+
+	require.Error(t, err)
+}
+
+func TestGetAllVariablesFromVarFileBadFileJSON(t *testing.T) {
+	randomFileName := fmt.Sprintf("./%s.tfvars.json", random.UniqueId())
+	testJSON := []byte(`
+		{
+			thiswillnotwork
+		}`)
+
+	WriteFile(t, randomFileName, testJSON)
+	defer os.Remove(randomFileName)
+
+	var variables map[string]interface{}
+	err := GetAllVariablesFromVarFileE(t, randomFileName, &variables)
+	require.Error(t, err)
+
+	// HCL library could change their error string, so we are only testing the error string contains what we add to it
+	require.Regexp(t, fmt.Sprintf("^%s:3,7-22: ", randomFileName), err.Error())
+}
+
+func TestGetAllVariablesFromVarFileJSON(t *testing.T) {
+	randomFileName := fmt.Sprintf("./%s.tfvars.json", random.UniqueId())
+	testJSON := []byte(`
+	{
+		"aws_region": "us-east-2"
+	}
+	`)
+
+	WriteFile(t, randomFileName, testJSON)
+	defer os.Remove(randomFileName)
+
+	var variables map[string]interface{}
+	err := GetAllVariablesFromVarFileE(t, randomFileName, &variables)
+	require.NoError(t, err)
+
+	expected := make(map[string]interface{})
+	expected["aws_region"] = "us-east-2"
+
+	require.Equal(t, expected, variables)
+}
+
+func TestGetAllVariablesFromVarFileStructOutJSON(t *testing.T) {
+	randomFileName := fmt.Sprintf("./%s.tfvars.json", random.UniqueId())
+	testJSON := []byte(`
+	{
+		"aws_region": "us-east-2"
+	}
+	`)
+
+	WriteFile(t, randomFileName, testJSON)
+	defer os.Remove(randomFileName)
+
+	var region struct {
+		AwsRegion string `cty:"aws_region"`
+	}
+	err := GetAllVariablesFromVarFileE(t, randomFileName, &region)
+	require.NoError(t, err)
+	require.Equal(t, "us-east-2", region.AwsRegion)
 }
 
 // Helper function to write a file to the filesystem