add support for providing a dest folder for tf copy

modules/files/files.go

@@ -39,13 +39,14 @@ func IsExistingDir(path string) bool {
 	return err == nil && fileInfo.IsDir()
 }
 
-// CopyTerraformFolderToTemp creates a copy of the given folder and all its contents in a temp folder with a unique name and the given prefix.
+// CopyTerraformFolderToDest creates a copy of the given folder and all its contents in a specified folder with a unique name and the given prefix.
 // This is useful when running multiple tests in parallel against the same set of Terraform files to ensure the
 // tests don't overwrite each other's .terraform working directory and terraform.tfstate files. This method returns
-// the path to the temp folder with the copied contents. Hidden files and folders (with the exception of the `.terraform-version` files used
+// the path to the dest folder with the copied contents. Hidden files and folders (with the exception of the `.terraform-version` files used
 // by the [tfenv tool](https://github.com/tfutils/tfenv)), Terraform state files, and terraform.tfvars files are not copied to this temp folder,
 // as you typically don't want them interfering with your tests.
-func CopyTerraformFolderToTemp(folderPath string, tempFolderPrefix string) (string, error) {
+// This method is useful when running through a build tool so the files are copied to a destination that is cleaned on each run of the pipeline.
+func CopyTerraformFolderToDest(folderPath string, destRootFolder string, tempFolderPrefix string) (string, error) {
 	filter := func(path string) bool {
 		if PathIsTerraformVersionFile(path) {
 			return true
@@ -56,33 +57,51 @@ func CopyTerraformFolderToTemp(folderPath string, tempFolderPrefix string) (stri
 		return true
 	}
 
-	destFolder, err := CopyFolderToTemp(folderPath, tempFolderPrefix, filter)
+	destFolder, err := CopyFolderToDest(folderPath, destRootFolder, tempFolderPrefix, filter)
 	if err != nil {
 		return "", err
 	}
 
 	return destFolder, nil
 }
 
-// CopyTerragruntFolderToTemp creates a copy of the given folder and all its contents in a temp folder with a unique name and the given prefix.
-// Since terragrunt uses tfvars files to specify modules, they are copied to the temporary directory as well.
+// CopyTerraformFolderToTemp calls CopyTerraformFolderToDest, passing os.TempDir() as the root destination folder.
+func CopyTerraformFolderToTemp(folderPath string, tempFolderPrefix string) (string, error) {
+	return CopyTerraformFolderToDest(folderPath, os.TempDir(), tempFolderPrefix)
+}
+
+// CopyTerragruntFolderToDest creates a copy of the given folder and all its contents in a specified folder with a unique name and the given prefix.
+// Since terragrunt uses tfvars files to specify modules, they are copied to the directory as well.
 // Terraform state files are excluded as well as .terragrunt-cache to avoid overwriting contents.
-func CopyTerragruntFolderToTemp(folderPath string, tempFolderPrefix string) (string, error) {
+func CopyTerragruntFolderToDest(folderPath string, destRootFolder string, tempFolderPrefix string) (string, error) {
 	filter := func(path string) bool {
 		return !PathContainsHiddenFileOrFolder(path) && !PathContainsTerraformState(path)
 	}
 
-	destFolder, err := CopyFolderToTemp(folderPath, tempFolderPrefix, filter)
+	destFolder, err := CopyFolderToDest(folderPath, destRootFolder, tempFolderPrefix, filter)
 	if err != nil {
 		return "", err
 	}
 
 	return destFolder, nil
 }
 
-// CopyFolderToTemp creates a copy of the given folder and all its filtered contents in a temp folder
+// CopyTerragruntFolderToTemp calls CopyTerragruntFolderToDest, passing os.TempDir() as the root destination folder.
+func CopyTerragruntFolderToTemp(folderPath string, tempFolderPrefix string) (string, error) {
+	return CopyTerragruntFolderToDest(folderPath, os.TempDir(), tempFolderPrefix)
+}
+
+// CopyFolderToDest creates a copy of the given folder and all its filtered contents in a temp folder
 // with a unique name and the given prefix.
-func CopyFolderToTemp(folderPath string, tempFolderPrefix string, filter func(path string) bool) (string, error) {
+func CopyFolderToDest(folderPath string, destRootFolder string, tempFolderPrefix string, filter func(path string) bool) (string, error) {
+	destRootExists, err := FileExistsE(destRootFolder)
+	if err != nil {
+		return "", err
+	}
+	if !destRootExists {
+		return "", DirNotFoundError{Directory: destRootFolder}
+	}
+
 	exists, err := FileExistsE(folderPath)
 	if err != nil {
 		return "", err
@@ -91,7 +110,7 @@ func CopyFolderToTemp(folderPath string, tempFolderPrefix string, filter func(pa
 		return "", DirNotFoundError{Directory: folderPath}
 	}
 
-	tmpDir, err := ioutil.TempDir("", tempFolderPrefix)
+	tmpDir, err := ioutil.TempDir(destRootFolder, tempFolderPrefix)
 	if err != nil {
 		return "", err
 	}
@@ -115,6 +134,11 @@ func CopyFolderToTemp(folderPath string, tempFolderPrefix string, filter func(pa
 	return destFolder, nil
 }
 
+// CopyFolderToTemp calls CopyFolderToDest, passing os.TempDir() as the root destination folder.
+func CopyFolderToTemp(folderPath string, tempFolderPrefix string, filter func(path string) bool) (string, error) {
+	return CopyFolderToDest(folderPath, os.TempDir(), tempFolderPrefix, filter)
+}
+
 // CopyFolderContents copies all the files and folders within the given source folder to the destination folder.
 func CopyFolderContents(source string, destination string) error {
 	return CopyFolderContentsWithFilter(source, destination, func(path string) bool {

modules/files/files_test.go

@@ -48,22 +48,23 @@ func TestIsExistingDir(t *testing.T) {
 	assert.True(t, IsExistingDir(currentFileDir))
 }
 
-func TestCopyFolderToTemp(t *testing.T) {
+func TestCopyFolderToDest(t *testing.T) {
 	t.Parallel()
 
 	tempFolderPrefix := "someprefix"
+	destFolder := os.TempDir()
 	tmpDir, err := ioutil.TempDir("", "TestCopyFolderContents")
 	require.NoError(t, err)
 
 	filter := func(path string) bool {
 		return !PathContainsHiddenFileOrFolder(path) && !PathContainsTerraformState(path)
 	}
 
-	folder, err := CopyFolderToTemp("/not/a/real/path", tempFolderPrefix, filter)
+	folder, err := CopyFolderToDest("/not/a/real/path", destFolder, tempFolderPrefix, filter)
 	require.Error(t, err)
 	assert.False(t, FileExists(folder))
 
-	folder, err = CopyFolderToTemp(tmpDir, tempFolderPrefix, filter)
+	folder, err = CopyFolderToDest(tmpDir, destFolder, tempFolderPrefix, filter)
 	assert.DirExists(t, folder)
 	assert.NoError(t, err)
 }
@@ -161,6 +162,19 @@ func TestCopyTerraformFolderToTemp(t *testing.T) {
 	requireDirectoriesEqual(t, expectedDir, tmpDir)
 }
 
+func TestCopyTerraformFolderToDest(t *testing.T) {
+	t.Parallel()
+
+	originalDir := filepath.Join(copyFolderContentsFixtureRoot, "original")
+	expectedDir := filepath.Join(copyFolderContentsFixtureRoot, "no-hidden-files-no-terraform-files")
+	destFolder := os.TempDir()
+
+	tmpDir, err := CopyTerraformFolderToDest(originalDir, destFolder, "TestCopyTerraformFolderToTemp")
+	require.NoError(t, err)
+
+	requireDirectoriesEqual(t, expectedDir, tmpDir)
+}
+
 func TestCopyTerragruntFolderToTemp(t *testing.T) {
 	t.Parallel()
 
@@ -173,6 +187,19 @@ func TestCopyTerragruntFolderToTemp(t *testing.T) {
 	requireDirectoriesEqual(t, expectedDir, tmpDir)
 }
 
+func TestCopyTerragruntFolderToDest(t *testing.T) {
+	t.Parallel()
+
+	originalDir := filepath.Join(copyFolderContentsFixtureRoot, "terragrunt-files")
+	expectedDir := filepath.Join(copyFolderContentsFixtureRoot, "no-state-files")
+	destFolder := os.TempDir()
+
+	tmpDir, err := CopyTerragruntFolderToDest(originalDir, destFolder, t.Name())
+	require.NoError(t, err)
+
+	requireDirectoriesEqual(t, expectedDir, tmpDir)
+}
+
 func TestPathContainsTerraformStateOrVars(t *testing.T) {
 	var data = []struct {
 		desc     string

modules/test-structure/test_structure.go

@@ -97,6 +97,64 @@ func CopyTerraformFolderToTemp(t testing.TestingT, rootFolder string, terraformM
 	return tmpTestFolder
 }
 
+// CopyTerraformFolderToDest copies the given root folder to a randomly-named temp folder and return the path to the
+// given terraform modules folder within the new temp root folder. This is useful when running multiple tests in
+// parallel against the same set of Terraform files to ensure the tests don't overwrite each other's .terraform working
+// directory and terraform.tfstate files. To ensure relative paths work, we copy over the entire root folder to a temp
+// folder, and then return the path within that temp folder to the given terraform module dir, which is where the actual
+// test will be running.
+// For example, suppose you had the target terraform folder you want to test in "/examples/terraform-aws-example"
+// relative to the repo root. If your tests reside in the "/test" relative to the root, then you will use this as
+// follows:
+//
+//       // Destination for the copy of the files.  In this example we are using the Azure Dev Ops variable
+//       // for the folder that is cleaned after each pipeline job.
+//       destRootFolder := os.Getenv("AGENT_TEMPDIRECTORY")
+//
+//       // Root folder where terraform files should be (relative to the test folder)
+//       rootFolder := ".."
+//
+//       // Relative path to terraform module being tested from the root folder
+//       terraformFolderRelativeToRoot := "examples/terraform-aws-example"
+//
+//       // Copy the terraform folder to a temp folder
+//       tempTestFolder := test_structure.CopyTerraformFolderToTemp(t, destRootFolder, rootFolder, terraformFolderRelativeToRoot)
+//
+//       // Make sure to use the temp test folder in the terraform options
+//       terraformOptions := &terraform.Options{
+//       		TerraformDir: tempTestFolder,
+//       }
+//
+// Note that if any of the SKIP_<stage> environment variables is set, we assume this is a test in the local dev where
+// there are no other concurrent tests running and we want to be able to cache test data between test stages, so in that
+// case, we do NOT copy anything to a temp folder, and return the path to the original terraform module folder instead.
+func CopyTerraformFolderToDest(t testing.TestingT, rootFolder string, terraformModuleFolder string, destRootFolder string) string {
+	if SkipStageEnvVarSet() {
+		logger.Logf(t, "A SKIP_XXX environment variable is set. Using original examples folder rather than a temp folder so we can cache data between stages for faster local testing.")
+		return filepath.Join(rootFolder, terraformModuleFolder)
+	}
+
+	fullTerraformModuleFolder := filepath.Join(rootFolder, terraformModuleFolder)
+
+	exists, err := files.FileExistsE(fullTerraformModuleFolder)
+	require.NoError(t, err)
+	if !exists {
+		t.Fatal(files.DirNotFoundError{Directory: fullTerraformModuleFolder})
+	}
+
+	tmpRootFolder, err := files.CopyTerraformFolderToDest(rootFolder, cleanName(t.Name()), destRootFolder)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tmpTestFolder := filepath.Join(tmpRootFolder, terraformModuleFolder)
+
+	// Log temp folder so we can see it
+	logger.Logf(t, "Copied terraform folder %s to %s", fullTerraformModuleFolder, tmpTestFolder)
+
+	return tmpTestFolder
+}
+
 func cleanName(originalName string) string {
 	parts := strings.Split(originalName, "/")
 	return parts[len(parts)-1]