add support for providing a dest folder for tf copy

modules/files/files.go

@@ -64,6 +64,32 @@ func CopyTerraformFolderToTemp(folderPath string, tempFolderPrefix string) (stri
 	return destFolder, nil
 }
 
+// CopyTerraformFolderToDest creates a copy of the given folder and all its contents in a specified folder with a unique name and the given prefix.
+// This is useful when running multiple tests in parallel against the same set of Terraform files to ensure the
+// tests don't overwrite each other's .terraform working directory and terraform.tfstate files. This method returns
+// the path to the dest folder with the copied contents. Hidden files and folders (with the exception of the `.terraform-version` files used
+// by the [tfenv tool](https://github.com/tfutils/tfenv)), Terraform state files, and terraform.tfvars files are not copied to this temp folder,
+// as you typically don't want them interfering with your tests.
+// This method is useful when running through a build tool so the files are copied to a destination that is cleaned on each run of the pipeline.
+func CopyTerraformFolderToDest(folderPath string, destRootFolder string, tempFolderPrefix string) (string, error) {
+	filter := func(path string) bool {
+		if PathIsTerraformVersionFile(path) {
+			return true
+		}
+		if PathContainsHiddenFileOrFolder(path) || PathContainsTerraformStateOrVars(path) {
+			return false
+		}
+		return true
+	}
+
+	destFolder, err := CopyFolderToDest(folderPath, destRootFolder, tempFolderPrefix, filter)
+	if err != nil {
+		return "", err
+	}
+
+	return destFolder, nil
+}
+
 // CopyTerragruntFolderToTemp creates a copy of the given folder and all its contents in a temp folder with a unique name and the given prefix.
 // Since terragrunt uses tfvars files to specify modules, they are copied to the temporary directory as well.
 // Terraform state files are excluded as well as .terragrunt-cache to avoid overwriting contents.
@@ -80,6 +106,22 @@ func CopyTerragruntFolderToTemp(folderPath string, tempFolderPrefix string) (str
 	return destFolder, nil
 }
 
+// CopyTerragruntFolderToDest creates a copy of the given folder and all its contents in a specified folder with a unique name and the given prefix.
+// Since terragrunt uses tfvars files to specify modules, they are copied to the directory as well.
+// Terraform state files are excluded as well as .terragrunt-cache to avoid overwriting contents.
+func CopyTerragruntFolderToDest(folderPath string, destRootFolder string, tempFolderPrefix string) (string, error) {
+	filter := func(path string) bool {
+		return !PathContainsHiddenFileOrFolder(path) && !PathContainsTerraformState(path)
+	}
+
+	destFolder, err := CopyFolderToDest(folderPath, destRootFolder, tempFolderPrefix, filter)
+	if err != nil {
+		return "", err
+	}
+
+	return destFolder, nil
+}
+
 // CopyFolderToTemp creates a copy of the given folder and all its filtered contents in a temp folder
 // with a unique name and the given prefix.
 func CopyFolderToTemp(folderPath string, tempFolderPrefix string, filter func(path string) bool) (string, error) {
@@ -115,6 +157,49 @@ func CopyFolderToTemp(folderPath string, tempFolderPrefix string, filter func(pa
 	return destFolder, nil
 }
 
+// CopyFolderToDest creates a copy of the given folder and all its filtered contents in a temp folder
+// with a unique name and the given prefix.
+func CopyFolderToDest(folderPath string, destRootFolder string, tempFolderPrefix string, filter func(path string) bool) (string, error) {
+	destRootExists, err := FileExistsE(destRootFolder)
+	if err != nil {
+		return "", err
+	}
+	if !destRootExists {
+		return "", DirNotFoundError{Directory: destRootFolder}
+	}
+
+	exists, err := FileExistsE(folderPath)
+	if err != nil {
+		return "", err
+	}
+	if !exists {
+		return "", DirNotFoundError{Directory: folderPath}
+	}
+
+	tmpDir, err := ioutil.TempDir(destRootFolder, tempFolderPrefix)
+	if err != nil {
+		return "", err
+	}
+
+	// Inside of the temp folder, we create a subfolder that preserves the name of the folder we're copying from.
+	absFolderPath, err := filepath.Abs(folderPath)
+	if err != nil {
+		return "", err
+	}
+	folderName := filepath.Base(absFolderPath)
+	destFolder := filepath.Join(tmpDir, folderName)
+
+	if err := os.MkdirAll(destFolder, 0777); err != nil {
+		return "", err
+	}
+
+	if err := CopyFolderContentsWithFilter(folderPath, destFolder, filter); err != nil {
+		return "", err
+	}
+
+	return destFolder, nil
+}
+
 // CopyFolderContents copies all the files and folders within the given source folder to the destination folder.
 func CopyFolderContents(source string, destination string) error {
 	return CopyFolderContentsWithFilter(source, destination, func(path string) bool {

modules/files/files_test.go

@@ -68,6 +68,27 @@ func TestCopyFolderToTemp(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+func TestCopyFolderToDest(t *testing.T) {
+	t.Parallel()
+
+	tempFolderPrefix := "someprefix"
+	destFolder := os.TempDir()
+	tmpDir, err := ioutil.TempDir("", "TestCopyFolderContents")
+	require.NoError(t, err)
+
+	filter := func(path string) bool {
+		return !PathContainsHiddenFileOrFolder(path) && !PathContainsTerraformState(path)
+	}
+
+	folder, err := CopyFolderToDest("/not/a/real/path", destFolder, tempFolderPrefix, filter)
+	require.Error(t, err)
+	assert.False(t, FileExists(folder))
+
+	folder, err = CopyFolderToDest(tmpDir, destFolder, tempFolderPrefix, filter)
+	assert.DirExists(t, folder)
+	assert.NoError(t, err)
+}
+
 func TestCopyFolderContents(t *testing.T) {
 	t.Parallel()
 
@@ -161,6 +182,19 @@ func TestCopyTerraformFolderToTemp(t *testing.T) {
 	requireDirectoriesEqual(t, expectedDir, tmpDir)
 }
 
+func TestCopyTerraformFolderToDest(t *testing.T) {
+	t.Parallel()
+
+	originalDir := filepath.Join(copyFolderContentsFixtureRoot, "original")
+	expectedDir := filepath.Join(copyFolderContentsFixtureRoot, "no-hidden-files-no-terraform-files")
+	destFolder := os.TempDir()
+
+	tmpDir, err := CopyTerraformFolderToDest(originalDir, destFolder, "TestCopyTerraformFolderToTemp")
+	require.NoError(t, err)
+
+	requireDirectoriesEqual(t, expectedDir, tmpDir)
+}
+
 func TestCopyTerragruntFolderToTemp(t *testing.T) {
 	t.Parallel()
 
@@ -173,6 +207,19 @@ func TestCopyTerragruntFolderToTemp(t *testing.T) {
 	requireDirectoriesEqual(t, expectedDir, tmpDir)
 }
 
+func TestCopyTerragruntFolderToDest(t *testing.T) {
+	t.Parallel()
+
+	originalDir := filepath.Join(copyFolderContentsFixtureRoot, "terragrunt-files")
+	expectedDir := filepath.Join(copyFolderContentsFixtureRoot, "no-state-files")
+	destFolder := os.TempDir()
+
+	tmpDir, err := CopyTerragruntFolderToDest(originalDir, destFolder, t.Name())
+	require.NoError(t, err)
+
+	requireDirectoriesEqual(t, expectedDir, tmpDir)
+}
+
 func TestPathContainsTerraformStateOrVars(t *testing.T) {
 	var data = []struct {
 		desc     string

modules/test-structure/test_structure.go

@@ -97,6 +97,64 @@ func CopyTerraformFolderToTemp(t testing.TestingT, rootFolder string, terraformM
 	return tmpTestFolder
 }
 
+// CopyTerraformFolderToDest copies the given root folder to a randomly-named temp folder and return the path to the
+// given terraform modules folder within the new temp root folder. This is useful when running multiple tests in
+// parallel against the same set of Terraform files to ensure the tests don't overwrite each other's .terraform working
+// directory and terraform.tfstate files. To ensure relative paths work, we copy over the entire root folder to a temp
+// folder, and then return the path within that temp folder to the given terraform module dir, which is where the actual
+// test will be running.
+// For example, suppose you had the target terraform folder you want to test in "/examples/terraform-aws-example"
+// relative to the repo root. If your tests reside in the "/test" relative to the root, then you will use this as
+// follows:
+//
+//       // Destination for the copy of the files.  In this example we are using the Azure Dev Ops variable
+//       // for the folder that is cleaned after each pipeline job.
+//       destRootFolder := os.Getenv("AGENT_TEMPDIRECTORY")
+//
+//       // Root folder where terraform files should be (relative to the test folder)
+//       rootFolder := ".."
+//
+//       // Relative path to terraform module being tested from the root folder
+//       terraformFolderRelativeToRoot := "examples/terraform-aws-example"
+//
+//       // Copy the terraform folder to a temp folder
+//       tempTestFolder := test_structure.CopyTerraformFolderToTemp(t, destRootFolder, rootFolder, terraformFolderRelativeToRoot)
+//
+//       // Make sure to use the temp test folder in the terraform options
+//       terraformOptions := &terraform.Options{
+//       		TerraformDir: tempTestFolder,
+//       }
+//
+// Note that if any of the SKIP_<stage> environment variables is set, we assume this is a test in the local dev where
+// there are no other concurrent tests running and we want to be able to cache test data between test stages, so in that
+// case, we do NOT copy anything to a temp folder, and return the path to the original terraform module folder instead.
+func CopyTerraformFolderToDest(t testing.TestingT, rootFolder string, terraformModuleFolder string, destRootFolder string) string {
+	if SkipStageEnvVarSet() {
+		logger.Logf(t, "A SKIP_XXX environment variable is set. Using original examples folder rather than a temp folder so we can cache data between stages for faster local testing.")
+		return filepath.Join(rootFolder, terraformModuleFolder)
+	}
+
+	fullTerraformModuleFolder := filepath.Join(rootFolder, terraformModuleFolder)
+
+	exists, err := files.FileExistsE(fullTerraformModuleFolder)
+	require.NoError(t, err)
+	if !exists {
+		t.Fatal(files.DirNotFoundError{Directory: fullTerraformModuleFolder})
+	}
+
+	tmpRootFolder, err := files.CopyTerraformFolderToDest(rootFolder, cleanName(t.Name()), destRootFolder)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tmpTestFolder := filepath.Join(tmpRootFolder, terraformModuleFolder)
+
+	// Log temp folder so we can see it
+	logger.Logf(t, "Copied terraform folder %s to %s", fullTerraformModuleFolder, tmpTestFolder)
+
+	return tmpTestFolder
+}
+
 func cleanName(originalName string) string {
 	parts := strings.Split(originalName, "/")
 	return parts[len(parts)-1]