diff --git a/modules/files/files.go b/modules/files/files.go index 2c928bb4a..a93b2ac5f 100644 --- a/modules/files/files.go +++ b/modules/files/files.go @@ -39,13 +39,14 @@ func IsExistingDir(path string) bool { return err == nil && fileInfo.IsDir() } -// CopyTerraformFolderToTemp creates a copy of the given folder and all its contents in a temp folder with a unique name and the given prefix. +// CopyTerraformFolderToDest creates a copy of the given folder and all its contents in a specified folder with a unique name and the given prefix. // This is useful when running multiple tests in parallel against the same set of Terraform files to ensure the // tests don't overwrite each other's .terraform working directory and terraform.tfstate files. This method returns -// the path to the temp folder with the copied contents. Hidden files and folders (with the exception of the `.terraform-version` files used +// the path to the dest folder with the copied contents. Hidden files and folders (with the exception of the `.terraform-version` files used // by the [tfenv tool](https://github.com/tfutils/tfenv)), Terraform state files, and terraform.tfvars files are not copied to this temp folder, // as you typically don't want them interfering with your tests. -func CopyTerraformFolderToTemp(folderPath string, tempFolderPrefix string) (string, error) { +// This method is useful when running through a build tool so the files are copied to a destination that is cleaned on each run of the pipeline. +func CopyTerraformFolderToDest(folderPath string, destRootFolder string, tempFolderPrefix string) (string, error) { filter := func(path string) bool { if PathIsTerraformVersionFile(path) { return true @@ -56,7 +57,7 @@ func CopyTerraformFolderToTemp(folderPath string, tempFolderPrefix string) (stri return true } - destFolder, err := CopyFolderToTemp(folderPath, tempFolderPrefix, filter) + destFolder, err := CopyFolderToDest(folderPath, destRootFolder, tempFolderPrefix, filter) if err != nil { return "", err } @@ -64,15 +65,20 @@ func CopyTerraformFolderToTemp(folderPath string, tempFolderPrefix string) (stri return destFolder, nil } -// CopyTerragruntFolderToTemp creates a copy of the given folder and all its contents in a temp folder with a unique name and the given prefix. -// Since terragrunt uses tfvars files to specify modules, they are copied to the temporary directory as well. +// CopyTerraformFolderToTemp calls CopyTerraformFolderToDest, passing os.TempDir() as the root destination folder. +func CopyTerraformFolderToTemp(folderPath string, tempFolderPrefix string) (string, error) { + return CopyTerraformFolderToDest(folderPath, os.TempDir(), tempFolderPrefix) +} + +// CopyTerragruntFolderToDest creates a copy of the given folder and all its contents in a specified folder with a unique name and the given prefix. +// Since terragrunt uses tfvars files to specify modules, they are copied to the directory as well. // Terraform state files are excluded as well as .terragrunt-cache to avoid overwriting contents. -func CopyTerragruntFolderToTemp(folderPath string, tempFolderPrefix string) (string, error) { +func CopyTerragruntFolderToDest(folderPath string, destRootFolder string, tempFolderPrefix string) (string, error) { filter := func(path string) bool { return !PathContainsHiddenFileOrFolder(path) && !PathContainsTerraformState(path) } - destFolder, err := CopyFolderToTemp(folderPath, tempFolderPrefix, filter) + destFolder, err := CopyFolderToDest(folderPath, destRootFolder, tempFolderPrefix, filter) if err != nil { return "", err } @@ -80,9 +86,22 @@ func CopyTerragruntFolderToTemp(folderPath string, tempFolderPrefix string) (str return destFolder, nil } -// CopyFolderToTemp creates a copy of the given folder and all its filtered contents in a temp folder +// CopyTerragruntFolderToTemp calls CopyTerragruntFolderToDest, passing os.TempDir() as the root destination folder. +func CopyTerragruntFolderToTemp(folderPath string, tempFolderPrefix string) (string, error) { + return CopyTerragruntFolderToDest(folderPath, os.TempDir(), tempFolderPrefix) +} + +// CopyFolderToDest creates a copy of the given folder and all its filtered contents in a temp folder // with a unique name and the given prefix. -func CopyFolderToTemp(folderPath string, tempFolderPrefix string, filter func(path string) bool) (string, error) { +func CopyFolderToDest(folderPath string, destRootFolder string, tempFolderPrefix string, filter func(path string) bool) (string, error) { + destRootExists, err := FileExistsE(destRootFolder) + if err != nil { + return "", err + } + if !destRootExists { + return "", DirNotFoundError{Directory: destRootFolder} + } + exists, err := FileExistsE(folderPath) if err != nil { return "", err @@ -91,7 +110,7 @@ func CopyFolderToTemp(folderPath string, tempFolderPrefix string, filter func(pa return "", DirNotFoundError{Directory: folderPath} } - tmpDir, err := ioutil.TempDir("", tempFolderPrefix) + tmpDir, err := ioutil.TempDir(destRootFolder, tempFolderPrefix) if err != nil { return "", err } @@ -115,6 +134,11 @@ func CopyFolderToTemp(folderPath string, tempFolderPrefix string, filter func(pa return destFolder, nil } +// CopyFolderToTemp calls CopyFolderToDest, passing os.TempDir() as the root destination folder. +func CopyFolderToTemp(folderPath string, tempFolderPrefix string, filter func(path string) bool) (string, error) { + return CopyFolderToDest(folderPath, os.TempDir(), tempFolderPrefix, filter) +} + // CopyFolderContents copies all the files and folders within the given source folder to the destination folder. func CopyFolderContents(source string, destination string) error { return CopyFolderContentsWithFilter(source, destination, func(path string) bool { diff --git a/modules/files/files_test.go b/modules/files/files_test.go index 520567479..cf81d2f2e 100644 --- a/modules/files/files_test.go +++ b/modules/files/files_test.go @@ -48,10 +48,11 @@ func TestIsExistingDir(t *testing.T) { assert.True(t, IsExistingDir(currentFileDir)) } -func TestCopyFolderToTemp(t *testing.T) { +func TestCopyFolderToDest(t *testing.T) { t.Parallel() tempFolderPrefix := "someprefix" + destFolder := os.TempDir() tmpDir, err := ioutil.TempDir("", "TestCopyFolderContents") require.NoError(t, err) @@ -59,11 +60,11 @@ func TestCopyFolderToTemp(t *testing.T) { return !PathContainsHiddenFileOrFolder(path) && !PathContainsTerraformState(path) } - folder, err := CopyFolderToTemp("/not/a/real/path", tempFolderPrefix, filter) + folder, err := CopyFolderToDest("/not/a/real/path", destFolder, tempFolderPrefix, filter) require.Error(t, err) assert.False(t, FileExists(folder)) - folder, err = CopyFolderToTemp(tmpDir, tempFolderPrefix, filter) + folder, err = CopyFolderToDest(tmpDir, destFolder, tempFolderPrefix, filter) assert.DirExists(t, folder) assert.NoError(t, err) } @@ -161,6 +162,19 @@ func TestCopyTerraformFolderToTemp(t *testing.T) { requireDirectoriesEqual(t, expectedDir, tmpDir) } +func TestCopyTerraformFolderToDest(t *testing.T) { + t.Parallel() + + originalDir := filepath.Join(copyFolderContentsFixtureRoot, "original") + expectedDir := filepath.Join(copyFolderContentsFixtureRoot, "no-hidden-files-no-terraform-files") + destFolder := os.TempDir() + + tmpDir, err := CopyTerraformFolderToDest(originalDir, destFolder, "TestCopyTerraformFolderToTemp") + require.NoError(t, err) + + requireDirectoriesEqual(t, expectedDir, tmpDir) +} + func TestCopyTerragruntFolderToTemp(t *testing.T) { t.Parallel() @@ -173,6 +187,19 @@ func TestCopyTerragruntFolderToTemp(t *testing.T) { requireDirectoriesEqual(t, expectedDir, tmpDir) } +func TestCopyTerragruntFolderToDest(t *testing.T) { + t.Parallel() + + originalDir := filepath.Join(copyFolderContentsFixtureRoot, "terragrunt-files") + expectedDir := filepath.Join(copyFolderContentsFixtureRoot, "no-state-files") + destFolder := os.TempDir() + + tmpDir, err := CopyTerragruntFolderToDest(originalDir, destFolder, t.Name()) + require.NoError(t, err) + + requireDirectoriesEqual(t, expectedDir, tmpDir) +} + func TestPathContainsTerraformStateOrVars(t *testing.T) { var data = []struct { desc string diff --git a/modules/test-structure/test_structure.go b/modules/test-structure/test_structure.go index dcc5678e7..bd9706794 100644 --- a/modules/test-structure/test_structure.go +++ b/modules/test-structure/test_structure.go @@ -97,6 +97,64 @@ func CopyTerraformFolderToTemp(t testing.TestingT, rootFolder string, terraformM return tmpTestFolder } +// CopyTerraformFolderToDest copies the given root folder to a randomly-named temp folder and return the path to the +// given terraform modules folder within the new temp root folder. This is useful when running multiple tests in +// parallel against the same set of Terraform files to ensure the tests don't overwrite each other's .terraform working +// directory and terraform.tfstate files. To ensure relative paths work, we copy over the entire root folder to a temp +// folder, and then return the path within that temp folder to the given terraform module dir, which is where the actual +// test will be running. +// For example, suppose you had the target terraform folder you want to test in "/examples/terraform-aws-example" +// relative to the repo root. If your tests reside in the "/test" relative to the root, then you will use this as +// follows: +// +// // Destination for the copy of the files. In this example we are using the Azure Dev Ops variable +// // for the folder that is cleaned after each pipeline job. +// destRootFolder := os.Getenv("AGENT_TEMPDIRECTORY") +// +// // Root folder where terraform files should be (relative to the test folder) +// rootFolder := ".." +// +// // Relative path to terraform module being tested from the root folder +// terraformFolderRelativeToRoot := "examples/terraform-aws-example" +// +// // Copy the terraform folder to a temp folder +// tempTestFolder := test_structure.CopyTerraformFolderToTemp(t, destRootFolder, rootFolder, terraformFolderRelativeToRoot) +// +// // Make sure to use the temp test folder in the terraform options +// terraformOptions := &terraform.Options{ +// TerraformDir: tempTestFolder, +// } +// +// Note that if any of the SKIP_ environment variables is set, we assume this is a test in the local dev where +// there are no other concurrent tests running and we want to be able to cache test data between test stages, so in that +// case, we do NOT copy anything to a temp folder, and return the path to the original terraform module folder instead. +func CopyTerraformFolderToDest(t testing.TestingT, rootFolder string, terraformModuleFolder string, destRootFolder string) string { + if SkipStageEnvVarSet() { + logger.Logf(t, "A SKIP_XXX environment variable is set. Using original examples folder rather than a temp folder so we can cache data between stages for faster local testing.") + return filepath.Join(rootFolder, terraformModuleFolder) + } + + fullTerraformModuleFolder := filepath.Join(rootFolder, terraformModuleFolder) + + exists, err := files.FileExistsE(fullTerraformModuleFolder) + require.NoError(t, err) + if !exists { + t.Fatal(files.DirNotFoundError{Directory: fullTerraformModuleFolder}) + } + + tmpRootFolder, err := files.CopyTerraformFolderToDest(rootFolder, cleanName(t.Name()), destRootFolder) + if err != nil { + t.Fatal(err) + } + + tmpTestFolder := filepath.Join(tmpRootFolder, terraformModuleFolder) + + // Log temp folder so we can see it + logger.Logf(t, "Copied terraform folder %s to %s", fullTerraformModuleFolder, tmpTestFolder) + + return tmpTestFolder +} + func cleanName(originalName string) string { parts := strings.Split(originalName, "/") return parts[len(parts)-1]