diff --git a/cli/cli_app.go b/cli/cli_app.go index 4fd4f3aa9d..93573c2352 100644 --- a/cli/cli_app.go +++ b/cli/cli_app.go @@ -75,12 +75,6 @@ var ALL_TERRAGRUNT_STRING_OPTS = []string{ OPT_TERRAGRUNT_LOGLEVEL, } -const CMD_PLAN_ALL = "plan-all" -const CMD_APPLY_ALL = "apply-all" -const CMD_DESTROY_ALL = "destroy-all" -const CMD_OUTPUT_ALL = "output-all" -const CMD_VALIDATE_ALL = "validate-all" - const CMD_INIT = "init" const CMD_INIT_FROM_MODULE = "init-from-module" const CMD_PROVIDERS = "providers" @@ -91,18 +85,63 @@ const CMD_TERRAGRUNT_READ_CONFIG = "terragrunt-read-config" const CMD_HCLFMT = "hclfmt" const CMD_AWS_PROVIDER_PATCH = "aws-provider-patch" -// CMD_SPIN_UP is deprecated. -const CMD_SPIN_UP = "spin-up" +// START: Constants useful for multimodule command handling +const CMD_RUN_ALL = "run-all" + +// Known terraform commands that are explicitly not supported in run-all due to the nature of the command. This is +// tracked as a map that maps the terraform command to the reasoning behind disallowing the command in run-all. +var runAllDisabledCommands = map[string]string{ + "import": "terraform import should only be run against a single state representation to avoid injecting the wrong object in the wrong state representation.", + "taint": "terraform taint should only be run against a single state representation to avoid using the wrong state address.", + "untaint": "terraform untaint should only be run against a single state representation to avoid using the wrong state address.", + "console": "terraform console requires stdin, which is shared across all instances of run-all when multiple modules run concurrently.", + "force-unlock": "lock IDs are unique per state representation and thus should not be run with run-all.", + + // MAINTAINER'S NOTE: There are a few other commands that might not make sense, but we deliberately allow it for + // certain use cases that are documented here: + // - state : Supporting `state` with run-all could be useful for a mass pull and push operation, which can + // be done en masse with the use of relative pathing. + // - login / logout : Supporting `login` with run-all could be useful when used in conjunction with tfenv and + // multi-terraform version setups, where multiple terraform versions need to be configured. + // - version : Supporting `version` with run-all could be useful for sanity checking a multi-version setup. +} + +var MULTI_MODULE_COMMANDS = []string{ + CMD_RUN_ALL, -// CMD_TEAR_DOWN is deprecated. -const CMD_TEAR_DOWN = "tear-down" + // The rest of the commands are deprecated, and are only here for legacy reasons to ensure that terragrunt knows to + // filter them out during arg parsing. + CMD_APPLY_ALL, + CMD_DESTROY_ALL, + CMD_OUTPUT_ALL, + CMD_PLAN_ALL, + CMD_VALIDATE_ALL, +} -var MULTI_MODULE_COMMANDS = []string{CMD_APPLY_ALL, CMD_DESTROY_ALL, CMD_OUTPUT_ALL, CMD_PLAN_ALL, CMD_VALIDATE_ALL} +// END: Constants useful for multimodule command handling + +// The following commands are DEPRECATED +const ( + CMD_SPIN_UP = "spin-up" + CMD_TEAR_DOWN = "tear-down" + CMD_PLAN_ALL = "plan-all" + CMD_APPLY_ALL = "apply-all" + CMD_DESTROY_ALL = "destroy-all" + CMD_OUTPUT_ALL = "output-all" + CMD_VALIDATE_ALL = "validate-all" +) -// DEPRECATED_COMMANDS is a map of deprecated commands to the commands that replace them. -var DEPRECATED_COMMANDS = map[string]string{ - CMD_SPIN_UP: CMD_APPLY_ALL, - CMD_TEAR_DOWN: CMD_DESTROY_ALL, +// deprecatedCommands is a map of deprecated commands to a handler that knows how to convert the command to the known +// alternative. The handler should return the new TerragruntOptions (if any modifications are needed) and command +// string. +var deprecatedCommands = map[string]func(origOptions *options.TerragruntOptions) (*options.TerragruntOptions, string, string){ + CMD_SPIN_UP: spinUpDeprecationHandler, + CMD_TEAR_DOWN: tearDownDeprecationHandler, + CMD_APPLY_ALL: applyAllDeprecationHandler, + CMD_DESTROY_ALL: destroyAllDeprecationHandler, + CMD_PLAN_ALL: planAllDeprecationHandler, + CMD_VALIDATE_ALL: validateAllDeprecationHandler, + CMD_OUTPUT_ALL: outputAllDeprecationHandler, } var TERRAFORM_COMMANDS_THAT_USE_STATE = []string{ @@ -157,11 +196,7 @@ USAGE: {{.Usage}} COMMANDS: - plan-all Display the plans of a 'stack' by running 'terragrunt plan' in each subfolder - apply-all Apply a 'stack' by running 'terragrunt apply' in each subfolder - output-all Display the outputs of a 'stack' by running 'terragrunt output' in each subfolder - destroy-all Destroy a 'stack' by running 'terragrunt destroy' in each subfolder - validate-all Validate 'stack' by running 'terragrunt validate' in each subfolder + run-all Run a terraform command against a 'stack' by running the specified command in each subfolder. E.g., to run 'terragrunt apply' in each subfolder, use 'terragrunt run-all apply'. terragrunt-info Emits limited terragrunt state on stdout and exits graph-dependencies Prints the terragrunt dependency graph to stdout hclfmt Recursively find terragrunt.hcl files and rewrite them into a canonical format. @@ -255,25 +290,32 @@ func runApp(cliContext *cli.Context) (finalErr error) { shell.PrepareConsole(terragruntOptions) givenCommand := cliContext.Args().First() - command := checkDeprecated(givenCommand, terragruntOptions) - return runCommand(command, terragruntOptions) + newOptions, command := checkDeprecated(givenCommand, terragruntOptions) + return runCommand(command, newOptions) } // checkDeprecated checks if the given command is deprecated. If so: prints a message and returns the new command. -func checkDeprecated(command string, terragruntOptions *options.TerragruntOptions) string { - newCommand, deprecated := DEPRECATED_COMMANDS[command] +func checkDeprecated(command string, terragruntOptions *options.TerragruntOptions) (*options.TerragruntOptions, string) { + deprecationHandler, deprecated := deprecatedCommands[command] if deprecated { - terragruntOptions.Logger.Infof("%v is deprecated; running %v instead.\n", command, newCommand) - return newCommand - } - return command + newOptions, newCommand, newCommandFriendly := deprecationHandler(terragruntOptions) + terragruntOptions.Logger.Warnf( + "'%s' is deprecated. Running '%s' instead. Please update your workflows to use '%s', as '%s' may be removed in the future!\n", + command, + newCommandFriendly, + newCommandFriendly, + command, + ) + return newOptions, newCommand + } + return terragruntOptions, command } // runCommand runs one or many terraform commands based on the type of // terragrunt command func runCommand(command string, terragruntOptions *options.TerragruntOptions) (finalEff error) { - if isMultiModuleCommand(command) { - return runMultiModuleCommand(command, terragruntOptions) + if command == CMD_RUN_ALL { + return runAll(terragruntOptions) } return RunTerragrunt(terragruntOptions) } @@ -310,8 +352,10 @@ func RunTerragrunt(terragruntOptions *options.TerragruntOptions) error { } if terragruntConfig.Skip { - terragruntOptions.Logger.Infof("Skipping terragrunt module %s due to skip = true.", - terragruntOptions.TerragruntConfigPath) + terragruntOptions.Logger.Infof( + "Skipping terragrunt module %s due to skip = true.", + terragruntOptions.TerragruntConfigPath, + ) return nil } @@ -835,30 +879,6 @@ func prepareInitOptions(terragruntOptions *options.TerragruntOptions, terraformS return initOptions, nil } -// Returns true if the command the user wants to execute is supposed to affect multiple Terraform modules, such as the -// apply-all or destroy-all command. -func isMultiModuleCommand(command string) bool { - return util.ListContainsElement(MULTI_MODULE_COMMANDS, command) -} - -// Execute a command that affects multiple Terraform modules, such as the apply-all or destroy-all command. -func runMultiModuleCommand(command string, terragruntOptions *options.TerragruntOptions) error { - switch command { - case CMD_PLAN_ALL: - return planAll(terragruntOptions) - case CMD_APPLY_ALL: - return applyAll(terragruntOptions) - case CMD_DESTROY_ALL: - return destroyAll(terragruntOptions) - case CMD_OUTPUT_ALL: - return outputAll(terragruntOptions) - case CMD_VALIDATE_ALL: - return validateAll(terragruntOptions) - default: - return errors.WithStackTrace(UnrecognizedCommand(command)) - } -} - // Return true if modules aren't already downloaded and the Terraform templates in this project reference modules. // Note that to keep the logic in this code very simple, this code ONLY detects the case where you haven't downloaded // modules at all. Detecting if your downloaded modules are out of date (as opposed to missing entirely) is more @@ -884,81 +904,43 @@ func remoteStateNeedsInit(remoteState *remote.RemoteState, terragruntOptions *op return false, nil } -// planAll prints the plans from all configuration in a stack, in the order -// specified in the terraform_remote_state dependencies -func planAll(terragruntOptions *options.TerragruntOptions) error { - stack, err := configstack.FindStackInSubfolders(terragruntOptions) - if err != nil { - return err +// runAll runs the provided terraform command against all the modules that are found in the directory tree. +func runAll(terragruntOptions *options.TerragruntOptions) error { + reason, isDisabled := runAllDisabledCommands[terragruntOptions.TerraformCommand] + if isDisabled { + return RunAllDisabledErr{ + command: terragruntOptions.TerraformCommand, + reason: reason, + } } - terragruntOptions.Logger.Printf("%s", stack.String()) - return stack.Plan(terragruntOptions) -} - -// Spin up an entire "stack" by running 'terragrunt apply' in each subfolder, processing them in the right order based -// on terraform_remote_state dependencies. -func applyAll(terragruntOptions *options.TerragruntOptions) error { stack, err := configstack.FindStackInSubfolders(terragruntOptions) if err != nil { return err } - terragruntOptions.Logger.Printf("%s", stack.String()) - shouldApplyAll, err := shell.PromptUserForYesNo("Are you sure you want to run 'terragrunt apply' in each folder of the stack described above?", terragruntOptions) - if err != nil { - return err - } - - if shouldApplyAll { - return stack.Apply(terragruntOptions) - } - - return nil -} + terragruntOptions.Logger.Infof("%s", stack.String()) -// Tear down an entire "stack" by running 'terragrunt destroy' in each subfolder, processing them in the right order -// based on terraform_remote_state dependencies. -func destroyAll(terragruntOptions *options.TerragruntOptions) error { - stack, err := configstack.FindStackInSubfolders(terragruntOptions) - if err != nil { - return err + var prompt string + switch terragruntOptions.TerraformCommand { + case "apply": + prompt = "Are you sure you want to run 'terragrunt apply' in each folder of the stack described above?" + case "destroy": + prompt = "WARNING: Are you sure you want to run `terragrunt destroy` in each folder of the stack described above? There is no undo!" + case "state": + prompt = "Are you sure you want to manipulate the state with `terragrunt state` in each folder of the stack described above? Note that absolute paths are shared, while relative paths will be relative to each working directory." } - - terragruntOptions.Logger.Printf("%s", stack.String()) - shouldDestroyAll, err := shell.PromptUserForYesNo("WARNING: Are you sure you want to run `terragrunt destroy` in each folder of the stack described above? There is no undo!", terragruntOptions) - if err != nil { - return err - } - - if shouldDestroyAll { - return stack.Destroy(terragruntOptions) - } - - return nil -} - -// outputAll prints the outputs from all configuration in a stack, in the order -// specified in the terraform_remote_state dependencies -func outputAll(terragruntOptions *options.TerragruntOptions) error { - stack, err := configstack.FindStackInSubfolders(terragruntOptions) - if err != nil { - return err - } - - terragruntOptions.Logger.Printf("%s", stack.String()) - return stack.Output(terragruntOptions) -} - -// validateAll validates runs terraform validate on all the modules -func validateAll(terragruntOptions *options.TerragruntOptions) error { - stack, err := configstack.FindStackInSubfolders(terragruntOptions) - if err != nil { - return err + if prompt != "" { + shouldRunAll, err := shell.PromptUserForYesNo(prompt, terragruntOptions) + if err != nil { + return err + } + if shouldRunAll == false { + return nil + } } - terragruntOptions.Logger.Printf("%s", stack.String()) - return stack.Validate(terragruntOptions) + return stack.Run(terragruntOptions) } // checkProtectedModule checks if module is protected via the "prevent_destroy" flag @@ -1033,3 +1015,12 @@ type MaxRetriesExceeded struct { func (err MaxRetriesExceeded) Error() string { return fmt.Sprintf("Exhausted retries (%v) for command %v %v", err.Opts.MaxRetryAttempts, err.Opts.TerraformPath, strings.Join(err.Opts.TerraformCliArgs, " ")) } + +type RunAllDisabledErr struct { + command string + reason string +} + +func (err RunAllDisabledErr) Error() string { + return fmt.Sprintf("%s with run-all is disabled: %s", err.command, err.reason) +} diff --git a/cli/deprecated_handler.go b/cli/deprecated_handler.go new file mode 100644 index 0000000000..8d6827ff63 --- /dev/null +++ b/cli/deprecated_handler.go @@ -0,0 +1,61 @@ +package cli + +import ( + "fmt" + "strings" + + "github.com/gruntwork-io/terragrunt/options" +) + +func spinUpDeprecationHandler(origOptions *options.TerragruntOptions) (*options.TerragruntOptions, string, string) { + return origOptions, CMD_APPLY_ALL, CMD_APPLY_ALL +} + +func tearDownDeprecationHandler(origOptions *options.TerragruntOptions) (*options.TerragruntOptions, string, string) { + return origOptions, CMD_DESTROY_ALL, CMD_DESTROY_ALL +} + +func applyAllDeprecationHandler(origOptions *options.TerragruntOptions) (*options.TerragruntOptions, string, string) { + opts := origOptions.Clone(origOptions.TerragruntConfigPath) + opts.TerraformCommand = "apply" + opts.OriginalTerraformCommand = "apply" + opts.TerraformCliArgs = append([]string{"apply"}, opts.TerraformCliArgs...) + newCmdFriendly := fmt.Sprintf("terragrunt run-all %s", strings.Join(opts.TerraformCliArgs, " ")) + return opts, CMD_RUN_ALL, newCmdFriendly +} + +func destroyAllDeprecationHandler(origOptions *options.TerragruntOptions) (*options.TerragruntOptions, string, string) { + opts := origOptions.Clone(origOptions.TerragruntConfigPath) + opts.TerraformCommand = "destroy" + opts.OriginalTerraformCommand = "destroy" + opts.TerraformCliArgs = append([]string{"destroy"}, opts.TerraformCliArgs...) + newCmdFriendly := fmt.Sprintf("terragrunt run-all %s", strings.Join(opts.TerraformCliArgs, " ")) + return opts, CMD_RUN_ALL, newCmdFriendly +} + +func planAllDeprecationHandler(origOptions *options.TerragruntOptions) (*options.TerragruntOptions, string, string) { + opts := origOptions.Clone(origOptions.TerragruntConfigPath) + opts.TerraformCommand = "plan" + opts.OriginalTerraformCommand = "plan" + opts.TerraformCliArgs = append([]string{"plan"}, opts.TerraformCliArgs...) + newCmdFriendly := fmt.Sprintf("terragrunt run-all %s", strings.Join(opts.TerraformCliArgs, " ")) + return opts, CMD_RUN_ALL, newCmdFriendly +} + +func validateAllDeprecationHandler(origOptions *options.TerragruntOptions) (*options.TerragruntOptions, string, string) { + opts := origOptions.Clone(origOptions.TerragruntConfigPath) + opts.TerraformCommand = "validate" + opts.OriginalTerraformCommand = "validate" + opts.TerraformCliArgs = append([]string{"validate"}, opts.TerraformCliArgs...) + newCmdFriendly := fmt.Sprintf("terragrunt run-all %s", strings.Join(opts.TerraformCliArgs, " ")) + return opts, CMD_RUN_ALL, newCmdFriendly +} + +func outputAllDeprecationHandler(origOptions *options.TerragruntOptions) (*options.TerragruntOptions, string, string) { + opts := origOptions.Clone(origOptions.TerragruntConfigPath) + opts.TerraformCommand = "output" + opts.OriginalTerraformCommand = "output" + opts.TerraformCliArgs = append([]string{"output"}, opts.TerraformCliArgs...) + newCmdFriendly := fmt.Sprintf("terragrunt run-all %s", strings.Join(opts.TerraformCliArgs, " ")) + return opts, CMD_RUN_ALL, newCmdFriendly +} diff --git a/configstack/stack.go b/configstack/stack.go index 03e441c973..145eb8d844 100644 --- a/configstack/stack.go +++ b/configstack/stack.go @@ -34,19 +34,38 @@ func (stack *Stack) Graph(terragruntOptions *options.TerragruntOptions) { WriteDot(terragruntOptions.Writer, terragruntOptions, stack.Modules) } -// Plan execute plan in the given stack in their specified order. -func (stack *Stack) Plan(terragruntOptions *options.TerragruntOptions) error { - stack.setTerraformCommand([]string{"plan"}) - - // We capture the out stream for each module - errorStreams := make([]bytes.Buffer, len(stack.Modules)) - for n, module := range stack.Modules { - module.TerragruntOptions.ErrWriter = &errorStreams[n] +func (stack *Stack) Run(terragruntOptions *options.TerragruntOptions) error { + stackCmd := terragruntOptions.TerraformCommand + + // For any command that needs input, run in non-interactive mode to avoid cominglint stdin across multiple + // concurrent runs. + if util.ListContainsElement(config.TERRAFORM_COMMANDS_NEED_INPUT, stackCmd) { + terragruntOptions.TerraformCliArgs = append(terragruntOptions.TerraformCliArgs, "-input=false") + stack.syncTerraformCliArgs(terragruntOptions) + } + + // For apply and destroy, run with auto-approve due to the co-mingling of the prompts. This is not ideal, but until + // we have a better way of handling interactivity with run-all, we take the evil of having a global prompt (managed + // in cli/cli_app.go) be the gate keeper. + switch stackCmd { + case "apply", "destroy": + terragruntOptions.TerraformCliArgs = append(terragruntOptions.TerraformCliArgs, "-auto-approve") + stack.syncTerraformCliArgs(terragruntOptions) + } + + if stackCmd == "plan" { + // We capture the out stream for each module + errorStreams := make([]bytes.Buffer, len(stack.Modules)) + for n, module := range stack.Modules { + module.TerragruntOptions.ErrWriter = &errorStreams[n] + } + defer stack.summarizePlanAllErrors(terragruntOptions, errorStreams) } - defer stack.summarizePlanAllErrors(terragruntOptions, errorStreams) if terragruntOptions.IgnoreDependencyOrder { return RunModulesIgnoreOrder(stack.Modules, terragruntOptions.Parallelism) + } else if stackCmd == "destroy" { + return RunModulesReverseOrder(stack.Modules, terragruntOptions.Parallelism) } else { return RunModules(stack.Modules, terragruntOptions.Parallelism) } @@ -75,52 +94,6 @@ func (stack *Stack) summarizePlanAllErrors(terragruntOptions *options.Terragrunt } } -// Apply all the modules in the given stack, making sure to apply the dependencies of each module in the stack in the -// proper order. -func (stack *Stack) Apply(terragruntOptions *options.TerragruntOptions) error { - stack.setTerraformCommand([]string{"apply", "-input=false", "-auto-approve"}) - - if terragruntOptions.IgnoreDependencyOrder { - return RunModulesIgnoreOrder(stack.Modules, terragruntOptions.Parallelism) - } else { - return RunModules(stack.Modules, terragruntOptions.Parallelism) - } -} - -// Destroy all the modules in the given stack, making sure to destroy the dependencies of each module in the stack in -// the proper order. -func (stack *Stack) Destroy(terragruntOptions *options.TerragruntOptions) error { - stack.setTerraformCommand([]string{"destroy", "-force", "-input=false"}) - - if terragruntOptions.IgnoreDependencyOrder { - return RunModulesIgnoreOrder(stack.Modules, terragruntOptions.Parallelism) - } else { - return RunModulesReverseOrder(stack.Modules, terragruntOptions.Parallelism) - } -} - -// Output prints the outputs of all the modules in the given stack in their specified order. -func (stack *Stack) Output(terragruntOptions *options.TerragruntOptions) error { - stack.setTerraformCommand([]string{"output"}) - - if terragruntOptions.IgnoreDependencyOrder { - return RunModulesIgnoreOrder(stack.Modules, terragruntOptions.Parallelism) - } else { - return RunModules(stack.Modules, terragruntOptions.Parallelism) - } -} - -// Validate runs terraform validate on each module -func (stack *Stack) Validate(terragruntOptions *options.TerragruntOptions) error { - stack.setTerraformCommand([]string{"validate"}) - - if terragruntOptions.IgnoreDependencyOrder { - return RunModulesIgnoreOrder(stack.Modules, terragruntOptions.Parallelism) - } else { - return RunModules(stack.Modules, terragruntOptions.Parallelism) - } -} - // Return an error if there is a dependency cycle in the modules of this stack. func (stack *Stack) CheckForCycles() error { return CheckForCycles(stack.Modules) @@ -138,12 +111,10 @@ func FindStackInSubfolders(terragruntOptions *options.TerragruntOptions) (*Stack return createStackForTerragruntConfigPaths(terragruntOptions.WorkingDir, terragruntConfigFiles, terragruntOptions, howThesePathsWereFound) } -// Set the command in the TerragruntOptions object of each module in this stack to the given command. -func (stack *Stack) setTerraformCommand(command []string) { +// Sync the TerraformCliArgs for each module in the stack to match the provided terragruntOptions struct. +func (stack *Stack) syncTerraformCliArgs(terragruntOptions *options.TerragruntOptions) { for _, module := range stack.Modules { - module.TerragruntOptions.TerraformCliArgs = append(command, module.TerragruntOptions.TerraformCliArgs...) - module.TerragruntOptions.OriginalTerraformCommand = util.FirstArg(command) - module.TerragruntOptions.TerraformCommand = util.FirstArg(command) + module.TerragruntOptions.TerraformCliArgs = terragruntOptions.TerraformCliArgs } } diff --git a/docs/_docs/01_getting-started/configuration.md b/docs/_docs/01_getting-started/configuration.md index 3f8662f7d6..33b75e7ed3 100644 --- a/docs/_docs/01_getting-started/configuration.md +++ b/docs/_docs/01_getting-started/configuration.md @@ -76,7 +76,7 @@ Note that the parsing order is slightly different when using the `-all` flavors The results of this pass are then used to build the dependency graph of the modules in the tree. Once the graph is constructed, Terragrunt will loop through the modules and run the specified command. It will then revert to the single configuration parsing order specified above for each module as it runs the command. -This allows Terragrunt to avoid resolving `dependency` on modules that haven’t been applied yet when doing a clean deployment from scratch with `apply-all`. +This allows Terragrunt to avoid resolving `dependency` on modules that haven’t been applied yet when doing a clean deployment from scratch with `run-all apply`. ## Formatting terragrunt.hcl diff --git a/docs/_docs/02_features/caching.md b/docs/_docs/02_features/caching.md index 390b5343c2..a888466fb5 100644 --- a/docs/_docs/02_features/caching.md +++ b/docs/_docs/02_features/caching.md @@ -14,7 +14,7 @@ nav_title_link: /docs/ Terragrunt creates a `.terragrunt-cache` folder in the current working directory as its scratch directory. It downloads your remote Terraform configurations into this folder, runs your Terraform commands in this folder, and any modules and providers those commands download also get stored in this folder. You can safely delete this folder any time and Terragrunt will recreate it as necessary. -If you need to clean up a lot of these folders (e.g., after `terragrunt apply-all`), you can use the following commands on Mac and Linux: +If you need to clean up a lot of these folders (e.g., after `terragrunt run-all apply`), you can use the following commands on Mac and Linux: Recursively find all the `.terragrunt-cache` folders that are children of the current folder: diff --git a/docs/_docs/02_features/execute-terraform-commands-on-multiple-modules-at-once.md b/docs/_docs/02_features/execute-terraform-commands-on-multiple-modules-at-once.md index 9b556d00c1..702ca4ffe1 100644 --- a/docs/_docs/02_features/execute-terraform-commands-on-multiple-modules-at-once.md +++ b/docs/_docs/02_features/execute-terraform-commands-on-multiple-modules-at-once.md @@ -14,14 +14,14 @@ nav_title_link: /docs/ - [Motivation](#motivation) - - [The apply-all, destroy-all, output-all and plan-all commands](#the-apply-all-destroy-all-output-all-and-plan-all-commands) + - [The run-all command](#the-run-all-command) - [Passing outputs between modules](#passing-outputs-between-modules) - [Dependencies between modules](#dependencies-between-modules) - [Testing multiple modules locally](#testing-multiple-modules-locally) - + - [Limiting the module execution parallelism](#limiting-the-module-execution-parallelism) ### Motivation @@ -42,7 +42,7 @@ Let’s say your infrastructure is defined across multiple Terraform modules: There is one module to deploy a frontend-app, another to deploy a backend-app, another for the MySQL database, and so on. To deploy such an environment, you’d have to manually run `terraform apply` in each of the subfolder, wait for it to complete, and then run `terraform apply` in the next subfolder. How do you avoid this tedious and time-consuming process? -### The apply-all, destroy-all, output-all and plan-all commands +### The run-all command To be able to deploy multiple Terraform modules in a single command, add a `terragrunt.hcl` file to each module: @@ -63,35 +63,37 @@ To be able to deploy multiple Terraform modules in a single command, add a `terr ├── main.tf └── terragrunt.hcl -Now you can go into the `root` folder and deploy all the modules within it by using the `apply-all` command: +Now you can go into the `root` folder and deploy all the modules within it by using the `run-all` command with +`apply`: cd root - terragrunt apply-all + terragrunt run-all apply When you run this command, Terragrunt will recursively look through all the subfolders of the current working directory, find all folders with a `terragrunt.hcl` file, and run `terragrunt apply` in each of those folders concurrently. -Similarly, to undeploy all the Terraform modules, you can use the `destroy-all` command: +Similarly, to undeploy all the Terraform modules, you can use the `run-all` command with `destroy`: cd root - terragrunt destroy-all + terragrunt run-all destroy -To see the currently applied outputs of all of the subfolders, you can use the `output-all` command: +To see the currently applied outputs of all of the subfolders, you can use the `run-all` command with `output`: cd root - terragrunt output-all + terragrunt run-all output -Finally, if you make some changes to your project, you could evaluate the impact by using `plan-all` command: +Finally, if you make some changes to your project, you could evaluate the impact by using `run-all` command with `plan`: -Note: It is important to realize that you could get errors running `plan-all` if you have dependencies between your projects and some of those dependencies haven’t been applied yet. +Note: It is important to realize that you could get errors running `run-all plan` if you have dependencies between your +projects and some of those dependencies haven’t been applied yet. *Ex: If module A depends on module B and module B hasn’t been applied yet, then plan-all will show the plan for B, but exit with an error when trying to show the plan for A.* cd root - terragrunt plan-all + terragrunt run-all plan If your modules have dependencies between them—for example, you can’t deploy the backend-app until MySQL and redis are deployed—you’ll need to express those dependencies in your Terragrunt configuration as explained in the next section. -Additional note: If your modules have dependencies between them, and you run a `terragrunt destroy-all` command, Terragrunt will destroy all the modules under the current working directory, *as well as each of the module dependencies* (that is, modules you depend on via `dependencies` and `dependency` blocks)! If you wish to use exclude dependencies from being destroyed, add the `--terragrunt-ignore-external-dependencies` flag, or use the `--terragrunt-exclude-dir` once for each directory you wish to exclude. +Additional note: If your modules have dependencies between them, and you run a `terragrunt run-all destroy` command, Terragrunt will destroy all the modules under the current working directory, *as well as each of the module dependencies* (that is, modules you depend on via `dependencies` and `dependency` blocks)! If you wish to use exclude dependencies from being destroyed, add the `--terragrunt-ignore-external-dependencies` flag, or use the `--terragrunt-exclude-dir` once for each directory you wish to exclude. ### Passing outputs between modules @@ -142,7 +144,7 @@ You can also specify multiple `dependency` blocks to access multiple different m redis_url = dependency.redis.outputs.domain } -Note that each `dependency` is automatically considered a dependency in Terragrunt. This means that when you run `apply-all` on a config that has `dependency` blocks, Terragrunt will not attempt to deploy the config until all the modules referenced in `dependency` blocks have been applied. So for the above example, the order for the `apply-all` command would be: +Note that each `dependency` is automatically considered a dependency in Terragrunt. This means that when you run `run-all apply` on a config that has `dependency` blocks, Terragrunt will not attempt to deploy the config until all the modules referenced in `dependency` blocks have been applied. So for the above example, the order for the `run-all apply` command would be: 1. Deploy the VPC @@ -264,7 +266,7 @@ dependencies { } ``` -Once you’ve specified the dependencies in each `terragrunt.hcl` file, when you run the `terragrunt apply-all` or `terragrunt destroy-all`, Terragrunt will ensure that the dependencies are applied or destroyed, respectively, in the correct order. For the example at the start of this section, the order for the `apply-all` command would be: +Once you’ve specified the dependencies in each `terragrunt.hcl` file, when you run the `terragrunt run-all apply` or `terragrunt run-all destroy`, Terragrunt will ensure that the dependencies are applied or destroyed, respectively, in the correct order. For the example at the start of this section, the order for the `run-all apply` command would be: 1. Deploy the VPC @@ -274,7 +276,7 @@ Once you’ve specified the dependencies in each `terragrunt.hcl` file, when you 4. Deploy the frontend-app -If any of the modules fail to deploy, then Terragrunt will not attempt to deploy the modules that depend on them. Once you’ve fixed the error, it’s usually safe to re-run the `apply-all` or `destroy-all` command again, since it’ll be a no-op for the modules that already deployed successfully, and should only affect the ones that had an error the last time around. +If any of the modules fail to deploy, then Terragrunt will not attempt to deploy the modules that depend on them. Once you’ve fixed the error, it’s usually safe to re-run the `run-all apply` or `run-all destroy` command again, since it’ll be a no-op for the modules that already deployed successfully, and should only affect the ones that had an error the last time around. To check all of your dependencies and validate the code in them, you can use the `validate-all` command. @@ -295,9 +297,9 @@ in reverse order (bottom up) If you are using Terragrunt to configure [remote Terraform configurations]({{site.baseurl}}/docs/features/keep-your-terraform-code-dry/#remote-terraform-configurations) and all of your modules have the `source` parameter set to a Git URL, but you want to test with a local checkout of the code, you can use the `--terragrunt-source` parameter: cd root - terragrunt plan-all --terragrunt-source /source/modules + terragrunt run-all plan --terragrunt-source /source/modules -If you set the `--terragrunt-source` parameter, the `xxx-all` commands will assume that parameter is pointing to a folder on your local file system that has a local checkout of all of your Terraform modules. For each module that is being processed via a `xxx-all` command, Terragrunt will read in the `source` parameter in that module’s `terragrunt.hcl` file, parse out the path (the portion after the double-slash), and append the path to the `--terragrunt-source` parameter to create the final local path for that module. +If you set the `--terragrunt-source` parameter, the `run-all` commands will assume that parameter is pointing to a folder on your local file system that has a local checkout of all of your Terraform modules. For each module that is being processed via a `run-all` command, Terragrunt will read in the `source` parameter in that module’s `terragrunt.hcl` file, parse out the path (the portion after the double-slash), and append the path to the `--terragrunt-source` parameter to create the final local path for that module. For example, consider the following `terragrunt.hcl` file: @@ -307,7 +309,7 @@ terraform { } ``` -If you run `terragrunt apply-all --terragrunt-source /source/infrastructure-modules`, then the local path Terragrunt will compute for the module above will be `/source/infrastructure-modules//networking/vpc`. +If you run `terragrunt run-all apply --terragrunt-source /source/infrastructure-modules`, then the local path Terragrunt will compute for the module above will be `/source/infrastructure-modules//networking/vpc`. ### Limiting the module execution parallelism @@ -319,5 +321,5 @@ cloud provider. To limit the maximum number of module executions at any given time use the `--terragrunt-parallelism [number]` flag ```sh -terragrunt apply-all --terragrunt-parallelism 4 +terragrunt run-all apply --terragrunt-parallelism 4 ``` diff --git a/docs/_docs/02_features/keep-your-cli-flags-dry.md b/docs/_docs/02_features/keep-your-cli-flags-dry.md index b0fcf8ef94..d092d1bd75 100644 --- a/docs/_docs/02_features/keep-your-cli-flags-dry.md +++ b/docs/_docs/02_features/keep-your-cli-flags-dry.md @@ -179,17 +179,17 @@ terraform { See the [get\_terragrunt\_dir()]({{site.baseurl}}/docs/reference/built-in-functions/#get_terragrunt_dir) and [get\_parent\_terragrunt\_dir()]({{site.baseurl}}/docs/reference/built-in-functions/#get_parent_terragrunt_dir) documentation for more details. -With the configuration above, when you run `terragrunt apply-all`, Terragrunt will call Terraform as follows: +With the configuration above, when you run `terragrunt run-all apply`, Terragrunt will call Terraform as follows: - $ terragrunt apply-all + $ terragrunt run-all apply [backend-app] terraform apply -var-file=/my/tf/terraform.tfvars -var-file=/my/tf/backend-app/dev.tfvars [frontend-app] terraform apply -var-file=/my/tf/terraform.tfvars -var-file=/my/tf/frontend-app/us-east-1.tfvars - $ TF_VAR_env=prod terragrunt apply-all + $ TF_VAR_env=prod terragrunt run-all apply [backend-app] terraform apply -var-file=/my/tf/terraform.tfvars -var-file=/my/tf/prod.tfvars [frontend-app] terraform apply -var-file=/my/tf/terraform.tfvars -var-file=/my/tf/prod.tfvars -var-file=/my/tf/frontend-app/us-east-1.tfvars - $ TF_VAR_env=prod TF_VAR_region=us-west-2 terragrunt apply-all + $ TF_VAR_env=prod TF_VAR_region=us-west-2 terragrunt run-all apply [backend-app] terraform apply -var-file=/my/tf/terraform.tfvars -var-file=/my/tf/prod.tfvars -var-file=/my/tf/us-west-2.tfvars [frontend-app] terraform apply -var-file=/my/tf/terraform.tfvars -var-file=/my/tf/prod.tfvars -var-file=/my/tf/us-west-2.tfvars diff --git a/docs/_docs/04_reference/cli-options.md b/docs/_docs/04_reference/cli-options.md index 8812455a14..7c2ea53ad4 100644 --- a/docs/_docs/04_reference/cli-options.md +++ b/docs/_docs/04_reference/cli-options.md @@ -24,11 +24,12 @@ This page documents the CLI commands and options available with Terragrunt: Terragrunt supports the following CLI commands: - [All Terraform built-in commands](#all-terraform-built-in-commands) - - [plan-all](#plan-all) - - [apply-all](#apply-all) - - [output-all](#output-all) - - [destroy-all](#destroy-all) - - [validate-all](#validate-all) + - [run-all](#run-all) + - [plan-all (DEPRECATED: use run-all)](#plan-all-deprecated-use-run-all) + - [apply-all (DEPRECATED: use run-all)](#apply-all-deprecated-use-run-all) + - [output-all (DEPRECATED: use run-all)](#output-all-deprecated-use-run-all) + - [destroy-all (DEPRECATED: use run-all)](#destroy-all-deprecated-use-run-all) + - [validate-all (DEPRECATED: use run-all)](#validate-all-deprecated-use-run-all) - [terragrunt-info](#terragrunt-info) - [graph-dependencies](#graph-dependencies) - [hclfmt](#hclfmt) @@ -52,7 +53,42 @@ terragrunt destroy Run `terraform --help` to get the full list. -### plan-all + +### run-all + +Runs the provided terraform command against a 'stack', where a 'stack' is a +tree of terragrunt modules. The command will recursively find terragrunt +modules in the current directory tree and run the terraform command in +dependency order (unless the command is destroy, in which case the command is +run in reverse dependency order). + +Make sure to read [Execute Terraform +commands on multiple modules at once](/docs/features/execute-terraform-commands-on-multiple-modules-at-once/) for +context. + +Example: + +```bash +terragrunt run-all apply +``` + +This will recursively search the current working directory for any folders that contain Terragrunt modules and run +`apply` in each one, concurrently, while respecting ordering defined via +[`dependency`](/docs/reference/config-blocks-and-attributes/#dependency) and +[`dependencies`](/docs/reference/config-blocks-and-attributes/#dependencies) blocks. + +**[WARNING] Using `run-all` with `plan` is currently broken for certain use cases**. If you have a stack of Terragrunt modules with +dependencies between them—either via `dependency` blocks or `terraform_remote_state` data sources—and you've never +deployed them, then `plan-all` will fail as it will not be possible to resolve the `dependency` blocks or +`terraform_remote_state` data sources! Please [see here for more +information](https://github.com/gruntwork-io/terragrunt/issues/720#issuecomment-497888756). + + + + +### plan-all (DEPRECATED: use run-all) + +**DEPRECATED: Use `run-all plan` instead.** Display the plans of a 'stack' by running 'terragrunt plan' in each subfolder. Make sure to read [Execute Terraform commands on multiple modules at once](/docs/features/execute-terraform-commands-on-multiple-modules-at-once/) for @@ -76,7 +112,9 @@ deployed them, then `plan-all` will fail as it will not be possible to resolve t information](https://github.com/gruntwork-io/terragrunt/issues/720#issuecomment-497888756). -### apply-all +### apply-all (DEPRECATED: use run-all) + +**DEPRECATED: Use `run-all apply` instead.** Apply a 'stack' by running 'terragrunt apply' in each subfolder. Make sure to read [Execute Terraform commands on multiple modules at once](/docs/features/execute-terraform-commands-on-multiple-modules-at-once/) for @@ -93,7 +131,9 @@ This will recursively search the current working directory for any folders that [`dependency`](/docs/reference/config-blocks-and-attributes/#dependency) and [`dependencies`](/docs/reference/config-blocks-and-attributes/#dependencies) blocks. -### output-all +### output-all (DEPRECATED: use run-all) + +**DEPRECATED: Use `run-all output` instead.** Display the outputs of a 'stack' by running 'terragrunt output' in each subfolder. Make sure to read [Execute Terraform commands on multiple modules at once](/docs/features/execute-terraform-commands-on-multiple-modules-at-once/) for @@ -116,7 +156,9 @@ deployed them, then `output-all` will fail as it will not be possible to resolve `terraform_remote_state` data sources! Please [see here for more information](https://github.com/gruntwork-io/terragrunt/issues/720#issuecomment-497888756). -### destroy-all +### destroy-all (DEPRECATED: use run-all) + +**DEPRECATED: Use `run-all destroy` instead.** Destroy a 'stack' by running 'terragrunt destroy' in each subfolder. Make sure to read [Execute Terraform commands on multiple modules at once](/docs/features/execute-terraform-commands-on-multiple-modules-at-once/) for @@ -133,7 +175,9 @@ This will recursively search the current working directory for any folders that [`dependency`](/docs/reference/config-blocks-and-attributes/#dependency) and [`dependencies`](/docs/reference/config-blocks-and-attributes/#dependencies) blocks. -### validate-all +### validate-all (DEPRECATED: use run-all) + +**DEPRECATED: Use `run-all validate` instead.** Validate 'stack' by running 'terragrunt validate' in each subfolder. Make sure to read [Execute Terraform commands on multiple modules at once](/docs/features/execute-terraform-commands-on-multiple-modules-at-once/) for diff --git a/docs/_docs/04_reference/config-blocks-and-attributes.md b/docs/_docs/04_reference/config-blocks-and-attributes.md index cb1325688e..63ca212d29 100644 --- a/docs/_docs/04_reference/config-blocks-and-attributes.md +++ b/docs/_docs/04_reference/config-blocks-and-attributes.md @@ -468,7 +468,7 @@ state for the target module without parsing the `dependency` blocks, avoiding th ### dependencies The `dependencies` block is used to enumerate all the Terragrunt modules that need to be applied in order for this -module to be able to apply. Note that this is purely for ordering the operations when using `xxx-all` commands of +module to be able to apply. Note that this is purely for ordering the operations when using `run-all` commands of Terraform. This does not expose or pull in the outputs like `dependency` blocks. The `dependencies` block supports the following arguments: @@ -478,7 +478,7 @@ The `dependencies` block supports the following arguments: Example: ```hcl -# When applying this terragrunt config in an `xxx-all` command, make sure the modules at "../vpc" and "../rds" are +# When applying this terragrunt config in an `run-all` command, make sure the modules at "../vpc" and "../rds" are # handled first. dependencies { paths = ["../vpc", "../rds"] @@ -639,8 +639,8 @@ Consider the following file structure: └── terragrunt.hcl In some cases, the root level `terragrunt.hcl` file is solely used to DRY up your Terraform configuration by being -included in the other `terragrunt.hcl` files. In this case, you do not want the `xxx-all` commands to process the root -level `terragrunt.hcl` since it does not define any infrastructure by itself. To make the `xxx-all` commands skip the +included in the other `terragrunt.hcl` files. In this case, you do not want the `run-all` commands to process the root +level `terragrunt.hcl` since it does not define any infrastructure by itself. To make the `run-all` commands skip the root level `terragrunt.hcl` file, you can set `skip = true`: ``` hcl diff --git a/options/options.go b/options/options.go index 082ccb3315..d9031ce987 100644 --- a/options/options.go +++ b/options/options.go @@ -256,6 +256,7 @@ func (terragruntOptions *TerragruntOptions) Clone(terragruntConfigPath string) * TerraformCliArgs: util.CloneStringList(terragruntOptions.TerraformCliArgs), WorkingDir: workingDir, Logger: util.CreateLogEntryWithWriter(terragruntOptions.ErrWriter, workingDir, terragruntOptions.LogLevel), + LogLevel: terragruntOptions.LogLevel, Env: util.CloneStringMap(terragruntOptions.Env), Source: terragruntOptions.Source, SourceUpdate: terragruntOptions.SourceUpdate, diff --git a/shell/run_shell_cmd.go b/shell/run_shell_cmd.go index d07e961392..42ed1d625c 100644 --- a/shell/run_shell_cmd.go +++ b/shell/run_shell_cmd.go @@ -84,6 +84,7 @@ func RunShellCommandWithOutput( } else { cmd.Dir = workingDir } + // Inspired by https://blog.kowalczyk.info/article/wOYk/advanced-command-execution-in-go-with-osexec.html cmdStderr := io.MultiWriter(errWriter, &stderrBuf) var cmdStdout io.Writer diff --git a/test/integration_test.go b/test/integration_test.go index b12a11456c..29c05bd472 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -281,15 +281,16 @@ func TestTerragruntInitHookWithSourceNoBackend(t *testing.T) { ) err := runTerragruntCommand(t, fmt.Sprintf("terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-working-dir %s --terragrunt-log-level debug", rootPath), &stdout, &stderr) + logBufferContentsLineByLine(t, stdout, "apply stdout") + logBufferContentsLineByLine(t, stderr, "apply stderr") output := stderr.String() if err != nil { t.Errorf("Did not expect to get error: %s", err.Error()) } - assert.Equal(t, 1, strings.Count(output, "AFTER_INIT_ONLY_ONCE"), "Hooks on init command executed more than once") - // `init-from-module` hook should execute only once (2 occurrences due to the echo and its output) - assert.Equal(t, 2, strings.Count(output, "AFTER_INIT_FROM_MODULE_ONLY_ONCE"), "Hooks on init-from-module command executed more than once") + assert.Equal(t, 1, strings.Count(output, "AFTER_INIT_ONLY_ONCE\n"), "Hooks on init command executed more than once") + assert.Equal(t, 1, strings.Count(output, "AFTER_INIT_FROM_MODULE_ONLY_ONCE\n"), "Hooks on init-from-module command executed more than once") } func TestTerragruntInitHookWithSourceWithBackend(t *testing.T) { @@ -784,6 +785,22 @@ digraph { `))) } +func TestTerragruntRunAllCommand(t *testing.T) { + t.Parallel() + + s3BucketName := fmt.Sprintf("terragrunt-test-bucket-%s", strings.ToLower(uniqueId())) + defer deleteS3Bucket(t, TERRAFORM_REMOTE_STATE_S3_REGION, s3BucketName) + + tmpEnvPath := copyEnvironment(t, TEST_FIXTURE_OUTPUT_ALL) + + rootTerragruntConfigPath := util.JoinPath(tmpEnvPath, TEST_FIXTURE_OUTPUT_ALL, config.DefaultTerragruntConfigPath) + copyTerragruntConfigAndFillPlaceholders(t, rootTerragruntConfigPath, rootTerragruntConfigPath, s3BucketName, "not-used", "not-used") + + environmentPath := fmt.Sprintf("%s/%s/env1", tmpEnvPath, TEST_FIXTURE_OUTPUT_ALL) + + runTerragrunt(t, fmt.Sprintf("terragrunt run-all init --terragrunt-non-interactive --terragrunt-working-dir %s", environmentPath)) +} + func TestTerragruntOutputAllCommand(t *testing.T) { t.Parallel() @@ -868,6 +885,9 @@ func TestTerragruntOutputAllCommandSpecificVariableIgnoreDependencyErrors(t *tes runTerragruntCommand(t, fmt.Sprintf("terragrunt output-all app2_text --terragrunt-ignore-dependency-errors --terragrunt-non-interactive --terragrunt-working-dir %s", environmentPath), &stdout, &stderr) output := stdout.String() + logBufferContentsLineByLine(t, stdout, "output-all stdout") + logBufferContentsLineByLine(t, stderr, "output-all stderr") + // Without --terragrunt-ignore-dependency-errors, app2 never runs because its dependencies have "errors" since they don't have the output "app2_text". assert.True(t, strings.Contains(output, "app2 output")) }