fix: Wrap TF stdout and stderr in JSON (#3602)

* fix: wrap tf stdout and stderr in json

* fix: provider cache

* fix: cache provider

* chore: update docs

* chore: fix strict lint

* fix: markdownlint

* chore: fix punctuations

* chore: fix test

cli/commands/flags.go

@@ -68,9 +68,6 @@ const (
 	TerragruntDebugFlagName = "terragrunt-debug"
 	TerragruntDebugEnvName  = "TERRAGRUNT_DEBUG"
 
-	TerragruntTfLogJSONFlagName = "terragrunt-tf-logs-to-json"
-	TerragruntTfLogJSONEnvName  = "TERRAGRUNT_TF_JSON_LOG"
-
 	TerragruntModulesThatIncludeFlagName = "terragrunt-modules-that-include"
 	TerragruntModulesThatIncludeEnvName  = "TERRAGRUNT_MODULES_THAT_INCLUDE"
 
@@ -401,12 +398,6 @@ func NewGlobalFlags(opts *options.TerragruntOptions) cli.Flags {
 				return nil
 			},
 		},
-		&cli.BoolFlag{
-			Name:        TerragruntTfLogJSONFlagName,
-			EnvVar:      TerragruntTfLogJSONEnvName,
-			Destination: &opts.TerraformLogsToJSON,
-			Usage:       "If specified, Terragrunt will wrap Terraform stdout and stderr in JSON.",
-		},
 		&cli.BoolFlag{
 			Name:        TerragruntUsePartialParseConfigCacheFlagName,
 			EnvVar:      TerragruntUsePartialParseConfigCacheEnvName,
@@ -439,8 +430,11 @@ func NewGlobalFlags(opts *options.TerragruntOptions) cli.Flags {
 					return nil
 				}
 
-				if val == format.BareFormatName {
+				switch val {
+				case format.BareFormatName:
 					opts.ForwardTFStdout = true
+				case format.JSONFormatName:
+					opts.JSONLogFormat = true
 				}
 
 				opts.LogFormatter.SetFormat(phs)

cli/deprecated_flags.go

@@ -19,6 +19,9 @@ const (
 
 	TerragruntJSONLogFlagName = "terragrunt-json-log"
 	TerragruntJSONLogEnvName  = "TERRAGRUNT_JSON_LOG"
+
+	TerragruntTfLogJSONFlagName = "terragrunt-tf-logs-to-json"
+	TerragruntTfLogJSONEnvName  = "TERRAGRUNT_TF_JSON_LOG"
 )
 
 // NewDeprecatedFlags creates and returns deprecated flags.
@@ -39,6 +42,7 @@ func NewDeprecatedFlags(opts *options.TerragruntOptions) cli.Flags {
 			EnvVar:      TerragruntDisableLogFormattingEnvName,
 			Destination: &opts.DisableLogFormatting,
 			Usage:       "If specified, logs will be displayed in key/value format. By default, logs are formatted in a human readable format.",
+			Hidden:      true,
 			Action: func(_ *cli.Context, _ bool) error {
 				opts.LogFormatter.SetFormat(format.NewKeyValueFormat())
 
@@ -59,6 +63,7 @@ func NewDeprecatedFlags(opts *options.TerragruntOptions) cli.Flags {
 			EnvVar:      TerragruntJSONLogEnvName,
 			Destination: &opts.JSONLogFormat,
 			Usage:       "If specified, Terragrunt will output its logs in JSON format.",
+			Hidden:      true,
 			Action: func(_ *cli.Context, _ bool) error {
 				opts.LogFormatter.SetFormat(format.NewJSONFormat())
 
@@ -71,6 +76,24 @@ func NewDeprecatedFlags(opts *options.TerragruntOptions) cli.Flags {
 					opts.Logger.Warnf(warn)
 				}
 
+				return nil
+			},
+		},
+		&cli.BoolFlag{
+			Name:   TerragruntTfLogJSONFlagName,
+			EnvVar: TerragruntTfLogJSONEnvName,
+			Usage:  "If specified, Terragrunt will wrap Terraform stdout and stderr in JSON.",
+			Hidden: true,
+			Action: func(_ *cli.Context, _ bool) error {
+				if control, ok := strict.GetStrictControl(strict.JSONLog); ok {
+					warn, err := control.Evaluate(opts)
+					if err != nil {
+						return err
+					}
+
+					opts.Logger.Warnf(warn)
+				}
+
 				return nil
 			},
 		},

cli/provider_cache.go

@@ -333,10 +333,11 @@ func (cache *ProviderCache) createLocalCLIConfig(ctx context.Context, opts *opti
 
 func runTerraformCommand(ctx context.Context, opts *options.TerragruntOptions, args []string, envs map[string]string) (*util.CmdOutput, error) {
 	// We use custom writer in order to trap the log from `terraform providers lock -platform=provider-cache` command, which terraform considers an error, but to us a success.
-	errWriter := util.NewTrapWriter(opts.ErrWriter, httpStatusCacheProviderReg)
+	errWriter := util.NewTrapWriter(opts.ErrWriter)
 
 	// add -no-color flag to args if it was set in Terragrunt arguments
-	if util.ListContainsElement(opts.TerraformCliArgs, terraform.FlagNameNoColor) {
+	if util.ListContainsElement(opts.TerraformCliArgs, terraform.FlagNameNoColor) &&
+		!util.ListContainsElement(args, terraform.FlagNameNoColor) {
 		args = append(args, terraform.FlagNameNoColor)
 	}
 
@@ -350,14 +351,18 @@ func runTerraformCommand(ctx context.Context, opts *options.TerragruntOptions, a
 	cloneOpts.WorkingDir = opts.WorkingDir
 	cloneOpts.TerraformCliArgs = args
 	cloneOpts.Env = envs
-	cloneOpts.ForwardTFStdout = true
 
+	output, err := shell.RunTerraformCommandWithOutput(ctx, cloneOpts, cloneOpts.TerraformCliArgs...)
 	// If the Terraform error matches `httpStatusCacheProviderReg` we ignore it and hide the log from users, otherwise we process the error as is.
-	if output, err := shell.RunTerraformCommandWithOutput(ctx, cloneOpts, cloneOpts.TerraformCliArgs...); err != nil && len(errWriter.Msgs()) == 0 {
-		return output, err
+	if err != nil && httpStatusCacheProviderReg.Match(output.Stderr.Bytes()) {
+		return new(util.CmdOutput), nil
+	}
+
+	if err := errWriter.Flush(); err != nil {
+		return nil, err
 	}
 
-	return nil, nil
+	return output, err
 }
 
 // providerCacheEnvironment returns TF_* name/value ENVs, which we use to force terraform processes to make requests through our cache server (proxy) instead of making direct requests to the origin servers.

config/dependency.go

@@ -1051,7 +1051,7 @@ func runTerragruntOutputJSON(ctx *ParsingContext, targetConfig string) ([]byte,
 	newOpts := *ctx.TerragruntOptions
 	// explicit disable json formatting and prefixing to read json output
 	newOpts.ForwardTFStdout = false
-	newOpts.TerraformLogsToJSON = false
+	newOpts.JSONLogFormat = false
 	newOpts.Writer = stdoutBufferWriter
 	ctx = ctx.WithTerragruntOptions(&newOpts)
 

configstack/module.go

@@ -256,7 +256,7 @@ func FindWhereWorkingDirIsIncluded(ctx context.Context, opts *options.Terragrunt
 		cfgOptions.OriginalTerragruntConfigPath = opts.OriginalTerragruntConfigPath
 		cfgOptions.TerraformCommand = opts.TerraformCommand
 		cfgOptions.NonInteractive = true
-		cfgOptions.Logger.SetOptions(log.WithHooks(NewForceLogLevelHook(log.DebugLevel)))
+		cfgOptions.Logger = opts.Logger.WithOptions(log.WithHooks(NewForceLogLevelHook(log.DebugLevel)))
 
 		// build stack from config directory
 		stack, err := FindStackInSubfolders(ctx, cfgOptions, WithChildTerragruntConfig(terragruntConfig))

configstack/running_module.go

@@ -140,7 +140,7 @@ func (module *RunningModule) runNow(ctx context.Context, rootOptions *options.Te
 
 			stdout := bytes.Buffer{}
 			jsonOptions.ForwardTFStdout = true
-			jsonOptions.TerraformLogsToJSON = false
+			jsonOptions.JSONLogFormat = false
 			jsonOptions.Writer = &stdout
 			jsonOptions.TerraformCommand = terraform.CommandNameShow
 			jsonOptions.TerraformCliArgs = []string{terraform.CommandNameShow, "-json", module.Module.planFile(rootOptions)}

docs/_docs/02_features/custom-log-format.md

@@ -56,10 +56,12 @@ Placeholders have preset names:
 
 * `%prefix` - Path to the working directory were Terragrunt is running.
 
-* `%tfpath` - Path to the OpenTofu/Terraform executable (as defined by [terragrunt-tfpath](https://terragrunt.gruntwork.io/docs/reference/cli-options/#terragrunt-tfpath)).
-
 * `%msg` - Log message.
 
+* `%tf-path` - Path to the OpenTofu/Terraform executable (as defined by [terragrunt-tfpath](https://terragrunt.gruntwork.io/docs/reference/cli-options/#terragrunt-tfpath)).
+
+* `%tf-command-args` - Arguments of the executed OpenTofu/Terraform command.
+
 * `%t` - Tab.
 
 * `%n` - Newline.
@@ -256,7 +258,7 @@ Specific options for placeholders:
 
     * `short` - Outputs an absolute path, but hides the working directory path.
 
-* `%tfpath`
+* `%tf-path`
 
   * `path=[filename|dir]`
 
@@ -277,7 +279,7 @@ The examples below replicate the preset formats specified with `--terragrunt-log
 `--terragrunt-log-format pretty`
 
 ```shell
---terragrunt-log-custom-format "%time(color=light-black) %level(case=upper,width=6,color=preset) %prefix(path=short-relative,color=gradient,suffix=' ')%tfpath(color=cyan,suffix=': ')%msg(path=relative)"
+--terragrunt-log-custom-format "%time(color=light-black) %level(case=upper,width=6,color=preset) %prefix(path=short-relative,color=gradient,suffix=' ')%tf-path(color=cyan,suffix=': ')%msg(path=relative)"
 ```
 
 `--terragrunt-log-format bare`
@@ -289,11 +291,11 @@ The examples below replicate the preset formats specified with `--terragrunt-log
 `--terragrunt-log-format key-value`
 
 ```shell
---terragrunt-log-custom-format "time=%time(format=rfc3339) level=%level prefix=%prefix(path=short-relative) tfpath=%tfpath(path=filename) msg=%msg(path=relative,color=disable)"
+--terragrunt-log-custom-format "time=%time(format=rfc3339) level=%level prefix=%prefix(path=short-relative) tf-path=%tf-path(path=filename) msg=%msg(path=relative,color=disable)"
 ```
 
 `--terragrunt-log-format json`
 
 ```shell
---terragrunt-log-custom-format '{"time":"%time(format=rfc3339,escape=json)", "level":"%level(escape=json)", "prefix":"%prefix(path=short-relative,escape=json)", "tfpath":"%tfpath(path=filename,escape=json)", "msg":"%msg(path=relative,escape=json,color=disable)"}'
+--terragrunt-log-custom-format '{"time":"%time(format=rfc3339,escape=json)", "level":"%level(escape=json)", "prefix":"%prefix(path=short-relative,escape=json)", "tf-path":"%tf-path(path=filename,escape=json)", "msg":"%msg(path=relative,escape=json,color=disable)"}'
 ```

docs/_docs/04_reference/cli-options.md

@@ -82,7 +82,7 @@ This page documents the CLI commands and options available with Terragrunt:
   - [terragrunt-disable-bucket-update](#terragrunt-disable-bucket-update)
   - [terragrunt-disable-command-validation](#terragrunt-disable-command-validation)
   - [terragrunt-json-log](#terragrunt-json-log) (DEPRECATED: use [terragrunt-log-format](#terragrunt-log-format))
-  - [terragrunt-tf-logs-to-json](#terragrunt-tf-logs-to-json)
+  - [terragrunt-tf-logs-to-json](#terragrunt-tf-logs-to-json) (DEPRECATED: use [terragrunt-log-format](#terragrunt-log-format))
   - [terragrunt-provider-cache](#terragrunt-provider-cache)
   - [terragrunt-provider-cache-dir](#terragrunt-provider-cache-dir)
   - [terragrunt-provider-cache-hostname](#terragrunt-provider-cache-hostname)
@@ -809,7 +809,7 @@ prefix `--terragrunt-` (e.g., `--terragrunt-config`). The currently available op
   - [terragrunt-disable-bucket-update](#terragrunt-disable-bucket-update)
   - [terragrunt-disable-command-validation](#terragrunt-disable-command-validation)
   - [terragrunt-json-log](#terragrunt-json-log) (DEPRECATED: use [terragrunt-log-format](#terragrunt-log-format))
-  - [terragrunt-tf-logs-to-json](#terragrunt-tf-logs-to-json)
+  - [terragrunt-tf-logs-to-json](#terragrunt-tf-logs-to-json) (DEPRECATED: use [terragrunt-log-format](#terragrunt-log-format))
   - [terragrunt-provider-cache](#terragrunt-provider-cache)
   - [terragrunt-provider-cache-dir](#terragrunt-provider-cache-dir)
   - [terragrunt-provider-cache-hostname](#terragrunt-provider-cache-hostname)
@@ -1485,6 +1485,9 @@ When this flag is set, Terragrunt will output its logs in JSON format.
 
 ### terragrunt-tf-logs-to-json
 
+DEPRECATED: Use [terragrunt-log-format](#terragrunt-log-format). OpenTofu/Terraform `stdout` and `stderr` is wrapped in JSON by default with `--terragurnt-log-format json` flag if `--terragrunt-forward-tf-stdout` flag is not specified.
+In other words, the previous behavior with the `--terragrunt-json-log --terragrunt-tf-logs-to-json` flags is now equivalent to `--terragrunt-log-format json` and the previous behavior with the `--terragrunt-json-log` is now equivalent to `--terragrunt-log-format json --terragrunt-forward-tf-stdout`.
+
 **CLI Arg**: `--terragrunt-tf-logs-to-json`<br/>
 **Environment Variable**: `TERRAGRUNT_TF_JSON_LOG` (set to `true`)<br/>
 

internal/strict/strict.go

@@ -47,6 +47,8 @@ const (
 	DisableLogFormatting = "terragrunt-disable-log-formatting"
 	// JSONLog is the control that prevents the deprecated `--terragrunt-json-log` flag from being used.
 	JSONLog = "terragrunt-json-log"
+	// TfLogJSON is the control that prevents the deprecated `--terragrunt-tf-logs-to-json` flag from being used.
+	TfLogJSON = "terragrunt-tf-logs-to-json"
 )
 
 // GetStrictControl returns the strict control with the given name.
@@ -83,45 +85,49 @@ type Controls map[string]Control
 //nolint:lll,gochecknoglobals,stylecheck
 var StrictControls = Controls{
 	SpinUp: {
-		Error:   errors.Errorf("The `%s` command is no longer supported. Use `terragrunt run-all apply` instead.", SpinUp),
+		Error:   errors.Errorf("The `%s` command is no longer supported. Use `terragrunt run-all apply` instead.", SpinUp), //nolint:revive
 		Warning: fmt.Sprintf("The `%s` command is deprecated and will be removed in a future version. Use `terragrunt run-all apply` instead.", SpinUp),
 	},
 	TearDown: {
-		Error:   errors.Errorf("The `%s` command is no longer supported. Use `terragrunt run-all destroy` instead.", TearDown),
+		Error:   errors.Errorf("The `%s` command is no longer supported. Use `terragrunt run-all destroy` instead.", TearDown), //nolint:revive
 		Warning: fmt.Sprintf("The `%s` command is deprecated and will be removed in a future version. Use `terragrunt run-all destroy` instead.", TearDown),
 	},
 	PlanAll: {
-		Error:   errors.Errorf("The `%s` command is no longer supported. Use `terragrunt run-all plan` instead.", PlanAll),
+		Error:   errors.Errorf("The `%s` command is no longer supported. Use `terragrunt run-all plan` instead.", PlanAll), //nolint:revive
 		Warning: fmt.Sprintf("The `%s` command is deprecated and will be removed in a future version. Use `terragrunt run-all plan` instead.", PlanAll),
 	},
 	ApplyAll: {
-		Error:   errors.Errorf("The `%s` command is no longer supported. Use `terragrunt run-all apply` instead.", ApplyAll),
+		Error:   errors.Errorf("The `%s` command is no longer supported. Use `terragrunt run-all apply` instead.", ApplyAll), //nolint:revive
 		Warning: fmt.Sprintf("The `%s` command is deprecated and will be removed in a future version. Use `terragrunt run-all apply` instead.", ApplyAll),
 	},
 	DestroyAll: {
-		Error:   errors.Errorf("The `%s` command is no longer supported. Use `terragrunt run-all destroy` instead.", DestroyAll),
+		Error:   errors.Errorf("The `%s` command is no longer supported. Use `terragrunt run-all destroy` instead.", DestroyAll), //nolint:revive
 		Warning: fmt.Sprintf("The `%s` command is deprecated and will be removed in a future version. Use `terragrunt run-all destroy` instead.", DestroyAll),
 	},
 	OutputAll: {
-		Error:   errors.Errorf("The `%s` command is no longer supported. Use `terragrunt run-all output` instead.", OutputAll),
+		Error:   errors.Errorf("The `%s` command is no longer supported. Use `terragrunt run-all output` instead.", OutputAll), //nolint:revive
 		Warning: fmt.Sprintf("The `%s` command is deprecated and will be removed in a future version. Use `terragrunt run-all output` instead.", OutputAll),
 	},
 	ValidateAll: {
-		Error:   errors.Errorf("The `%s` command is no longer supported. Use `terragrunt run-all validate` instead.", ValidateAll),
+		Error:   errors.Errorf("The `%s` command is no longer supported. Use `terragrunt run-all validate` instead.", ValidateAll), //nolint:revive
 		Warning: fmt.Sprintf("The `%s` command is deprecated and will be removed in a future version. Use `terragrunt run-all validate` instead.", ValidateAll),
 	},
 	SkipDependenciesInputs: {
-		Error:   errors.Errorf("The `%s` option is deprecated. Reading inputs from dependencies has been deprecated and will be removed in a future version of Terragrunt. To continue using inputs from dependencies, forward them as outputs.", SkipDependenciesInputs),
+		Error:   errors.Errorf("The `%s` option is deprecated. Reading inputs from dependencies has been deprecated and will be removed in a future version of Terragrunt. To continue using inputs from dependencies, forward them as outputs.", SkipDependenciesInputs), //nolint:revive
 		Warning: fmt.Sprintf("The `%s` option is deprecated and will be removed in a future version of Terragrunt. Reading inputs from dependencies has been deprecated. To continue using inputs from dependencies, forward them as outputs.", SkipDependenciesInputs),
 	},
 	DisableLogFormatting: {
-		Error:   errors.Errorf("The `--%s` flag is no longer supported. Use `--terragrunt-log-format=key-value` instead.", DisableLogFormatting),
+		Error:   errors.Errorf("The `--%s` flag is no longer supported. Use `--terragrunt-log-format=key-value` instead.", DisableLogFormatting), //nolint:revive
 		Warning: fmt.Sprintf("The `--%s` flag is deprecated and will be removed in a future version. Use `--terragrunt-log-format=key-value` instead.", DisableLogFormatting),
 	},
 	JSONLog: {
-		Error:   errors.Errorf("The `--%s` flag is no longer supported. Use `--terragrunt-log-format=json` instead.", JSONLog),
+		Error:   errors.Errorf("The `--%s` flag is no longer supported. Use `--terragrunt-log-format=json` instead.", JSONLog), //nolint:revive
 		Warning: fmt.Sprintf("The `--%s` flag is deprecated and will be removed in a future version. Use `--terragrunt-log-format=json` instead.", JSONLog),
 	},
+	TfLogJSON: {
+		Error:   errors.Errorf("The `--%s` flag is no longer supported. Use `--terragrunt-log-format=json` instead.", TfLogJSON), //nolint:revive
+		Warning: fmt.Sprintf("The `--%s` flag is deprecated and will be removed in a future version. Use `--terragrunt-log-format=json` instead.", TfLogJSON),
+	},
 }
 
 // Names returns the names of all strict controls.

options/options.go

@@ -147,9 +147,6 @@ type TerragruntOptions struct {
 	// If true, logs will be displayed in formatter key/value, by default logs are formatted in human-readable formatter.
 	DisableLogFormatting bool
 
-	// Wrap Terraform logs in JSON format
-	TerraformLogsToJSON bool
-
 	// ValidateStrict mode for the validate-inputs command
 	ValidateStrict bool
 
@@ -469,7 +466,6 @@ func NewTerragruntOptionsWithWriters(stdout, stderr io.Writer) *TerragruntOption
 		ForwardTFStdout:                false,
 		JSONOut:                        DefaultJSONOutName,
 		TerraformImplementation:        UnknownImpl,
-		TerraformLogsToJSON:            false,
 		JSONDisableDependentModules:    false,
 		RunTerragrunt: func(ctx context.Context, opts *TerragruntOptions) error {
 			return errors.New(ErrRunTerragruntCommandNotSet)
@@ -618,7 +614,6 @@ func (opts *TerragruntOptions) Clone(terragruntConfigPath string) (*TerragruntOp
 		FailIfBucketCreationRequired:   opts.FailIfBucketCreationRequired,
 		DisableBucketUpdate:            opts.DisableBucketUpdate,
 		TerraformImplementation:        opts.TerraformImplementation,
-		TerraformLogsToJSON:            opts.TerraformLogsToJSON,
 		GraphRoot:                      opts.GraphRoot,
 		ScaffoldVars:                   opts.ScaffoldVars,
 		ScaffoldVarFiles:               opts.ScaffoldVarFiles,

pkg/log/format/format.go

@@ -70,32 +70,42 @@ func NewPrettyFormat() Placeholders {
 
 func NewJSONFormat() Placeholders {
 	return Placeholders{
-		PlainText(`{"time":"`),
+		PlainText(`{`),
 		Time(
+			Prefix(`"time":"`),
+			Suffix(`"`),
 			TimeFormat(RFC3339),
 			Escape(JSONEscape),
 		),
-		PlainText(`", "level":"`),
 		Level(
+			Prefix(`, "level":"`),
+			Suffix(`"`),
 			Escape(JSONEscape),
 		),
-		PlainText(`", "prefix":"`),
 		Field(WorkDirKeyName,
-			PathFormat(ShortPath),
+			Prefix(`, "working-dir":"`),
+			Suffix(`"`),
 			Escape(JSONEscape),
 		),
-		PlainText(`", "tfpath":"`),
 		Field(TFPathKeyName,
+			Prefix(`, "tf-path":"`),
+			Suffix(`"`),
 			PathFormat(FilenamePath),
 			Escape(JSONEscape),
 		),
-		PlainText(`", "msg":"`),
+		Field(TFCmdArgsKeyName,
+			Prefix(`, "tf-command-args":[`),
+			Suffix(`]`),
+			Escape(JSONEscape),
+		),
 		Message(
+			Prefix(`, "msg":"`),
+			Suffix(`"`),
 			PathFormat(RelativePath),
 			Color(DisableColor),
 			Escape(JSONEscape),
 		),
-		PlainText(`"}`),
+		PlainText(`}`),
 	}
 }
 
@@ -113,7 +123,7 @@ func NewKeyValueFormat() Placeholders {
 			PathFormat(ShortRelativePath),
 		),
 		Field(TFPathKeyName,
-			Prefix(" tfpath="),
+			Prefix(" tf-path="),
 			PathFormat(FilenamePath),
 		),
 		Message(