Improved errors retry/ignore (#3795)

* Add handling of multiline errors

* Simplified errors message cleanup

* Strict lint updates

* Log message update

options/options.go

@@ -10,6 +10,7 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"strings"
 	"time"
 
 	"github.com/gruntwork-io/terragrunt/internal/errors"
@@ -65,6 +66,9 @@ var (
 		"force-unlock",
 		"state",
 	}
+
+	// Pattern used to clean error message when looking for retry and ignore patterns.
+	errorCleanPattern = regexp.MustCompile(`[^a-zA-Z0-9./'"(): ]+`)
 )
 
 type ctxKey byte
@@ -962,7 +966,7 @@ func (opts *TerragruntOptions) RunWithErrorHandling(ctx context.Context, operati
 		}
 
 		// Process the error through our error handling configuration
-		action, processErr := opts.Errors.ProcessError(err, currentAttempt)
+		action, processErr := opts.Errors.ProcessError(opts, err, currentAttempt)
 		if processErr != nil {
 			return fmt.Errorf("error processing error handling rules: %w", processErr)
 		}
@@ -1041,14 +1045,16 @@ type ErrorAction struct {
 }
 
 // ProcessError evaluates an error against the configuration and returns the appropriate action
-func (c *ErrorsConfig) ProcessError(err error, currentAttempt int) (*ErrorAction, error) {
+func (c *ErrorsConfig) ProcessError(opts *TerragruntOptions, err error, currentAttempt int) (*ErrorAction, error) {
 	if err == nil {
 		return nil, nil
 	}
 
-	errStr := err.Error()
+	errStr := extractErrorMessage(err)
 	action := &ErrorAction{}
 
+	opts.Logger.Debugf("Processing error message: %s", errStr)
+
 	// First check ignore rules
 	for _, ignoreBlock := range c.Ignore {
 		isIgnorable := matchesAnyRegexpPattern(errStr, ignoreBlock.IgnorableErrors)
@@ -1087,6 +1093,14 @@ func (c *ErrorsConfig) ProcessError(err error, currentAttempt int) (*ErrorAction
 	return nil, err
 }
 
+func extractErrorMessage(err error) string {
+	// fetch the error string and remove any ASCII escape sequences
+	multilineText := log.RemoveAllASCISeq(err.Error())
+	errorText := errorCleanPattern.ReplaceAllString(multilineText, " ")
+
+	return strings.Join(strings.Fields(errorText), " ")
+}
+
 // matchesAnyRegexpPattern checks if the input string matches any of the provided compiled patterns
 func matchesAnyRegexpPattern(input string, patterns []*ErrorsPattern) bool {
 	for _, pattern := range patterns {

test/fixtures/errors/multi-line/main.tf

@@ -0,0 +1,17 @@
+
+resource "null_resource" "script_runner" {
+  provisioner "local-exec" {
+    command = "./script.sh"
+
+    interpreter = ["/bin/sh", "-c"]
+    on_failure  = fail
+  }
+
+  triggers = {
+    always_run = timestamp()
+  }
+}
+
+output "value" {
+  value = "valid value from failing_dep"
+}

test/fixtures/errors/multi-line/script.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+echo "Error: creating Route in Route Table (rtb-46521694) with destination (10.0.0.0/8): operation error EC2: CreateRoute, https response error StatusCode: 400, RequestID: JD40-14127-2022, api error InvalidTransitGatewayID.NotFound: The transitGateway ID 'tgw-xxxxxxxxxxxxxx' does not exist."
+exit 1

test/fixtures/errors/multi-line/terragrunt.hcl

@@ -0,0 +1,17 @@
+errors {
+  # Retry block for transient errors
+  retry "transient_errors" {
+    retryable_errors = [
+      "(?s).*cannot create resource \"storageclasses\" in API group.*",
+    ]
+    max_attempts       = 3
+    sleep_interval_sec = 20
+  }
+
+  ignore "transit_gateway_errors" {
+    ignorable_errors = [
+      ".*creating Route in Route Table*"
+    ]
+    message = "Ignoring transit gateway not found when creating internal route."
+  }
+}

test/integration_errors_test.go

@@ -22,6 +22,7 @@ const (
 	testRetryFailErrors       = "fixtures/errors/retry-fail"
 	testRunAllErrors          = "fixtures/errors/run-all"
 	testNegativePatternErrors = "fixtures/errors/ignore-negative-pattern"
+	testMultiLineErrors       = "fixtures/errors/multi-line"
 )
 
 func TestErrorsHandling(t *testing.T) {
@@ -159,3 +160,16 @@ func TestIgnoreNegativePattern(t *testing.T) {
 	require.Error(t, err)
 	assert.Contains(t, stdout, "Error: baz")
 }
+
+func TestHandleMultiLineErrors(t *testing.T) {
+	t.Parallel()
+
+	cleanupTerraformFolder(t, testMultiLineErrors)
+	tmpEnvPath := helpers.CopyEnvironment(t, testMultiLineErrors)
+	rootPath := util.JoinPath(tmpEnvPath, testMultiLineErrors)
+
+	_, stdout, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-working-dir "+rootPath)
+
+	require.NoError(t, err)
+	assert.Contains(t, stdout, "Ignoring transit gateway not found when creating internal route")
+}