Adding distinct error codes for custom test failures

docs/content/en/docs/references/api/grpc.md

@@ -967,6 +967,15 @@ For Cancelled Error code, use range 800 to 850.<br>
 | TEST_USER_CONFIG_ERR | 1101 | Error expanding paths |
 | TEST_CST_USER_ERR | 1102 | Error running container-structure-test |
 | TEST_IMG_PULL_ERR | 1103 | Unable to docker pull image |
+| TEST_CUSTOM_CMD_PARSE_ERR | 1104 | Unable to parse test command |
+| TEST_CUSTOM_CMD_RUN_NON_ZERO_EXIT_ERR | 1105 | Command returned non-zero exit code |
+| TEST_CUSTOM_CMD_RUN_TIMEDOUT_ERR | 1106 | command cancelled or timed out |
+| TEST_CUSTOM_CMD_RUN_CANCELLED_ERR | 1107 | command cancelled or timed out |
+| TEST_CUSTOM_CMD_RUN_EXECUTION_ERR | 1108 | command context error |
+| TEST_CUSTOM_CMD_RUN_EXITED_ERR | 1110 | command exited |
+| TEST_CUSTOM_CMD_RUN_ERR | 1111 | error running cmd |
+| TEST_CUSTOM_DEPENDENCIES_CMD_ERR | 1112 | Error getting dependencies from command |
+| TEST_CUSTOM_DEPENDENCIES_UNMARSHALL_ERR | 1113 | Unmarshalling dependency output error |
 | CONFIG_FILE_PARSING_ERR | 1201 | Catch-all configuration file parsing error |
 | CONFIG_FILE_NOT_FOUND_ERR | 1202 | Main configuration file not found |
 | CONFIG_DEPENDENCY_NOT_FOUND_ERR | 1203 | Dependency configuration file not found |
@@ -1034,6 +1043,10 @@ Enum for Suggestion codes
 | CONFIG_CHECK_PROFILE_DEFINITION | 704 | Check profile definition in current config |
 | CONFIG_CHECK_DEPENDENCY_PROFILES_SELECTION | 705 | Check active profile selection for dependency config |
 | OPEN_ISSUE | 900 | Open an issue so this situation can be diagnosed |
+| CHECK_CUSTOM_COMMAND | 1000 | Test error suggestion codes |
+| FIX_CUSTOM_COMMAND_TIMEOUT | 1001 |  |
+| CHECK_CUSTOM_COMMAND_DEPENDENCIES_CMD | 1002 |  |
+| CHECK_CUSTOM_COMMAND_DEPENDENCIES_PATHS | 1003 |  |
 
 
 

pkg/skaffold/test/custom/custom.go

@@ -19,7 +19,6 @@ package custom
 import (
 	"context"
 	"encoding/json"
-	"fmt"
 	"io"
 	"os/exec"
 	"runtime"
@@ -54,7 +53,7 @@ func New(cfg docker.Config, wd string, ct latest.CustomTest) (*Runner, error) {
 // Test is the entrypoint for running custom tests
 func (ct *Runner) Test(ctx context.Context, out io.Writer, _ []build.Artifact) error {
 	if err := doRunCustomCommand(ctx, out, ct.customTest); err != nil {
-		return fmt.Errorf("running custom test command: %w", err)
+		return err
 	}
 
 	return nil
@@ -64,7 +63,7 @@ func runCustomCommand(ctx context.Context, out io.Writer, test latest.CustomTest
 	// Expand command
 	command, err := util.ExpandEnvTemplate(test.Command, nil)
 	if err != nil {
-		return fmt.Errorf("unable to parse test command %q: %w", test.Command, err)
+		return cmdRunParsingErr(test.Command, err)
 	}
 
 	if test.TimeoutSeconds <= 0 {
@@ -93,22 +92,24 @@ func runCustomCommand(ctx context.Context, out io.Writer, test latest.CustomTest
 			// If the process exited by itself, just return the error
 			if e.Exited() {
 				color.Red.Fprintf(out, "Command finished with non-0 exit code.\n")
-				return fmt.Errorf("command finished with non-0 exit code: %w", e)
+				return cmdRunNonZeroExitErr(command, e)
 			}
 			// If the context is done, it has been killed by the exec.Command
 			select {
 			case <-ctx.Done():
 				if ctx.Err() == context.DeadlineExceeded {
 					color.Red.Fprintf(out, "Command timed out\n")
+					return cmdRunTimedoutErr(test.TimeoutSeconds, ctx.Err())
 				} else if ctx.Err() == context.Canceled {
 					color.Red.Fprintf(out, "Command cancelled\n")
+					return cmdRunCancelledErr(ctx.Err())
 				}
-				return ctx.Err()
+				return cmdRunExecutionErr(ctx.Err())
 			default:
-				return e
+				return cmdRunExited(e)
 			}
 		}
-		return err
+		return cmdRunErr(err)
 	}
 	color.Green.Fprintf(out, "Command finished successfully\n")
 
@@ -118,7 +119,6 @@ func runCustomCommand(ctx context.Context, out io.Writer, test latest.CustomTest
 // TestDependencies returns dependencies listed for a custom test
 func (ct *Runner) TestDependencies() ([]string, error) {
 	test := ct.customTest
-	// var set orderedFileSet
 
 	if test.Dependencies != nil {
 		switch {
@@ -133,11 +133,11 @@ func (ct *Runner) TestDependencies() ([]string, error) {
 
 			output, err := util.RunCmdOut(cmd)
 			if err != nil {
-				return nil, fmt.Errorf("getting dependencies from command: %q: %w", test.Dependencies.Command, err)
+				return nil, gettingDependenciesCommandErr(test.Dependencies.Command, err)
 			}
 			var deps []string
 			if err := json.Unmarshal(output, &deps); err != nil {
-				return nil, fmt.Errorf("unmarshalling dependency output into string array: %w", err)
+				return nil, dependencyOutputUnmarshallErr(test.Dependencies.Paths, err)
 			}
 			return deps, nil
 

pkg/skaffold/test/custom/custom_test.go

@@ -116,6 +116,7 @@ func TestCustomCommandError(t *testing.T) {
 			t.CheckNoError(err)
 			err = testRunner.Test(context.Background(), ioutil.Discard, nil)
 
+			// TODO(modali): Update the logic to check for error code instead of error string.
 			t.CheckError(test.shouldErr, err)
 			if test.expectedError != "" {
 				t.CheckErrorContains(test.expectedError, err)

pkg/skaffold/test/custom/error.go

@@ -0,0 +1,135 @@
+/*
+Copyright 2021 The Skaffold Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package custom
+
+import (
+	"fmt"
+
+	sErrors "github.com/GoogleContainerTools/skaffold/pkg/skaffold/errors"
+	"github.com/GoogleContainerTools/skaffold/proto/v1"
+)
+
+func cmdRunParsingErr(command string, err error) error {
+	return sErrors.NewError(err,
+		proto.ActionableErr{
+			Message: fmt.Sprintf("unable to parse test command %s: %s", command, err),
+			ErrCode: proto.StatusCode_TEST_CUSTOM_CMD_PARSE_ERR,
+			Suggestions: []*proto.Suggestion{
+				{
+					SuggestionCode: proto.SuggestionCode_CHECK_CUSTOM_COMMAND,
+					Action:         fmt.Sprintf("Check the custom command contents: %q", command),
+				},
+			},
+		},
+	)
+}
+
+func cmdRunNonZeroExitErr(command string, err error) error {
+	return sErrors.NewError(err,
+		proto.ActionableErr{
+			Message: fmt.Sprintf("command %s finished with non-0 exit code: %s", command, err),
+			ErrCode: proto.StatusCode_TEST_CUSTOM_CMD_RUN_NON_ZERO_EXIT_ERR,
+			Suggestions: []*proto.Suggestion{
+				{
+					SuggestionCode: proto.SuggestionCode_CHECK_CUSTOM_COMMAND,
+					Action:         fmt.Sprintf("Check the custom command contents: %q", command),
+				},
+			},
+		},
+	)
+}
+
+func cmdRunTimedoutErr(timeoutSeconds int, err error) error {
+	return sErrors.NewError(err,
+		proto.ActionableErr{
+			Message: fmt.Sprintf("command timed out: %s", err),
+			ErrCode: proto.StatusCode_TEST_CUSTOM_CMD_RUN_TIMEDOUT_ERR,
+			Suggestions: []*proto.Suggestion{
+				{
+					SuggestionCode: proto.SuggestionCode_FIX_CUSTOM_COMMAND_TIMEOUT,
+					Action:         fmt.Sprintf("Fix the custom command timeoutSeconds: %d", timeoutSeconds),
+				},
+			},
+		},
+	)
+}
+
+func cmdRunCancelledErr(err error) error {
+	return sErrors.NewError(err,
+		proto.ActionableErr{
+			Message: fmt.Sprintf("command cancelled: %s", err),
+			ErrCode: proto.StatusCode_TEST_CUSTOM_CMD_RUN_CANCELLED_ERR,
+		},
+	)
+}
+
+func cmdRunExecutionErr(err error) error {
+	return sErrors.NewError(err,
+		proto.ActionableErr{
+			Message: fmt.Sprintf("command execution error: %s", err),
+			ErrCode: proto.StatusCode_TEST_CUSTOM_CMD_RUN_EXECUTION_ERR,
+		},
+	)
+}
+
+func cmdRunExited(err error) error {
+	return sErrors.NewError(err,
+		proto.ActionableErr{
+			Message: fmt.Sprintf("command exited: %s", err),
+			ErrCode: proto.StatusCode_TEST_CUSTOM_CMD_RUN_EXITED_ERR,
+		},
+	)
+}
+
+func cmdRunErr(err error) error {
+	return sErrors.NewError(err,
+		proto.ActionableErr{
+			Message: fmt.Sprintf("error running cmd: %s", err),
+			ErrCode: proto.StatusCode_TEST_CUSTOM_CMD_RUN_ERR,
+		},
+	)
+}
+
+func gettingDependenciesCommandErr(command string, err error) error {
+	return sErrors.NewError(err,
+		proto.ActionableErr{
+			Message: fmt.Sprintf("getting dependencies from command: %s: %s", command, err),
+			ErrCode: proto.StatusCode_TEST_CUSTOM_DEPENDENCIES_CMD_ERR,
+			Suggestions: []*proto.Suggestion{
+				{
+					SuggestionCode: proto.SuggestionCode_CHECK_CUSTOM_COMMAND_DEPENDENCIES_CMD,
+					Action:         fmt.Sprintf("Check the custom command dependencies command: %q", command),
+				},
+			},
+		},
+	)
+}
+
+func dependencyOutputUnmarshallErr(paths []string, err error) error {
+	return sErrors.NewError(err,
+		proto.ActionableErr{
+			Message: fmt.Sprintf("unmarshalling dependency output into string array: %s", err),
+			ErrCode: proto.StatusCode_TEST_CUSTOM_DEPENDENCIES_UNMARSHALL_ERR,
+			Suggestions: []*proto.Suggestion{
+				{
+					SuggestionCode: proto.SuggestionCode_CHECK_CUSTOM_COMMAND_DEPENDENCIES_PATHS,
+					Action:         fmt.Sprintf("Check the custom command dependencies paths: %q", paths),
+				},
+			},
+		},
+	)
+}