diff --git a/pkg/api/types.go b/pkg/api/types.go index 6ad45516d7..1b5df8398e 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -149,6 +149,8 @@ const ( Migrating ProvisioningState = "Migrating" // Upgrading means an existing ContainerService resource is being upgraded Upgrading ProvisioningState = "Upgrading" + // Canceled means a deployment has been canceled + Canceled ProvisioningState = "Canceled" ) // OrchestratorProfile contains Orchestrator properties diff --git a/pkg/apierror/apierror.go b/pkg/apierror/apierror.go new file mode 100644 index 0000000000..d32b751180 --- /dev/null +++ b/pkg/apierror/apierror.go @@ -0,0 +1,16 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +package apierror + +// New creates an ErrorResponse +func New(errorCategory ErrorCategory, errorCode ErrorCode, message string) *ErrorResponse { + return &ErrorResponse{ + Body: Error{ + Code: errorCode, + Message: message, + Category: errorCategory, + }, + } +} diff --git a/pkg/apierror/apierror_test.go b/pkg/apierror/apierror_test.go new file mode 100644 index 0000000000..08f46a86b3 --- /dev/null +++ b/pkg/apierror/apierror_test.go @@ -0,0 +1,33 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +package apierror + +import ( + "testing" + + . "github.com/onsi/gomega" +) + +func TestNewAPIError(t *testing.T) { + RegisterTestingT(t) + + apiError := New( + ClientError, + InvalidParameter, + "error test") + + Expect(apiError.Body.Code).Should(Equal(ErrorCode("InvalidParameter"))) +} + +func TestAcsNewAPIError(t *testing.T) { + RegisterTestingT(t) + + apiError := New( + ClientError, + ScaleDownInternalError, + "error test") + + Expect(apiError.Body.Code).Should(Equal(ErrorCode("ScaleDownInternalError"))) +} diff --git a/pkg/apierror/const.go b/pkg/apierror/const.go new file mode 100644 index 0000000000..27cb6df165 --- /dev/null +++ b/pkg/apierror/const.go @@ -0,0 +1,63 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +package apierror + +// ErrorCategory indicates the kind of error +type ErrorCategory string + +const ( + // ClientError is expected error + ClientError ErrorCategory = "ClientError" + + // InternalError is system or internal error + InternalError ErrorCategory = "InternalError" +) + +// Common Azure Resource Provider API error code +type ErrorCode string + +const ( + // From Microsoft.Azure.ResourceProvider.API.ErrorCode + InvalidParameter ErrorCode = "InvalidParameter" + BadRequest ErrorCode = "BadRequest" + NotFound ErrorCode = "NotFound" + Conflict ErrorCode = "Conflict" + PreconditionFailed ErrorCode = "PreconditionFailed" + OperationNotAllowed ErrorCode = "OperationNotAllowed" + OperationPreempted ErrorCode = "OperationPreempted" + PropertyChangeNotAllowed ErrorCode = "PropertyChangeNotAllowed" + InternalOperationError ErrorCode = "InternalOperationError" + InvalidSubscriptionStateTransition ErrorCode = "InvalidSubscriptionStateTransition" + UnregisterWithResourcesNotAllowed ErrorCode = "UnregisterWithResourcesNotAllowed" + InvalidParameterConflictingProperties ErrorCode = "InvalidParameterConflictingProperties" + SubscriptionNotRegistered ErrorCode = "SubscriptionNotRegistered" + ConflictingUserInput ErrorCode = "ConflictingUserInput" + ProvisioningInternalError ErrorCode = "ProvisioningInternalError" + ProvisioningFailed ErrorCode = "ProvisioningFailed" + NetworkingInternalOperationError ErrorCode = "NetworkingInternalOperationError" + QuotaExceeded ErrorCode = "QuotaExceeded" + Unauthorized ErrorCode = "Unauthorized" + ResourcesOverConstrained ErrorCode = "ResourcesOverConstrained" + ControlPlaneProvisioningInternalError ErrorCode = "ControlPlaneProvisioningInternalError" + ControlPlaneProvisioningSyncError ErrorCode = "ControlPlaneProvisioningSyncError" + ControlPlaneInternalError ErrorCode = "ControlPlaneInternalError" + ControlPlaneCloudProviderNotSet ErrorCode = "ControlPlaneCloudProviderNotSet" + + // From Microsoft.WindowsAzure.ContainerService.API.AcsErrorCode + ScaleDownInternalError ErrorCode = "ScaleDownInternalError" + + // New + PreconditionCheckTimeOut ErrorCode = "PreconditionCheckTimeOut" + UpgradeFailed ErrorCode = "UpgradeFailed" + ScaleError ErrorCode = "ScaleError" + CreateRoleAssignmentError ErrorCode = "CreateRoleAssignmentError" + ServicePrincipalNotFound ErrorCode = "ServicePrincipalNotFound" + ClusterResourceGroupNotFound ErrorCode = "ClusterResourceGroupNotFound" + + // Error codes returned by HCP + UnderlayNotFound ErrorCode = "UnderlayNotFound" + UnderlaysOverConstrained ErrorCode = "UnderlaysOverConstrained" + UnexpectedUnderlayCount ErrorCode = "UnexpectedUnderlayCount" +) diff --git a/pkg/apierror/helper.go b/pkg/apierror/helper.go new file mode 100644 index 0000000000..4b96a479de --- /dev/null +++ b/pkg/apierror/helper.go @@ -0,0 +1,47 @@ +package apierror + +import ( + "encoding/json" + "net/http" + + "github.com/Azure/azure-sdk-for-go/arm/resources/resources" +) + +// ExtractCodeFromARMHttpResponse returns the ARM error's Code field +// If not found return defaultCode +func ExtractCodeFromARMHttpResponse(resp *http.Response, defaultCode ErrorCode) ErrorCode { + if resp == nil { + return defaultCode + } + decoder := json.NewDecoder(resp.Body) + errorJSON := ErrorResponse{} + if err := decoder.Decode(&errorJSON); err != nil { + return defaultCode + } + + if errorJSON.Body.Code == "" { + return defaultCode + } + return ErrorCode(errorJSON.Body.Code) +} + +//ConvertToAPIError turns a ManagementErrorWithDetails into a apierror.Error +func ConvertToAPIError(mError *resources.ManagementErrorWithDetails) *Error { + retVal := &Error{} + if mError.Code != nil { + retVal.Code = ErrorCode(*mError.Code) + } + if mError.Message != nil { + retVal.Message = *mError.Message + } + if mError.Target != nil { + retVal.Target = *mError.Target + } + if mError.Details != nil { + retVal.Details = []Error{} + for _, me := range *mError.Details { + retVal.Details = append(retVal.Details, *ConvertToAPIError(&me)) + } + } + return retVal +} diff --git a/pkg/apierror/helper_test.go b/pkg/apierror/helper_test.go new file mode 100644 index 0000000000..0c45711974 --- /dev/null +++ b/pkg/apierror/helper_test.go @@ -0,0 +1,21 @@ +package apierror + +import ( + "bytes" + "io/ioutil" + "net/http" + "testing" + + . "github.com/onsi/gomega" +) + +func TestExtractCodeFromARMHttpResponse(t *testing.T) { + RegisterTestingT(t) + + resp := &http.Response{ + Body: ioutil.NopCloser(bytes.NewBufferString(`{"error":{"code":"ResourceGroupNotFound","message":"Resource group 'jiren-fakegroup' could not be found."}}`)), + } + + code := ExtractCodeFromARMHttpResponse(resp, "") + Expect(code).To(Equal(ErrorCode("ResourceGroupNotFound"))) +} diff --git a/pkg/apierror/types.go b/pkg/apierror/types.go new file mode 100644 index 0000000000..307044a1a3 --- /dev/null +++ b/pkg/apierror/types.go @@ -0,0 +1,39 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +package apierror + +import "encoding/json" + +// Error is the OData v4 format, used by the RPC and +// will go into the v2.2 Azure REST API guidelines +type Error struct { + Code ErrorCode `json:"code"` + Message string `json:"message"` + Target string `json:"target,omitempty"` + Details []Error `json:"details,omitempty"` + + Category ErrorCategory `json:"-"` + ExceptionType string `json:"-"` + InternalDetail string `json:"-"` +} + +// ErrorResponse defines Resource Provider API 2.0 Error Response Content structure +type ErrorResponse struct { + Body Error `json:"error"` +} + +// Error implements error interface to return error in json +func (e *ErrorResponse) Error() string { + return e.Body.Error() +} + +// Error implements error interface to return error in json +func (e *Error) Error() string { + output, err := json.MarshalIndent(e, " ", " ") + if err != nil { + return err.Error() + } + return string(output) +} diff --git a/pkg/armhelpers/deploymentError.go b/pkg/armhelpers/deploymentError.go index 8d1a9a89e8..8efcb68660 100644 --- a/pkg/armhelpers/deploymentError.go +++ b/pkg/armhelpers/deploymentError.go @@ -5,74 +5,13 @@ import ( "strings" "github.com/Azure/acs-engine/pkg/api" + "github.com/Azure/acs-engine/pkg/apierror" "github.com/Azure/azure-sdk-for-go/arm/resources/resources" "github.com/sirupsen/logrus" ) -//TODO move pkg/core/apierror/ to acs-engine - -// ErrorCategory indicates the kind of error -type ErrorCategory string - -const ( - // ClientError is expected error - ClientError ErrorCategory = "ClientError" - - // InternalError is system or internal error - InternalError ErrorCategory = "InternalError" -) - -// Error is the OData v4 format, used by the RPC and -// will go into the v2.2 Azure REST API guidelines -type Error struct { - Code string `json:"code"` - Message string `json:"message"` - Target string `json:"target,omitempty"` - Details []Error `json:"details,omitempty"` - - Category ErrorCategory `json:"-"` -} - -// ErrorResponse defines Resource Provider API 2.0 Error Response Content structure -type ErrorResponse struct { - Body Error `json:"error"` -} - -// DeploymentError defines deployment error along with deployment operation errors -type DeploymentError struct { - RootError error - OperationErrors []*ErrorResponse -} - -// Error implements error interface to return error in json -func (e *DeploymentError) Error() string { - if len(e.OperationErrors) == 0 { - return e.RootError.Error() - } - errStrList := make([]string, len(e.OperationErrors)+1) - errStrList[0] = e.RootError.Error() - for i, errResp := range e.OperationErrors { - errStrList[i+1] = errResp.Error() - } - return strings.Join(errStrList, " | ") -} - -// Error implements error interface to return error in json -func (e *ErrorResponse) Error() string { - return e.Body.Error() -} - -// Error implements error interface to return error in json -func (e *Error) Error() string { - output, err := json.MarshalIndent(e, " ", " ") - if err != nil { - return err.Error() - } - return string(output) -} - -func toArmError(logger *logrus.Entry, operation resources.DeploymentOperation) (*ErrorResponse, error) { - errresp := &ErrorResponse{} +func toErrorResponse(logger *logrus.Entry, operation resources.DeploymentOperation) (*apierror.ErrorResponse, error) { + errresp := &apierror.ErrorResponse{} if operation.Properties != nil && operation.Properties.StatusMessage != nil { b, err := json.MarshalIndent(operation.Properties.StatusMessage, "", " ") if err != nil { @@ -84,101 +23,124 @@ func toArmError(logger *logrus.Entry, operation resources.DeploymentOperation) ( return nil, err } } + errresp.Body.Category = getErrorCategory(errresp.Body.Code) return errresp, nil } -func toArmErrors(logger *logrus.Entry, deploymentName string, operationsList resources.DeploymentOperationsListResult) ([]*ErrorResponse, error) { - ret := []*ErrorResponse{} - - if operationsList.Value == nil { - return ret, nil +func getErrorCategory(code apierror.ErrorCode) apierror.ErrorCategory { + switch code { + case apierror.InvalidParameter, + apierror.BadRequest, + apierror.OperationNotAllowed, + apierror.PropertyChangeNotAllowed, + apierror.UnregisterWithResourcesNotAllowed, + apierror.InvalidParameterConflictingProperties, + apierror.SubscriptionNotRegistered, + apierror.ConflictingUserInput, + apierror.QuotaExceeded, + apierror.Unauthorized, + apierror.ResourcesOverConstrained: + return apierror.ClientError + default: + return apierror.InternalError } +} - for _, operation := range *operationsList.Value { - if operation.Properties == nil || operation.Properties.ProvisioningState == nil || *operation.Properties.ProvisioningState != string(api.Failed) { - continue - } +// GetDeploymentError returns deployment error +func GetDeploymentError(res *resources.DeploymentExtended, az ACSEngineClient, logger *logrus.Entry, resourceGroupName, deploymentName string) (*apierror.Error, error) { + logger.Infof("Getting detailed deployment errors for %s", deploymentName) - errresp, err := toArmError(logger, operation) + if res == nil || res.Properties == nil || res.Properties.ProvisioningState == nil { + return nil, nil + } + properties := res.Properties + + switch *properties.ProvisioningState { + case string(api.Canceled): + logger.Warning("template deployment has been canceled") + return &apierror.Error{ + Code: apierror.ProvisioningFailed, + Message: "template deployment has been canceled", + Category: apierror.ClientError}, nil + + case string(api.Failed): + var top int32 = 1 + results := make([]resources.DeploymentOperationsListResult, top) + res, err := az.ListDeploymentOperations(resourceGroupName, deploymentName, &top) if err != nil { - logger.Warnf("unable to convert deployment operation to error response in deployment %s from ARM. error: %v", deploymentName, err) - continue + logger.Errorf("unable to list deployment operations %s. error: %v", deploymentName, err) + return nil, err } + results[0] = res - if len(errresp.Body.Code) > 0 { - logger.Warnf("got failed deployment operation in deployment %s. error: %v", deploymentName, errresp.Error()) + for res.NextLink != nil { + res, err = az.ListDeploymentOperationsNextResults(res) + if err != nil { + logger.Warningf("unable to list next deployment operations %s. error: %v", deploymentName, err) + break + } + + results = append(results, res) } - ret = append(ret, errresp) - } - return ret, nil -} + return analyzeDeploymentResultAndSaveError(resourceGroupName, deploymentName, results, logger) -//TODO errorCode is ErrorCode -func newErrorResponse(errorCategory ErrorCategory, errorCode string, message string) *ErrorResponse { - return &ErrorResponse{ - Body: Error{ - Code: errorCode, - Message: message, - Category: errorCategory, - }, + default: + return nil, nil } } -// GetDeploymentError returns deployment error -func GetDeploymentError(res *resources.DeploymentExtended, rootError error, az ACSEngineClient, logger *logrus.Entry, resourceGroupName, deploymentName string) (*DeploymentError, error) { - if rootError == nil { - return nil, nil - } - logger.Infof("Getting detailed deployment errors for %s", deploymentName) - deploymentError := &DeploymentError{RootError: rootError} - - if res != nil && res.Response.Response != nil && res.Body != nil { - armErr := &ErrorResponse{} - if d := json.NewDecoder(res.Body); d != nil { - if err := d.Decode(armErr); err == nil { - logger.Errorf("StatusCode: %d, ErrorCode: %s, ErrorMessage: %s", res.Response.StatusCode, armErr.Body.Code, armErr.Body.Message) - deploymentError.OperationErrors = append(deploymentError.OperationErrors, armErr) - switch { - case res.Response.StatusCode < 500 && res.Response.StatusCode >= 400: - armErr.Body.Category = ClientError - return deploymentError, nil - case res.Response.StatusCode >= 500: - armErr.Body.Category = InternalError - return deploymentError, nil - } +func analyzeDeploymentResultAndSaveError(resourceGroupName, deploymentName string, + operationLists []resources.DeploymentOperationsListResult, logger *logrus.Entry) (*apierror.Error, error) { + var errresp *apierror.ErrorResponse + var err error + errs := []string{} + isInternalErr := false + failedCnt := 0 + for _, operationsList := range operationLists { + if operationsList.Value == nil { + continue + } + + for _, operation := range *operationsList.Value { + if operation.Properties == nil || *operation.Properties.ProvisioningState != string(api.Failed) { + continue + } + + // log the full deployment operation error response + if operation.ID != nil && operation.OperationID != nil { + b, _ := json.Marshal(operation.Properties) + logger.Infof("deployment operation ID %s, operationID %s, prooperties: %s", *operation.ID, *operation.OperationID, b) } else { - logger.Errorf("unable to unmarshal response into apierror: %v", err) + logger.Error("either deployment ID or operationID is nil") } - } - } else { - logger.Errorf("Got error from Azure SDK without response from ARM, error: %v", rootError) - // This is the failed sdk validation before calling ARM path - deploymentError.OperationErrors = append(deploymentError.OperationErrors, newErrorResponse(InternalError, "InternalOperationError", rootError.Error())) - return deploymentError, nil - } - var top int32 = 1 - operationList, err := az.ListDeploymentOperations(resourceGroupName, deploymentName, &top) - if err != nil { - logger.Warnf("unable to list deployment operations: %v", err) - return nil, err - } - eList, err := toArmErrors(logger, deploymentName, operationList) - if err != nil { - return nil, err + failedCnt++ + errresp, err = toErrorResponse(logger, operation) + if err != nil { + logger.Errorf("unable to convert deployment operation to error response in deployment %s from ARM. error: %v", deploymentName, err) + return nil, err + } + if errresp.Body.Category == apierror.InternalError { + isInternalErr = true + } + errs = append(errs, errresp.Error()) + } } - deploymentError.OperationErrors = append(deploymentError.OperationErrors, eList...) - for operationList.NextLink != nil { - operationList, err = az.ListDeploymentOperationsNextResults(operationList) - if err != nil { - logger.Warnf("unable to list next deployment operations: %v", err) - break + provisionErr := &apierror.Error{} + if failedCnt > 0 { + if isInternalErr { + provisionErr.Category = apierror.InternalError + } else { + provisionErr.Category = apierror.ClientError } - eList, err := toArmErrors(logger, deploymentName, operationList) - if err != nil { - return nil, err + if failedCnt == 1 { + provisionErr = &errresp.Body + } else { + provisionErr.Code = apierror.ProvisioningFailed + provisionErr.Message = strings.Join(errs, "\n") } - deploymentError.OperationErrors = append(deploymentError.OperationErrors, eList...) + return provisionErr, nil } - return deploymentError, nil + + return nil, nil } diff --git a/pkg/armhelpers/deployments.go b/pkg/armhelpers/deployments.go index c354e7fce9..9c926a4d97 100644 --- a/pkg/armhelpers/deployments.go +++ b/pkg/armhelpers/deployments.go @@ -31,7 +31,7 @@ func (az *AzureClient) DeployTemplate(resourceGroupName, deploymentName string, return nil, err } - log.Infof("Finished ARM Deployment (%s).", deploymentName) + log.Infof("Finished ARM Deployment (%s). Error: %v", deploymentName, err) return &res, err } diff --git a/pkg/operations/kubernetesupgrade/upgradeagentnode.go b/pkg/operations/kubernetesupgrade/upgradeagentnode.go index b4dbab9e9a..deb2e94b49 100644 --- a/pkg/operations/kubernetesupgrade/upgradeagentnode.go +++ b/pkg/operations/kubernetesupgrade/upgradeagentnode.go @@ -93,7 +93,7 @@ func (kan *UpgradeAgentNode) CreateNode(poolName string, agentNo int) error { kan.logger.Errorf("Deployment %s failed with error %v", deploymentName, err) // Get deployment error details - deploymentError, e := armhelpers.GetDeploymentError(depExt, err, kan.Client, kan.logger, kan.ResourceGroup, deploymentName) + deploymentError, e := armhelpers.GetDeploymentError(depExt, kan.Client, kan.logger, kan.ResourceGroup, deploymentName) if e != nil { kan.logger.Errorf("Failed to get error details for deployment %s: %v", deploymentName, e) // return original deployment error