retrieving deployment error details during upgrade

pkg/armhelpers/deploymentError.go

@@ -0,0 +1,110 @@
+package armhelpers
+
+import (
+	"encoding/json"
+
+	"github.com/Azure/acs-engine/pkg/api"
+	"github.com/Azure/azure-sdk-for-go/arm/resources/resources"
+	"github.com/sirupsen/logrus"
+)
+
+// Error is the OData v4 format, used by the RPC and
+// will go into the v2.2 Azure REST API guidelines
+type Error struct {
+	Code    string  `json:"code"`
+	Message string  `json:"message"`
+	Target  string  `json:"target,omitempty"`
+	Details []Error `json:"details,omitempty"`
+}
+
+// ErrorResponse  defines Resource Provider API 2.0 Error Response Content structure
+type ErrorResponse struct {
+	Body Error `json:"error"`
+}
+
+// Error implements error interface to return error in json
+func (e *ErrorResponse) Error() string {
+	return e.Body.Error()
+}
+
+// Error implements error interface to return error in json
+func (e *Error) Error() string {
+	output, err := json.MarshalIndent(e, " ", " ")
+	if err != nil {
+		return err.Error()
+	}
+	return string(output)
+}
+
+func toArmError(logger *logrus.Entry, operation resources.DeploymentOperation) (*ErrorResponse, error) {
+	errresp := &ErrorResponse{}
+	if operation.Properties != nil && operation.Properties.StatusMessage != nil {
+		b, err := json.MarshalIndent(operation.Properties.StatusMessage, "", "  ")
+		if err != nil {
+			logger.Errorf("Error occurred marshalling JSON: '%v'", err)
+			return nil, err
+		}
+		if err := json.Unmarshal(b, errresp); err != nil {
+			logger.Errorf("Error occurred unmarshalling JSON: '%v' JSON: '%s'", err, string(b))
+			return nil, err
+		}
+	}
+	return errresp, nil
+}
+
+func toArmErrors(logger *logrus.Entry, deploymentName string, operationsList resources.DeploymentOperationsListResult) ([]*ErrorResponse, error) {
+	ret := []*ErrorResponse{}
+
+	if operationsList.Value == nil {
+		return ret, nil
+	}
+
+	for _, operation := range *operationsList.Value {
+		if operation.Properties == nil || operation.Properties.ProvisioningState == nil || *operation.Properties.ProvisioningState != string(api.Failed) {
+			continue
+		}
+
+		errresp, err := toArmError(logger, operation)
+		if err != nil {
+			logger.Warnf("unable to convert deployment operation to error response in deployment %s from ARM. error: %v", deploymentName, err)
+			continue
+		}
+
+		if len(errresp.Body.Code) > 0 {
+			logger.Warnf("got failed deployment operation in deployment %s. error: %v", deploymentName, errresp.Error())
+		}
+		ret = append(ret, errresp)
+	}
+	return ret, nil
+}
+
+// GetDeploymentError returns deployment error
+func GetDeploymentError(az ACSEngineClient, logger *logrus.Entry, resourceGroupName, deploymentName string) ([]*ErrorResponse, error) {
+	errList := []*ErrorResponse{}
+	logger.Infof("Getting detailed deployment errors for %s", deploymentName)
+
+	var top int32 = 1
+	operationList, err := az.ListDeploymentOperations(resourceGroupName, deploymentName, &top)
+	if err != nil {
+		logger.Warnf("unable to list deployment operations: %v", err)
+		return nil, err
+	}
+	eList, err := toArmErrors(logger, deploymentName, operationList)
+	if err != nil {
+		return nil, err
+	}
+	errList = append(errList, eList...)
+	for operationList.NextLink != nil {
+		operationList, err = az.ListDeploymentOperationsNextResults(operationList)
+		if err != nil {
+			logger.Warnf("unable to list next deployment operations: %v", err)
+			break
+		}
+		eList, err := toArmErrors(logger, deploymentName, operationList)
+		if err != nil {
+			return nil, err
+		}
+		errList = append(errList, eList...)
+	}
+	return errList, nil
+}

pkg/armhelpers/interfaces.go

@@ -78,6 +78,14 @@ type ACSEngineClient interface {
 	GetKubernetesClient(masterURL, kubeConfig string, interval, timeout time.Duration) (KubernetesClient, error)
 
 	ListProviders() (resources.ProviderListResult, error)
+
+	// DEPLOYMENTS
+
+	// ListDeploymentOperations gets all deployments operations for a deployment.
+	ListDeploymentOperations(resourceGroupName string, deploymentName string, top *int32) (result resources.DeploymentOperationsListResult, err error)
+
+	// ListDeploymentOperationsNextResults retrieves the next set of results, if any.
+	ListDeploymentOperationsNextResults(lastResults resources.DeploymentOperationsListResult) (result resources.DeploymentOperationsListResult, err error)
 }
 
 // ACSStorageClient interface models the azure storage client

pkg/armhelpers/mockclients.go

@@ -397,3 +397,13 @@ func (mc *MockACSEngineClient) ListProviders() (resources.ProviderListResult, er
 
 	return resources.ProviderListResult{}, nil
 }
+
+// ListDeploymentOperations gets all deployments operations for a deployment.
+func (mc *MockACSEngineClient) ListDeploymentOperations(resourceGroupName string, deploymentName string, top *int32) (result resources.DeploymentOperationsListResult, err error) {
+	return resources.DeploymentOperationsListResult{}, nil
+}
+
+// ListDeploymentOperationsNextResults retrieves the next set of results, if any.
+func (mc *MockACSEngineClient) ListDeploymentOperationsNextResults(lastResults resources.DeploymentOperationsListResult) (result resources.DeploymentOperationsListResult, err error) {
+	return resources.DeploymentOperationsListResult{}, nil
+}

pkg/operations/kubernetesupgrade/upgradeagentnode.go

@@ -3,6 +3,7 @@ package kubernetesupgrade
 import (
 	"fmt"
 	"math/rand"
+	"strings"
 	"time"
 
 	"k8s.io/client-go/pkg/api/v1/node"
@@ -87,11 +88,23 @@ func (kan *UpgradeAgentNode) CreateNode(poolName string, agentNo int) error {
 		kan.ParametersMap,
 		nil)
 
-	if err != nil {
-		return err
+	if err == nil {
+		return nil
 	}
 
-	return nil
+	kan.logger.Errorf("Deployment %s failed with error %v", deploymentName, err)
+	// Get deployment error details
+	errRespList, e := armhelpers.GetDeploymentError(kan.Client, kan.logger, kan.ResourceGroup, deploymentName)
+	if e != nil || len(errRespList) == 0 {
+		kan.logger.Errorf("Failed to get error details for deployment %s: %v", deploymentName, e)
+		// return original deployment error
+		return err
+	}
+	errStrList := make([]string, len(errRespList))
+	for i, errResp := range errRespList {
+		errStrList[i] = errResp.Error()
+	}
+	return fmt.Errorf("%s", strings.Join(errStrList, " | "))
 }
 
 // Validate will verify that agent node has been upgraded as expected.
@@ -100,7 +113,7 @@ func (kan *UpgradeAgentNode) Validate(vmName *string) error {
 		kan.logger.Warningf("VM name was empty. Skipping node condition check")
 		return nil
 	}
-
+	kan.logger.Infof("Validating %s", *vmName)
 	var masterURL string
 	if kan.UpgradeContainerService.Properties.HostedMasterProfile != nil {
 		masterURL = kan.UpgradeContainerService.Properties.HostedMasterProfile.FQDN