From b8cd9fa5212f939c3542e67b4abf3b98ea561c4c Mon Sep 17 00:00:00 2001 From: Brett Elliott Date: Tue, 16 Mar 2021 16:28:26 +0100 Subject: [PATCH] expose IP_SPACE_EXHAUSTED --- .../gce/autoscaling_gce_client.go | 21 ++++-- .../gce/autoscaling_gce_client_test.go | 65 +++++++++++++++++++ 2 files changed, 81 insertions(+), 5 deletions(-) diff --git a/cluster-autoscaler/cloudprovider/gce/autoscaling_gce_client.go b/cluster-autoscaler/cloudprovider/gce/autoscaling_gce_client.go index f198145c7822..5bede97359d5 100644 --- a/cluster-autoscaler/cloudprovider/gce/autoscaling_gce_client.go +++ b/cluster-autoscaler/cloudprovider/gce/autoscaling_gce_client.go @@ -39,14 +39,18 @@ const ( defaultOperationWaitTimeout = 20 * time.Second defaultOperationPollInterval = 100 * time.Millisecond defaultOperationDeletionPollInterval = 1 * time.Second - // ErrorCodeQuotaExceeded is error code used in InstanceErrorInfo if quota exceeded error occurs. + // ErrorCodeQuotaExceeded is an error code used in InstanceErrorInfo if quota exceeded error occurs. ErrorCodeQuotaExceeded = "QUOTA_EXCEEDED" - // ErrorCodeResourcePoolExhausted is error code used in InstanceErrorInfo if requested resources + // ErrorCodeResourcePoolExhausted is an error code used in InstanceErrorInfo if requested resources // cannot be provisioned by cloud provider. ErrorCodeResourcePoolExhausted = "RESOURCE_POOL_EXHAUSTED" - // ErrorCodeOther is error code used in InstanceErrorInfo if other error occurs. + // ErrorIPSpaceExhausted is an error code used in InstanceErrorInfo if the IP space has been + // exhausted. + ErrorIPSpaceExhausted = "IP_SPACE_EXHAUSTED" + + // ErrorCodeOther is an error code used in InstanceErrorInfo if other error occurs. ErrorCodeOther = "OTHER" ) @@ -260,9 +264,12 @@ func (client *autoscalingGceClientV1) FetchMigInstances(migRef GceRef) ([]cloudp if isResourcePoolExhaustedErrorCode(instanceError.Code) { errorInfo.ErrorClass = cloudprovider.OutOfResourcesErrorClass errorInfo.ErrorCode = ErrorCodeResourcePoolExhausted - } else if isQuotaExceededErrorCoce(instanceError.Code) { + } else if isQuotaExceededErrorCode(instanceError.Code) { errorInfo.ErrorClass = cloudprovider.OutOfResourcesErrorClass errorInfo.ErrorCode = ErrorCodeQuotaExceeded + } else if isIPSpaceExhaustedErrorCode(instanceError.Code) { + errorInfo.ErrorClass = cloudprovider.OtherErrorClass + errorInfo.ErrorCode = ErrorIPSpaceExhausted } else if isInstanceNotRunningYet(gceInstance) { if !errorFound { // do not override error code with OTHER @@ -314,10 +321,14 @@ func isResourcePoolExhaustedErrorCode(errorCode string) bool { return errorCode == "RESOURCE_POOL_EXHAUSTED" || errorCode == "ZONE_RESOURCE_POOL_EXHAUSTED" || errorCode == "ZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILS" } -func isQuotaExceededErrorCoce(errorCode string) bool { +func isQuotaExceededErrorCode(errorCode string) bool { return strings.Contains(errorCode, "QUOTA") } +func isIPSpaceExhaustedErrorCode(errorCode string) bool { + return strings.Contains(errorCode, "IP_SPACE_EXHAUSTED") +} + func isInstanceNotRunningYet(gceInstance *gce.ManagedInstance) bool { return gceInstance.InstanceStatus == "" || gceInstance.InstanceStatus == "PROVISIONING" || gceInstance.InstanceStatus == "STAGING" } diff --git a/cluster-autoscaler/cloudprovider/gce/autoscaling_gce_client_test.go b/cluster-autoscaler/cloudprovider/gce/autoscaling_gce_client_test.go index e8caa144dd6e..a15fac70e3ff 100644 --- a/cluster-autoscaler/cloudprovider/gce/autoscaling_gce_client_test.go +++ b/cluster-autoscaler/cloudprovider/gce/autoscaling_gce_client_test.go @@ -17,10 +17,12 @@ limitations under the License. package gce import ( + "encoding/json" "net/http" "testing" "time" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" test_util "k8s.io/autoscaler/cluster-autoscaler/utils/test" "github.com/stretchr/testify/assert" @@ -97,3 +99,66 @@ func TestWaitForOpTimeout(t *testing.T) { err := g.waitForOp(operation, projectId, zoneB, false) assert.Error(t, err) } + +func TestErrors(t *testing.T) { + const instanceUrl = "https://content.googleapis.com/compute/v1/projects/myprojid/zones/myzone/instances/myinst" + server := test_util.NewHttpServerMock() + defer server.Close() + g := newTestAutoscalingGceClient(t, "project1", server.URL) + + testCases := []struct { + errorCodes []string + expectedErrorCode string + expectedErrorClass cloudprovider.InstanceErrorClass + }{ + { + errorCodes: []string{"IP_SPACE_EXHAUSTED"}, + expectedErrorCode: "IP_SPACE_EXHAUSTED", + expectedErrorClass: cloudprovider.OtherErrorClass, + }, + { + errorCodes: []string{"RESOURCE_POOL_EXHAUSTED", "ZONE_RESOURCE_POOL_EXHAUSTED", "ZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILS"}, + expectedErrorCode: "RESOURCE_POOL_EXHAUSTED", + expectedErrorClass: cloudprovider.OutOfResourcesErrorClass, + }, + { + errorCodes: []string{"QUOTA"}, + expectedErrorCode: "QUOTA_EXCEEDED", + expectedErrorClass: cloudprovider.OutOfResourcesErrorClass, + }, + { + errorCodes: []string{"xyz", "abc"}, + expectedErrorCode: "OTHER", + expectedErrorClass: cloudprovider.OtherErrorClass, + }, + } + for _, tc := range testCases { + for _, errorCode := range tc.errorCodes { + lmiResponse := gce_api.InstanceGroupManagersListManagedInstancesResponse{ + ManagedInstances: []*gce_api.ManagedInstance{ + { + Instance: instanceUrl, + CurrentAction: "CREATING", + LastAttempt: &gce_api.ManagedInstanceLastAttempt{ + Errors: &gce_api.ManagedInstanceLastAttemptErrors{ + Errors: []*gce_api.ManagedInstanceLastAttemptErrorsErrors{ + { + Code: errorCode, + }, + }, + }, + }, + }, + }, + } + b, err := json.Marshal(lmiResponse) + assert.NoError(t, err) + server.On("handle", "/zones/instanceGroupManagers/listManagedInstances").Return(string(b)).Times(1) + instances, err := g.FetchMigInstances(GceRef{}) + assert.NoError(t, err) + assert.Equal(t, tc.expectedErrorCode, instances[0].Status.ErrorInfo.ErrorCode) + assert.Equal(t, tc.expectedErrorClass, instances[0].Status.ErrorInfo.ErrorClass) + } + } + mock.AssertExpectationsForObjects(t, server) +}