Merge pull request #3945 from brett-elliott/surface_exhausted

expose IP_SPACE_EXHAUSTED
kubernetes · Mar 16, 2021 · 0d408a3 · 0d408a3
2 parents b2951aa + b8cd9fa
commit 0d408a3
Show file tree

Hide file tree

Showing 2 changed files with 81 additions and 5 deletions.
diff --git a/cluster-autoscaler/cloudprovider/gce/autoscaling_gce_client.go b/cluster-autoscaler/cloudprovider/gce/autoscaling_gce_client.go
@@ -39,14 +39,18 @@ const (
 	defaultOperationWaitTimeout          = 20 * time.Second
 	defaultOperationPollInterval         = 100 * time.Millisecond
 	defaultOperationDeletionPollInterval = 1 * time.Second
-	// ErrorCodeQuotaExceeded is error code used in InstanceErrorInfo if quota exceeded error occurs.
+	// ErrorCodeQuotaExceeded is an error code used in InstanceErrorInfo if quota exceeded error occurs.
 	ErrorCodeQuotaExceeded = "QUOTA_EXCEEDED"
 
-	// ErrorCodeResourcePoolExhausted is error code used in InstanceErrorInfo if requested resources
+	// ErrorCodeResourcePoolExhausted is an error code used in InstanceErrorInfo if requested resources
 	// cannot be provisioned by cloud provider.
 	ErrorCodeResourcePoolExhausted = "RESOURCE_POOL_EXHAUSTED"
 
-	// ErrorCodeOther is error code used in InstanceErrorInfo if other error occurs.
+	// ErrorIPSpaceExhausted is an error code used in InstanceErrorInfo if the IP space has been
+	// exhausted.
+	ErrorIPSpaceExhausted = "IP_SPACE_EXHAUSTED"
+
+	// ErrorCodeOther is an error code used in InstanceErrorInfo if other error occurs.
 	ErrorCodeOther = "OTHER"
 )
 
@@ -260,9 +264,12 @@ func (client *autoscalingGceClientV1) FetchMigInstances(migRef GceRef) ([]cloudp
 				if isResourcePoolExhaustedErrorCode(instanceError.Code) {
 					errorInfo.ErrorClass = cloudprovider.OutOfResourcesErrorClass
 					errorInfo.ErrorCode = ErrorCodeResourcePoolExhausted
-				} else if isQuotaExceededErrorCoce(instanceError.Code) {
+				} else if isQuotaExceededErrorCode(instanceError.Code) {
 					errorInfo.ErrorClass = cloudprovider.OutOfResourcesErrorClass
 					errorInfo.ErrorCode = ErrorCodeQuotaExceeded
+				} else if isIPSpaceExhaustedErrorCode(instanceError.Code) {
+					errorInfo.ErrorClass = cloudprovider.OtherErrorClass
+					errorInfo.ErrorCode = ErrorIPSpaceExhausted
 				} else if isInstanceNotRunningYet(gceInstance) {
 					if !errorFound {
 						// do not override error code with OTHER
@@ -314,10 +321,14 @@ func isResourcePoolExhaustedErrorCode(errorCode string) bool {
 	return errorCode == "RESOURCE_POOL_EXHAUSTED" || errorCode == "ZONE_RESOURCE_POOL_EXHAUSTED" || errorCode == "ZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILS"
 }
 
-func isQuotaExceededErrorCoce(errorCode string) bool {
+func isQuotaExceededErrorCode(errorCode string) bool {
 	return strings.Contains(errorCode, "QUOTA")
 }
 
+func isIPSpaceExhaustedErrorCode(errorCode string) bool {
+	return strings.Contains(errorCode, "IP_SPACE_EXHAUSTED")
+}
+
 func isInstanceNotRunningYet(gceInstance *gce.ManagedInstance) bool {
 	return gceInstance.InstanceStatus == "" || gceInstance.InstanceStatus == "PROVISIONING" || gceInstance.InstanceStatus == "STAGING"
 }

diff --git a/cluster-autoscaler/cloudprovider/gce/autoscaling_gce_client_test.go b/cluster-autoscaler/cloudprovider/gce/autoscaling_gce_client_test.go
@@ -17,10 +17,12 @@ limitations under the License.
 package gce
 
 import (
+	"encoding/json"
 	"net/http"
 	"testing"
 	"time"
 
+	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider"
 	test_util "k8s.io/autoscaler/cluster-autoscaler/utils/test"
 
 	"github.com/stretchr/testify/assert"
@@ -97,3 +99,66 @@ func TestWaitForOpTimeout(t *testing.T) {
 	err := g.waitForOp(operation, projectId, zoneB, false)
 	assert.Error(t, err)
 }
+
+func TestErrors(t *testing.T) {
+	const instanceUrl = "https://content.googleapis.com/compute/v1/projects/myprojid/zones/myzone/instances/myinst"
+	server := test_util.NewHttpServerMock()
+	defer server.Close()
+	g := newTestAutoscalingGceClient(t, "project1", server.URL)
+
+	testCases := []struct {
+		errorCodes         []string
+		expectedErrorCode  string
+		expectedErrorClass cloudprovider.InstanceErrorClass
+	}{
+		{
+			errorCodes:         []string{"IP_SPACE_EXHAUSTED"},
+			expectedErrorCode:  "IP_SPACE_EXHAUSTED",
+			expectedErrorClass: cloudprovider.OtherErrorClass,
+		},
+		{
+			errorCodes:         []string{"RESOURCE_POOL_EXHAUSTED", "ZONE_RESOURCE_POOL_EXHAUSTED", "ZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILS"},
+			expectedErrorCode:  "RESOURCE_POOL_EXHAUSTED",
+			expectedErrorClass: cloudprovider.OutOfResourcesErrorClass,
+		},
+		{
+			errorCodes:         []string{"QUOTA"},
+			expectedErrorCode:  "QUOTA_EXCEEDED",
+			expectedErrorClass: cloudprovider.OutOfResourcesErrorClass,
+		},
+		{
+			errorCodes:         []string{"xyz", "abc"},
+			expectedErrorCode:  "OTHER",
+			expectedErrorClass: cloudprovider.OtherErrorClass,
+		},
+	}
+	for _, tc := range testCases {
+		for _, errorCode := range tc.errorCodes {
+			lmiResponse := gce_api.InstanceGroupManagersListManagedInstancesResponse{
+				ManagedInstances: []*gce_api.ManagedInstance{
+					{
+						Instance:      instanceUrl,
+						CurrentAction: "CREATING",
+						LastAttempt: &gce_api.ManagedInstanceLastAttempt{
+							Errors: &gce_api.ManagedInstanceLastAttemptErrors{
+								Errors: []*gce_api.ManagedInstanceLastAttemptErrorsErrors{
+									{
+										Code: errorCode,
+									},
+								},
+							},
+						},
+					},
+				},
+			}
+			b, err := json.Marshal(lmiResponse)
+			assert.NoError(t, err)
+			server.On("handle", "/zones/instanceGroupManagers/listManagedInstances").Return(string(b)).Times(1)
+			instances, err := g.FetchMigInstances(GceRef{})
+			assert.NoError(t, err)
+			assert.Equal(t, tc.expectedErrorCode, instances[0].Status.ErrorInfo.ErrorCode)
+			assert.Equal(t, tc.expectedErrorClass, instances[0].Status.ErrorInfo.ErrorClass)
+		}
+	}
+	mock.AssertExpectationsForObjects(t, server)
+}