Merge pull request #130 from openinfradev/TKS-822

feature. check resource quota

cmd/server/main.go

@@ -35,6 +35,7 @@ func init() {
 	flag.String("git-base-url", "https://github.com", "git base url")
 	flag.String("git-account", "decapod10", "git account of admin cluster")
 	flag.String("revision", "main", "revision")
+	flag.String("aws-secret", "awsconfig-secret", "aws secret")
 	flag.Int("migrate-db", 1, "If the values is true, enable db migration. recommend only development")
 
 	// console

go.mod

@@ -4,8 +4,8 @@ go 1.18
 
 require (
 	github.com/Nerzal/gocloak/v13 v13.1.0
-	github.com/aws/aws-sdk-go-v2 v1.17.8
-	github.com/aws/aws-sdk-go-v2/config v1.18.21
+	github.com/aws/aws-sdk-go-v2 v1.20.1
+	github.com/aws/aws-sdk-go-v2/config v1.18.32
 	github.com/aws/aws-sdk-go-v2/service/ses v1.15.7
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/go-playground/locales v0.14.1
@@ -36,16 +36,28 @@ require (
 
 require (
 	github.com/KyleBanks/depth v1.2.1 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.13.20 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.2 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.32 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.26 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.33 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.26 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.12.8 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.8 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.18.9 // indirect
-	github.com/aws/smithy-go v1.13.5 // indirect
+	github.com/aws/aws-sdk-go v1.44.317 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.12 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.13.31 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.7 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.38 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.32 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.38 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ec2 v1.110.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/eks v1.29.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing v1.16.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.20.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.13 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.33 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.32 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.38.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/servicequotas v1.15.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.13.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.21.1 // indirect
+	github.com/aws/smithy-go v1.14.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/emicklei/go-restful v2.9.5+incompatible // indirect
 	github.com/felixge/httpsnoop v1.0.1 // indirect

internal/kubernetes/kubernetes.go

@@ -2,6 +2,8 @@ package kubernetes
 
 import (
 	"context"
+	"fmt"
+	"strings"
 
 	"github.com/spf13/viper"
 
@@ -50,6 +52,46 @@ func GetClientAdminCluster() (*kubernetes.Clientset, error) {
 	return clientset, nil
 }
 
+func GetAwsSecret() (awsAccessKeyId string, awsSecretAccessKey string, err error) {
+	clientset, err := GetClientAdminCluster()
+	if err != nil {
+		return "", "", err
+	}
+
+	secrets, err := clientset.CoreV1().Secrets("argo").Get(context.TODO(), "awsconfig-secret", metav1.GetOptions{})
+	if err != nil {
+		log.Error(err)
+		return "", "", err
+	}
+
+	strCredentials := string(secrets.Data["credentials"][:])
+	arr := strings.Split(strCredentials, "\n")
+	if len(arr) < 3 {
+		return "", "", err
+	}
+
+	fmt.Sscanf(arr[1], "aws_access_key_id = %s", &awsAccessKeyId)
+	fmt.Sscanf(arr[2], "aws_secret_access_key = %s", &awsSecretAccessKey)
+
+	return
+}
+
+func GetAwsAccountIdSecret() (awsAccountId string, err error) {
+	clientset, err := GetClientAdminCluster()
+	if err != nil {
+		return "", err
+	}
+
+	secrets, err := clientset.CoreV1().Secrets("argo").Get(context.TODO(), "tks-aws-user", metav1.GetOptions{})
+	if err != nil {
+		log.Error(err)
+		return "", err
+	}
+
+	awsAccountId = string(secrets.Data["account_id"][:])
+	return
+}
+
 func GetKubeConfig(clusterId string) ([]byte, error) {
 	clientset, err := GetClientAdminCluster()
 	if err != nil {

internal/usecase/dashboard.go

@@ -356,6 +356,11 @@ func (u *DashboardUsecase) getChartFromPrometheus(organizationId string, chartTy
 			}
 		}
 	}
+	sort.Slice(xAxisData, func(i, j int) bool {
+		a, _ := strconv.Atoi(xAxisData[i])
+		b, _ := strconv.Atoi(xAxisData[j])
+		return a < b
+	})
 
 	// cluster 별 y축 계산
 	for _, val := range result.Data.Result {

internal/usecase/stack.go

@@ -6,6 +6,16 @@ import (
 	"strings"
 	"time"
 
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/credentials/stscreds"
+	"github.com/aws/aws-sdk-go-v2/service/ec2"
+	"github.com/aws/aws-sdk-go-v2/service/eks"
+	"github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing"
+	"github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2"
+	"github.com/aws/aws-sdk-go-v2/service/servicequotas"
+	"github.com/aws/aws-sdk-go-v2/service/sts"
 	"github.com/openinfradev/tks-api/internal/helper"
 	"github.com/openinfradev/tks-api/internal/kubernetes"
 	"github.com/openinfradev/tks-api/internal/middleware/auth/request"
@@ -69,7 +79,7 @@ func (u *StackUsecase) Create(ctx context.Context, dto domain.Stack) (stackId do
 		return "", httpErrors.NewInternalServerError(errors.Wrap(err, "Invalid stackTemplateId"), "S_INVALID_STACK_TEMPLATE", "")
 	}
 
-	_, err = u.cloudAccountRepo.Get(dto.CloudAccountId)
+	cloudAccount, err := u.cloudAccountRepo.Get(dto.CloudAccountId)
 	if err != nil {
 		return "", httpErrors.NewInternalServerError(errors.Wrap(err, "Invalid cloudAccountId"), "S_INVALID_CLOUD_ACCOUNT", "")
 	}
@@ -99,6 +109,11 @@ func (u *StackUsecase) Create(ctx context.Context, dto domain.Stack) (stackId do
 		log.InfoWithContext(ctx, err)
 	}
 
+	// Check service quota
+	if err = u.checkAwsResourceQuota(ctx, cloudAccount); err != nil {
+		return "", err
+	}
+
 	workflowId, err := u.argo.SumbitWorkflowFromWftpl(workflow, argowf.SubmitOptions{
 		Parameters: []string{
 			fmt.Sprintf("tks_api_url=%s", viper.GetString("external-address")),
@@ -145,6 +160,168 @@ func (u *StackUsecase) Create(ctx context.Context, dto domain.Stack) (stackId do
 	return dto.ID, nil
 }
 
+func (u *StackUsecase) checkAwsResourceQuota(ctx context.Context, cloudAccount domain.CloudAccount) (err error) {
+	awsAccessKeyId, awsSecretAccessKey, _ := kubernetes.GetAwsSecret()
+	if err != nil || awsAccessKeyId == "" || awsSecretAccessKey == "" {
+		log.ErrorWithContext(ctx, err)
+		return httpErrors.NewInternalServerError(fmt.Errorf("Invalid aws secret."), "", "")
+	}
+
+	cfg, err := config.LoadDefaultConfig(ctx,
+		config.WithCredentialsProvider(credentials.StaticCredentialsProvider{
+			Value: aws.Credentials{
+				AccessKeyID: awsAccessKeyId, SecretAccessKey: awsSecretAccessKey,
+			},
+		}))
+	if err != nil {
+		log.ErrorWithContext(ctx, err)
+	}
+
+	stsSvc := sts.NewFromConfig(cfg)
+
+	if !strings.Contains(cloudAccount.Name, domain.CLOUD_ACCOUNT_INCLUSTER) {
+		log.InfoWithContext(ctx, "Use assume role. awsAccountId : ", cloudAccount.AwsAccountId)
+		creds := stscreds.NewAssumeRoleProvider(stsSvc, "arn:aws:iam::"+cloudAccount.AwsAccountId+":role/controllers.cluster-api-provider-aws.sigs.k8s.io")
+		cfg.Credentials = aws.NewCredentialsCache(creds)
+	}
+	client := servicequotas.NewFromConfig(cfg)
+
+	quotaMap := map[string]string{
+		"L-69A177A2": "elasticloadbalancing", // NLB
+		"L-E9E9831D": "elasticloadbalancing", // Classic
+		"L-A4707A72": "vpc",                  // IGW
+		"L-1194D53C": "eks",                  // Cluster
+		"L-0263D0A3": "ec2",                  // Elastic IP
+	}
+
+	// current usage
+	type CurrentUsage struct {
+		NLB     int
+		CLB     int
+		IGW     int
+		Cluster int
+		EIP     int
+	}
+
+	// get current usage
+	currentUsage := CurrentUsage{}
+	{
+		c := elasticloadbalancingv2.NewFromConfig(cfg)
+		pageSize := int32(100)
+		res, err := c.DescribeLoadBalancers(ctx, &elasticloadbalancingv2.DescribeLoadBalancersInput{
+			PageSize: &pageSize,
+		}, func(o *elasticloadbalancingv2.Options) {
+			o.Region = "ap-northeast-2"
+		})
+		if err != nil {
+			return err
+		}
+
+		for _, elb := range res.LoadBalancers {
+			switch elb.Type {
+			case "network":
+				currentUsage.NLB += 1
+			}
+		}
+	}
+
+	{
+		c := elasticloadbalancing.NewFromConfig(cfg)
+		pageSize := int32(100)
+		res, err := c.DescribeLoadBalancers(ctx, &elasticloadbalancing.DescribeLoadBalancersInput{
+			PageSize: &pageSize,
+		}, func(o *elasticloadbalancing.Options) {
+			o.Region = "ap-northeast-2"
+		})
+		if err != nil {
+			return err
+		}
+		currentUsage.CLB = len(res.LoadBalancerDescriptions)
+	}
+
+	{
+		c := ec2.NewFromConfig(cfg)
+		res, err := c.DescribeInternetGateways(ctx, &ec2.DescribeInternetGatewaysInput{}, func(o *ec2.Options) {
+			o.Region = "ap-northeast-2"
+		})
+		if err != nil {
+			return err
+		}
+		currentUsage.IGW = len(res.InternetGateways)
+	}
+
+	{
+		c := eks.NewFromConfig(cfg)
+		res, err := c.ListClusters(ctx, &eks.ListClustersInput{}, func(o *eks.Options) {
+			o.Region = "ap-northeast-2"
+		})
+		if err != nil {
+			return err
+		}
+		currentUsage.Cluster = len(res.Clusters)
+	}
+
+	{
+		c := ec2.NewFromConfig(cfg)
+		res, err := c.DescribeAddresses(ctx, &ec2.DescribeAddressesInput{}, func(o *ec2.Options) {
+			o.Region = "ap-northeast-2"
+		})
+		if err != nil {
+			log.ErrorWithContext(ctx, err)
+			return err
+		}
+		currentUsage.EIP = len(res.Addresses)
+	}
+
+	for key, val := range quotaMap {
+		res, err := getServiceQuota(client, key, val)
+		if err != nil {
+			return err
+		}
+		log.DebugfWithContext(ctx, "%s %s %v", *res.Quota.QuotaName, *res.Quota.QuotaCode, *res.Quota.Value)
+
+		quotaValue := int(*res.Quota.Value)
+
+		// stack 1개 생성하는데 필요한 quota
+		// Classic 1
+		// Network 5
+		// IGW 1
+		// EIP 3
+		// Cluster 1
+		switch key {
+		case "L-69A177A2": // NLB
+			log.InfofWithContext(ctx, "NLB : usage %d, quota %d", currentUsage.NLB, quotaValue)
+			if quotaValue < currentUsage.NLB+5 {
+				return httpErrors.NewInternalServerError(fmt.Errorf("Not enough quota (NLB). current[%d], quota[%d]", currentUsage.NLB, quotaValue), "S_NOT_ENOUGH_QUOTA", "")
+			}
+		case "L-E9E9831D": // Classic
+			log.InfofWithContext(ctx, "CLB : usage %d, quota %d", currentUsage.CLB, quotaValue)
+			if quotaValue < currentUsage.CLB+1 {
+				return httpErrors.NewInternalServerError(fmt.Errorf("Not enough quota (Classic ELB). current[%d], quota[%d]", currentUsage.CLB, quotaValue), "S_NOT_ENOUGH_QUOTA", "")
+			}
+		case "L-A4707A72": // IGW
+			log.InfofWithContext(ctx, "IGW : usage %d, quota %d", currentUsage.IGW, quotaValue)
+			if quotaValue < currentUsage.IGW+1 {
+				return httpErrors.NewInternalServerError(fmt.Errorf("Not enough quota (Internet Gateway). current[%d], quota[%d]", currentUsage.IGW, quotaValue), "S_NOT_ENOUGH_QUOTA", "")
+			}
+		case "L-1194D53C": // Cluster
+			log.InfofWithContext(ctx, "Cluster : usage %d, quota %d", currentUsage.Cluster, quotaValue)
+			if quotaValue < currentUsage.Cluster+1 {
+				return httpErrors.NewInternalServerError(fmt.Errorf("Not enough quota (EKS cluster quota). current[%d], quota[%d]", currentUsage.Cluster, quotaValue), "S_NOT_ENOUGH_QUOTA", "")
+			}
+		case "L-0263D0A3": // Elastic IP
+			log.InfofWithContext(ctx, "Elastic IP : usage %d, quota %d", currentUsage.EIP, quotaValue)
+			if quotaValue < currentUsage.EIP+3 {
+				return httpErrors.NewInternalServerError(fmt.Errorf("Not enough quota (Elastic IP). current[%d], quota[%d]", currentUsage.EIP, quotaValue), "S_NOT_ENOUGH_QUOTA", "")
+			}
+		}
+
+	}
+
+	//return fmt.Errorf("Always return err")
+	return nil
+}
+
 func (u *StackUsecase) Get(ctx context.Context, stackId domain.StackId) (out domain.Stack, err error) {
 	cluster, err := u.clusterRepo.Get(domain.ClusterId(stackId))
 	if err != nil {
@@ -590,3 +767,16 @@ func parseStatusDescription(statusDesc string) (step int) {
 	}
 	return
 }
+
+func getServiceQuota(client *servicequotas.Client, quotaCode string, serviceCode string) (res *servicequotas.GetServiceQuotaOutput, err error) {
+	res, err = client.GetServiceQuota(context.TODO(), &servicequotas.GetServiceQuotaInput{
+		QuotaCode:   &quotaCode,
+		ServiceCode: &serviceCode,
+	}, func(o *servicequotas.Options) {
+		o.Region = "ap-northeast-2"
+	})
+	if err != nil {
+		return nil, err
+	}
+	return
+}

pkg/domain/stack.go

@@ -61,9 +61,9 @@ func (m StackStatus) FromString(s string) StackStatus {
 	return StackStatus_PENDING
 }
 
-const MAX_STEP_CLUSTER_CREATE = 13
+const MAX_STEP_CLUSTER_CREATE = 16
 const MAX_STEP_CLUSTER_REMOVE = 11
-const MAX_STEP_LMA_CREATE_PRIMARY = 39
+const MAX_STEP_LMA_CREATE_PRIMARY = 42
 const MAX_STEP_LMA_CREATE_MEMBER = 27
 const MAX_STEP_LMA_REMOVE = 11
 const MAX_STEP_SM_CREATE = 22

pkg/httpErrors/errorCode.go

@@ -4,6 +4,7 @@ type ErrorCode string
 
 var errorMap = map[ErrorCode]string{
 	// Common
+	"C_INTERNAL_ERROR":            "예상하지 못한 오류가 발생했습니다. 문제가 계속되면 관리자에게 문의해주세요.",
 	"C_INVALID_ACCOUNT_ID":        "유효하지 않은 어카운트 아이디입니다. 어카운트 아이디를 확인하세요.",
 	"C_INVALID_STACK_ID":          "유효하지 않은 스택 아이디입니다. 스택 아이디를 확인하세요.",
 	"C_INVALID_CLUSTER_ID":        "유효하지 않은 클러스터 아이디입니다. 클러스터 아이디를 확인하세요.",
@@ -53,6 +54,7 @@ var errorMap = map[ErrorCode]string{
 	"S_REMAIN_CLUSTER_FOR_DELETION": "프라이머리 클러스터를 지우기 위해서는 조직내의 모든 클러스터를 삭제해야 합니다.",
 	"S_FAILED_GET_CLUSTERS":         "클러스터를 가져오는데 실패했습니다.",
 	"S_FAILED_DELETE_EXISTED_ASA":   "지우고자 하는 스택에 남아 있는 앱서빙앱이 있습니다.",
+	"S_NOT_ENOUGH_QUOTA":            "AWS 의 resource quota 가 부족합니다. 관리자에게 문의하세요.",
 
 	// Alert
 	"AL_NOT_FOUND_ALERT": "지정한 앨럿이 존재하지 않습니다.",

pkg/httpErrors/httpErrors.go

@@ -63,11 +63,16 @@ func (e RestError) Text() string {
 }
 
 func NewRestError(status int, err error, code ErrorCode, text string) IRestError {
+	if code == "" && text == "" {
+		code = ErrorCode("C_INTERNAL_ERROR")
+	}
+
 	t := code.GetText()
 	if text != "" {
 		t = text
 	}
 	log.Info(t)
+
 	return RestError{
 		ErrStatus:  status,
 		ErrCode:    string(code),