Merge pull request #138 from openinfradev/eks_addons

EKS 클러스터 설치 지원
openinfradev · Feb 15, 2023 · bd3353a · bd3353a
2 parents ef87b6e + d551966
commit bd3353a
Show file tree

Hide file tree

Showing 6 changed files with 151 additions and 16 deletions.
diff --git a/dockerfiles/Dockerfile.tks_aws b/dockerfiles/Dockerfile.tks_aws
@@ -0,0 +1,8 @@
+FROM weaveworks/eksctl AS eksctl
+#FROM amazon/aws-cli AS awscli
+
+#make a docker image with this CLI: docker build -t sktcloud/tks-aws:v1.0.0 -f Dockerfile.tks_aws .
+FROM alpine
+COPY --from=eksctl /usr/local/bin/eksctl /usr/bin/eksctl
+RUN apk update
+RUN apk add aws-cli
diff --git a/git-repo/create-cluster-repo.yaml b/git-repo/create-cluster-repo.yaml
@@ -43,6 +43,11 @@ spec:
         INFRA_PROVIDER="$(cat ${CONTRACT_ID}/$TEMPLATE_NAME/tks-cluster/kustomization.yaml | grep /infra/ | awk -F \/ '{print $3}')"
         echo ${INFRA_PROVIDER} | tee /mnt/out/infra_provider.txt
 
+        if [ "$INFRA_PROVIDER" = "aws" ]; then
+            eks_enabled=$(cat ${CONTRACT_ID}/$TEMPLATE_NAME/tks-cluster/site-values.yaml  | grep eksEnabled | awk '{print $2}')
+            echo $eks_enabled | tee /mnt/out/managed_cluster.txt
+        fi
+
         cp -r ${CONTRACT_ID}/${TEMPLATE_NAME} ${CLUSTER_ID}/${CLUSTER_ID}
         cp -r ${CONTRACT_ID}/_github ${CLUSTER_ID}/.github
 
@@ -65,6 +70,7 @@ spec:
             sed -i "s/clusterName:\ cluster.local/clusterName:\ $CLUSTER_ID/g" $CLUSTER_ID/$CLUSTER_ID/tks-cluster/site-values.yaml
             sed -i "s/sshKeyName:\ CHANGEME/sshKeyName: $val_ssh_key/g" $CLUSTER_ID/$CLUSTER_ID/tks-cluster/site-values.yaml
             sed -i "s/clusterRegion:\ CHANGEME/clusterRegion: $val_region/g" $CLUSTER_ID/$CLUSTER_ID/tks-cluster/site-values.yaml
+            sed -i "s/awsAccountID:\ CHANGEME/awsAccountID: \"$AWS_ACCOUNT_ID\"/g" $CLUSTER_ID/$CLUSTER_ID/tks-cluster/site-values.yaml
             sed -i "s/mdNumOfAz:\ CHANGEME/mdNumOfAz: $val_num_of_az/g" $CLUSTER_ID/$CLUSTER_ID/tks-cluster/site-values.yaml
             sed -i "s/mdMinSizePerAz:\ CHANGEME/mdMinSizePerAz: $val_min_size/g" $CLUSTER_ID/$CLUSTER_ID/tks-cluster/site-values.yaml
             sed -i "s/mdMaxSizePerAz:\ CHANGEME/mdMaxSizePerAz: $val_max_size/g" $CLUSTER_ID/$CLUSTER_ID/tks-cluster/site-values.yaml
@@ -121,6 +127,8 @@ spec:
       envFrom:
         - secretRef:
             name: "git-svc-token"
+        - secretRef:
+            name: "aws-account-id"
       env:
       - name: CONTRACT_ID
         value: "{{workflow.parameters.contract_id}}"
@@ -142,6 +150,10 @@ spec:
         valueFrom:
           default: "Something wrong"
           path: /mnt/out/infra_provider.txt
+      - name: managed_cluster
+        valueFrom:
+          default: "Something wrong"
+          path: /mnt/out/managed_cluster.txt
 
   - name: createRepoCredential
     activeDeadlineSeconds: 120

diff --git a/tks-cluster/aws-ebs-csi-iam-yaml b/tks-cluster/aws-ebs-csi-iam-yaml
@@ -0,0 +1,69 @@
+apiVersion: argoproj.io/v1alpha1
+kind: WorkflowTemplate
+metadata:
+  name: aws-ebs-csi-iam
+  namespace: argo
+spec:
+  entrypoint: createIAMRole
+  arguments:
+    parameters:
+    - name: cluster_id
+      value: "Cc81dd656"
+
+  volumes:
+    - name: awsconfig
+      secret:
+        secretName: awsconfig-secret
+
+  templates:
+  - name: createIAMRole
+    activeDeadlineSeconds: 1800
+    container:
+      image: sktcloud/tks-aws:v1.0.0
+      command:
+      - /bin/bash
+      - -exc
+      - |
+        mkdir ~/.aws
+        cp /aws/* ~/.aws/
+
+        oidc_id=$(aws eks describe-cluster --name $CLUSTER --query "cluster.identity.oidc.issuer" --output text | cut -d '/' -f 5)
+        aws iam list-open-id-connect-providers | grep $oidc_id | cut -d "/" -f4
+
+        eksctl utils associate-iam-oidc-provider --cluster $CLUSTER_ID --approve
+
+        eksctl create iamserviceaccount \
+          --name ebs-csi-controller-sa \
+          --namespace kube-system \
+          --cluster $CLUSTER \
+          --attach-policy-arn arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy \
+          --approve \
+          --override-existing-serviceaccounts \
+          --role-name AmazonEKS_EBS_CSI_DriverRole_$CLUSTER
+
+      env:
+      - name: CLUSTER_ID
+        value: "{{workflow.parameters.cluster_id}}"
+      volumeMounts:
+      - name: awsconfig
+        mountPath: "/aws"
+
+  - name: deleteIAMRole
+    activeDeadlineSeconds: 1800
+    container:
+      image: sktcloud/tks-aws:v1.0.0
+      command:
+      - /bin/bash
+      - -exc
+      - |
+        mkdir ~/.aws
+        cp /aws/* ~/.aws/
+
+	eksctl delete iamserviceaccount --cluster $CLUSTER_ID --name ebs-csi-controller-sa --namespace kube-system
+
+      env:
+      - name: CLUSTER_ID
+        value: "{{workflow.parameters.cluster_id}}"
+      volumeMounts:
+      - name: awsconfig
+        mountPath: "/aws"
diff --git a/tks-cluster/create-aws-conf-secret.yaml b/tks-cluster/create-aws-conf-secret.yaml
@@ -11,11 +11,13 @@ spec:
       value: "aws_access_key_id"
     - name: aws_secret_access_key
       value: "aws_secret_access_key"
+    - name: aws_account_id
+      value: "aws_account_id"
   templates:
   - name: createTokenSecret
     activeDeadlineSeconds: 120
     container:
-      name: 'createClusterSite'
+      name: 'createSecret'
       image: k8s.gcr.io/hyperkube:v1.18.8
       imagePullPolicy: IfNotPresent
       command:
@@ -25,19 +27,21 @@ spec:
         kubectl delete secret -n argo awsconfig-secret || true
 
         echo "[default]
-        aws_access_key_id = $aws_access_key_id
-        aws_secret_access_key = $aws_secret_access_key" > /tmp/credentials
+        aws_access_key_id = $AWS_ACCESS_KEY_ID
+        aws_secret_access_key = $AWS_SECRET_ACCESS_KEY" > /tmp/credentials
 
         echo "[default]
         region = ap-northeast-2
         output = text" > /tmp/config
         
         kubectl create -n argo secret generic awsconfig-secret --from-file=config=/tmp/config --from-file=credentials=/tmp/credentials
+
+        kubectl delete secret -n argo aws-account-id || true
+        kubectl create secret generic aws-account-id --from-literal=AWS_ACCOUNT_ID=${AWS_ACCOUNT_ID}
       env:
-      - name: aws_access_key_id
+      - name: AWS_ACCESS_KEY_ID
         value: "{{workflow.parameters.aws_access_key_id}}"
-      - name: aws_secret_access_key
+      - name: AWS_SECRET_ACCESS_KEY
         value: "{{workflow.parameters.aws_secret_access_key}}"
-
-
-
+      - name: AWS_ACCOUNT_ID
+        value: "{{workflow.parameters.aws_account_id}}"
diff --git a/tks-cluster/create-usercluster-wftpl.yaml b/tks-cluster/create-usercluster-wftpl.yaml
@@ -103,6 +103,15 @@ spec:
             - name: infra_provider
               value: "{{steps.tks-create-cluster-repo.outputs.parameters.infra_provider}}"
 
+    - - name: create-aws-ebs-csi-iam
+        templateRef:
+          name: aws-ebs-csi-iam
+          template: createIAMRole
+        when: >-
+            ( {{steps.tks-create-cluster-repo.outputs.parameters.infra_provider}} == aws &&
+              {{steps.tks-create-cluster-repo.outputs.parameters.managed_cluster}} == true
+            )
+
     - - name: install-cluster-autoscaler-rbac
         templateRef:
           name: create-application
@@ -177,7 +186,10 @@ spec:
                   "target_cluster": ""
                 }
               ]
-        when: "{{steps.tks-create-cluster-repo.outputs.parameters.infra_provider}} == aws"
+        when: >-
+            ( {{steps.tks-create-cluster-repo.outputs.parameters.infra_provider}} == aws &&
+              {{steps.tks-create-cluster-repo.outputs.parameters.managed_cluster}} == false
+            )
 
     - - name: create-internal-communication
         templateRef:
@@ -221,13 +233,27 @@ spec:
           cp /kube/value kubeconfig_adm
           export KUBECONFIG=kubeconfig_adm
 
-          kubectl wait --for=condition=Available --timeout=600s kcp -n $CLUSTER_ID $CLUSTER_ID-control-plane
-
-          KUBECONFIG_WORKLOAD=$(kubectl get secret -n $CLUSTER_ID $CLUSTER_ID-kubeconfig -o jsonpath="{.data.value}" | base64 -d)
-          cat <<< "$KUBECONFIG_WORKLOAD" > kubeconfig_workload
-
           case $INFRA_PROVIDER in
             aws)
+              # check whether this workload cluster is managed or not
+              kcp_count=$(kubectl get kcp -n $CLUSTER_ID $CLUSTER_ID | wc -l)
+              awsmcp_count=$(kubectl get awsmcp -n $CLUSTER_ID $CLUSTER_ID | wc -l)
+
+              if [ $kcp_count = 1 ]; then
+                kubectl wait --for=condition=Available --timeout=600s kcp -n $CLUSTER_ID $CLUSTER_ID
+
+                KUBECONFIG_WORKLOAD=$(kubectl get secret -n $CLUSTER_ID $CLUSTER_ID-kubeconfig -o jsonpath="{.data.value}" | base64 -d)
+              elif [ $awsmcp_count = 1]; then
+                kubectl wait --for=condition=Available --timeout=600s awsmcp -n $CLUSTER_ID $CLUSTER_ID
+
+                KUBECONFIG_WORKLOAD=$(kubectl get secret -n $CLUSTER_ID $CLUSTER_ID-user-kubeconfig -o jsonpath="{.data.value}" | base64 -d)
+              else
+                echo "Wrong AWS Cluster type!"
+                exit 1
+              fi
+
+              cat <<< "$KUBECONFIG_WORKLOAD" > kubeconfig_workload
+
               echo "Wait for machinepool $CLUSTER_ID-mp-$TKS_NODE_NAME generated"
               while [ $(kubectl get machinepool -n $CLUSTER_ID $CLUSTER_ID-mp-$TKS_NODE_NAME --ignore-not-found | wc -l) == 0 ]
               do
@@ -249,7 +275,10 @@ spec:
               ;;
 
             byoh)
-              echo "BYOH"
+              kubectl wait --for=condition=Available --timeout=600s kcp -n $CLUSTER_ID $CLUSTER_ID
+
+              KUBECONFIG_WORKLOAD=$(kubectl get secret -n $CLUSTER_ID $CLUSTER_ID-kubeconfig -o jsonpath="{.data.value}" | base64 -d)
+              cat <<< "$KUBECONFIG_WORKLOAD" > kubeconfig_workload
               ;;
 
             *)

diff --git a/tks-cluster/remove-usercluster-wftpl.yaml b/tks-cluster/remove-usercluster-wftpl.yaml
@@ -97,6 +97,15 @@ spec:
           template: DeleteInternalCon
         when: "{{steps.findInfraProvider.outputs.parameters.infra_provider}} == aws"
 
+    - - name: delete-aws-ebs-csi-iam
+        templateRef:
+          name: aws-ebs-csi-iam
+          template: deleteIAMRole
+        when: >-
+            ( {{steps.tks-create-cluster-repo.outputs.parameters.infra_provider}} == aws &&
+              {{steps.tks-create-cluster-repo.outputs.parameters.managed_cluster}} == true
+            )
+
     - - name: deleteCsiDriverApp
         templateRef:
           name: delete-apps
@@ -105,14 +114,18 @@ spec:
           parameters:
             - name: app_name
               value: "{{workflow.parameters.app_prefix}}-aws-ebs-csi-driver"
-        when: "{{steps.findInfraProvider.outputs.parameters.infra_provider}} == aws"
+        when: >-
+            ( {{steps.tks-create-cluster-repo.outputs.parameters.infra_provider}} == aws &&
+              {{steps.tks-create-cluster-repo.outputs.parameters.managed_cluster}} == false
+            )
 
     - - name: deleteCalicoController
         template: deleteCalicoController
         arguments:
           parameters:
             - name: target_namespace
               value: "kube-system"
+        when: "{{steps.tks-create-cluster-repo.outputs.parameters.managed_cluster}} == false"
 
     - - name: deleteAddonsApp
         templateRef: