From a92854eada515f9e02ade21858964b95abd358a7 Mon Sep 17 00:00:00 2001 From: Jugwan Eom Date: Tue, 14 Feb 2023 08:53:26 +0000 Subject: [PATCH 1/5] Use an AWS account ID to create a cluster --- git-repo/create-cluster-repo.yaml | 3 +++ tks-cluster/create-aws-conf-secret.yaml | 20 ++++++++++++-------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/git-repo/create-cluster-repo.yaml b/git-repo/create-cluster-repo.yaml index 81de1dd3..ba7df7ae 100644 --- a/git-repo/create-cluster-repo.yaml +++ b/git-repo/create-cluster-repo.yaml @@ -65,6 +65,7 @@ spec: sed -i "s/clusterName:\ cluster.local/clusterName:\ $CLUSTER_ID/g" $CLUSTER_ID/$CLUSTER_ID/tks-cluster/site-values.yaml sed -i "s/sshKeyName:\ CHANGEME/sshKeyName: $val_ssh_key/g" $CLUSTER_ID/$CLUSTER_ID/tks-cluster/site-values.yaml sed -i "s/clusterRegion:\ CHANGEME/clusterRegion: $val_region/g" $CLUSTER_ID/$CLUSTER_ID/tks-cluster/site-values.yaml + sed -i "s/awsAccountID:\ CHANGEME/awsAccountID: \"$AWS_ACCOUNT_ID\"/g" $CLUSTER_ID/$CLUSTER_ID/tks-cluster/site-values.yaml sed -i "s/mdNumOfAz:\ CHANGEME/mdNumOfAz: $val_num_of_az/g" $CLUSTER_ID/$CLUSTER_ID/tks-cluster/site-values.yaml sed -i "s/mdMinSizePerAz:\ CHANGEME/mdMinSizePerAz: $val_min_size/g" $CLUSTER_ID/$CLUSTER_ID/tks-cluster/site-values.yaml sed -i "s/mdMaxSizePerAz:\ CHANGEME/mdMaxSizePerAz: $val_max_size/g" $CLUSTER_ID/$CLUSTER_ID/tks-cluster/site-values.yaml @@ -121,6 +122,8 @@ spec: envFrom: - secretRef: name: "git-svc-token" + - secretRef: + name: "aws-account-id" env: - name: CONTRACT_ID value: "{{workflow.parameters.contract_id}}" diff --git a/tks-cluster/create-aws-conf-secret.yaml b/tks-cluster/create-aws-conf-secret.yaml index 2340e55c..861d9e3e 100644 --- a/tks-cluster/create-aws-conf-secret.yaml +++ b/tks-cluster/create-aws-conf-secret.yaml @@ -11,11 +11,13 @@ spec: value: "aws_access_key_id" - name: aws_secret_access_key value: "aws_secret_access_key" + - name: aws_account_id + value: "aws_account_id" templates: - name: createTokenSecret activeDeadlineSeconds: 120 container: - name: 'createClusterSite' + name: 'createSecret' image: k8s.gcr.io/hyperkube:v1.18.8 imagePullPolicy: IfNotPresent command: @@ -25,19 +27,21 @@ spec: kubectl delete secret -n argo awsconfig-secret || true echo "[default] - aws_access_key_id = $aws_access_key_id - aws_secret_access_key = $aws_secret_access_key" > /tmp/credentials + aws_access_key_id = $AWS_ACCESS_KEY_ID + aws_secret_access_key = $AWS_SECRET_ACCESS_KEY" > /tmp/credentials echo "[default] region = ap-northeast-2 output = text" > /tmp/config kubectl create -n argo secret generic awsconfig-secret --from-file=config=/tmp/config --from-file=credentials=/tmp/credentials + + kubectl delete secret -n argo aws-account-id || true + kubectl create secret generic aws-account-id --from-literal=AWS_ACCOUNT_ID=${AWS_ACCOUNT_ID} env: - - name: aws_access_key_id + - name: AWS_ACCESS_KEY_ID value: "{{workflow.parameters.aws_access_key_id}}" - - name: aws_secret_access_key + - name: AWS_SECRET_ACCESS_KEY value: "{{workflow.parameters.aws_secret_access_key}}" - - - + - name: AWS_ACCOUNT_ID + value: "{{workflow.parameters.aws_account_id}}" From cef042e6ca8f9aedbfe5a06bede972afd17bbbbd Mon Sep 17 00:00:00 2001 From: Jugwan Eom Date: Tue, 14 Feb 2023 08:58:57 +0000 Subject: [PATCH 2/5] add a routine to check whether EKS enabled --- git-repo/create-cluster-repo.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/git-repo/create-cluster-repo.yaml b/git-repo/create-cluster-repo.yaml index ba7df7ae..54c2a5d7 100644 --- a/git-repo/create-cluster-repo.yaml +++ b/git-repo/create-cluster-repo.yaml @@ -43,6 +43,11 @@ spec: INFRA_PROVIDER="$(cat ${CONTRACT_ID}/$TEMPLATE_NAME/tks-cluster/kustomization.yaml | grep /infra/ | awk -F \/ '{print $3}')" echo ${INFRA_PROVIDER} | tee /mnt/out/infra_provider.txt + if [ "$INFRA_PROVIDER" = "aws" ]; then + eks_enabled=$(cat ${CONTRACT_ID}/$TEMPLATE_NAME/tks-cluster/site-values.yaml | grep eksEnabled | awk '{print $2}') + echo $eks_enabled | tee /mnt/out/managed_cluster.txt + fi + cp -r ${CONTRACT_ID}/${TEMPLATE_NAME} ${CLUSTER_ID}/${CLUSTER_ID} cp -r ${CONTRACT_ID}/_github ${CLUSTER_ID}/.github @@ -145,6 +150,10 @@ spec: valueFrom: default: "Something wrong" path: /mnt/out/infra_provider.txt + - name: managed_cluster + valueFrom: + default: "Something wrong" + path: /mnt/out/managed_cluster.txt - name: createRepoCredential activeDeadlineSeconds: 120 From ddb5091bcaa836110e7587e46fbfb818d5c7a199 Mon Sep 17 00:00:00 2001 From: Jugwan Eom Date: Tue, 14 Feb 2023 09:00:13 +0000 Subject: [PATCH 3/5] use aws-ebs-csi/calico app only when EKS is not enabled --- tks-cluster/create-usercluster-wftpl.yaml | 5 ++++- tks-cluster/remove-usercluster-wftpl.yaml | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/tks-cluster/create-usercluster-wftpl.yaml b/tks-cluster/create-usercluster-wftpl.yaml index c78d5a1e..5ecf3b11 100644 --- a/tks-cluster/create-usercluster-wftpl.yaml +++ b/tks-cluster/create-usercluster-wftpl.yaml @@ -177,7 +177,10 @@ spec: "target_cluster": "" } ] - when: "{{steps.tks-create-cluster-repo.outputs.parameters.infra_provider}} == aws" + when: >- + ( {{steps.tks-create-cluster-repo.outputs.parameters.infra_provider}} == aws && + {{steps.tks-create-cluster-repo.outputs.parameters.managed_cluster}} == false + ) - - name: create-internal-communication templateRef: diff --git a/tks-cluster/remove-usercluster-wftpl.yaml b/tks-cluster/remove-usercluster-wftpl.yaml index 00901ab0..c5547534 100644 --- a/tks-cluster/remove-usercluster-wftpl.yaml +++ b/tks-cluster/remove-usercluster-wftpl.yaml @@ -105,7 +105,10 @@ spec: parameters: - name: app_name value: "{{workflow.parameters.app_prefix}}-aws-ebs-csi-driver" - when: "{{steps.findInfraProvider.outputs.parameters.infra_provider}} == aws" + when: >- + ( {{steps.tks-create-cluster-repo.outputs.parameters.infra_provider}} == aws && + {{steps.tks-create-cluster-repo.outputs.parameters.managed_cluster}} == false + ) - - name: deleteCalicoController template: deleteCalicoController @@ -113,6 +116,7 @@ spec: parameters: - name: target_namespace value: "kube-system" + when: "{{steps.tks-create-cluster-repo.outputs.parameters.managed_cluster}} == false" - - name: deleteAddonsApp templateRef: From 7387927622f0259bc6913211cb3877a2c974dcb3 Mon Sep 17 00:00:00 2001 From: Jugwan Eom Date: Tue, 14 Feb 2023 11:43:50 +0000 Subject: [PATCH 4/5] add aws-ebs-csi-driver iam role jobs --- dockerfiles/Dockerfile.tks_aws | 8 +++ tks-cluster/aws-ebs-csi-iam-yaml | 69 +++++++++++++++++++++++ tks-cluster/create-usercluster-wftpl.yaml | 9 +++ tks-cluster/remove-usercluster-wftpl.yaml | 9 +++ 4 files changed, 95 insertions(+) create mode 100644 dockerfiles/Dockerfile.tks_aws create mode 100644 tks-cluster/aws-ebs-csi-iam-yaml diff --git a/dockerfiles/Dockerfile.tks_aws b/dockerfiles/Dockerfile.tks_aws new file mode 100644 index 00000000..773e84ee --- /dev/null +++ b/dockerfiles/Dockerfile.tks_aws @@ -0,0 +1,8 @@ +FROM weaveworks/eksctl AS eksctl +#FROM amazon/aws-cli AS awscli + +#make a docker image with this CLI: docker build -t sktcloud/tks-aws:v1.0.0 -f Dockerfile.tks_aws . +FROM alpine +COPY --from=eksctl /usr/local/bin/eksctl /usr/bin/eksctl +RUN apk update +RUN apk add aws-cli diff --git a/tks-cluster/aws-ebs-csi-iam-yaml b/tks-cluster/aws-ebs-csi-iam-yaml new file mode 100644 index 00000000..ae1af3df --- /dev/null +++ b/tks-cluster/aws-ebs-csi-iam-yaml @@ -0,0 +1,69 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: aws-ebs-csi-iam + namespace: argo +spec: + entrypoint: createIAMRole + arguments: + parameters: + - name: cluster_id + value: "Cc81dd656" + + volumes: + - name: awsconfig + secret: + secretName: awsconfig-secret + + templates: + - name: createIAMRole + activeDeadlineSeconds: 1800 + container: + image: sktcloud/tks-aws:v1.0.0 + command: + - /bin/bash + - -exc + - | + mkdir ~/.aws + cp /aws/* ~/.aws/ + + oidc_id=$(aws eks describe-cluster --name $CLUSTER --query "cluster.identity.oidc.issuer" --output text | cut -d '/' -f 5) + aws iam list-open-id-connect-providers | grep $oidc_id | cut -d "/" -f4 + + eksctl utils associate-iam-oidc-provider --cluster $CLUSTER_ID --approve + + eksctl create iamserviceaccount \ + --name ebs-csi-controller-sa \ + --namespace kube-system \ + --cluster $CLUSTER \ + --attach-policy-arn arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy \ + --approve \ + --override-existing-serviceaccounts \ + --role-name AmazonEKS_EBS_CSI_DriverRole_$CLUSTER + + env: + - name: CLUSTER_ID + value: "{{workflow.parameters.cluster_id}}" + volumeMounts: + - name: awsconfig + mountPath: "/aws" + + - name: deleteIAMRole + activeDeadlineSeconds: 1800 + container: + image: sktcloud/tks-aws:v1.0.0 + command: + - /bin/bash + - -exc + - | + mkdir ~/.aws + cp /aws/* ~/.aws/ + + eksctl delete iamserviceaccount --cluster $CLUSTER_ID --name ebs-csi-controller-sa --namespace kube-system + + env: + - name: CLUSTER_ID + value: "{{workflow.parameters.cluster_id}}" + volumeMounts: + - name: awsconfig + mountPath: "/aws" diff --git a/tks-cluster/create-usercluster-wftpl.yaml b/tks-cluster/create-usercluster-wftpl.yaml index 5ecf3b11..1e60abd1 100644 --- a/tks-cluster/create-usercluster-wftpl.yaml +++ b/tks-cluster/create-usercluster-wftpl.yaml @@ -103,6 +103,15 @@ spec: - name: infra_provider value: "{{steps.tks-create-cluster-repo.outputs.parameters.infra_provider}}" + - - name: create-aws-ebs-csi-iam + templateRef: + name: aws-ebs-csi-iam + template: createIAMRole + when: >- + ( {{steps.tks-create-cluster-repo.outputs.parameters.infra_provider}} == aws && + {{steps.tks-create-cluster-repo.outputs.parameters.managed_cluster}} == true + ) + - - name: install-cluster-autoscaler-rbac templateRef: name: create-application diff --git a/tks-cluster/remove-usercluster-wftpl.yaml b/tks-cluster/remove-usercluster-wftpl.yaml index c5547534..f8c93630 100644 --- a/tks-cluster/remove-usercluster-wftpl.yaml +++ b/tks-cluster/remove-usercluster-wftpl.yaml @@ -97,6 +97,15 @@ spec: template: DeleteInternalCon when: "{{steps.findInfraProvider.outputs.parameters.infra_provider}} == aws" + - - name: delete-aws-ebs-csi-iam + templateRef: + name: aws-ebs-csi-iam + template: deleteIAMRole + when: >- + ( {{steps.tks-create-cluster-repo.outputs.parameters.infra_provider}} == aws && + {{steps.tks-create-cluster-repo.outputs.parameters.managed_cluster}} == true + ) + - - name: deleteCsiDriverApp templateRef: name: delete-apps From d551966880a6955b6ca31ffbb1b7980262de727d Mon Sep 17 00:00:00 2001 From: Jugwan Eom Date: Tue, 14 Feb 2023 12:01:31 +0000 Subject: [PATCH 5/5] use a different secret for workload kubeconfig Ref: https://cluster-api-aws.sigs.k8s.io/topics/eks/creating-a-cluster.html#user-kubeconfig --- tks-cluster/create-usercluster-wftpl.yaml | 29 ++++++++++++++++++----- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/tks-cluster/create-usercluster-wftpl.yaml b/tks-cluster/create-usercluster-wftpl.yaml index 1e60abd1..ba9fdba5 100644 --- a/tks-cluster/create-usercluster-wftpl.yaml +++ b/tks-cluster/create-usercluster-wftpl.yaml @@ -233,13 +233,27 @@ spec: cp /kube/value kubeconfig_adm export KUBECONFIG=kubeconfig_adm - kubectl wait --for=condition=Available --timeout=600s kcp -n $CLUSTER_ID $CLUSTER_ID-control-plane - - KUBECONFIG_WORKLOAD=$(kubectl get secret -n $CLUSTER_ID $CLUSTER_ID-kubeconfig -o jsonpath="{.data.value}" | base64 -d) - cat <<< "$KUBECONFIG_WORKLOAD" > kubeconfig_workload - case $INFRA_PROVIDER in aws) + # check whether this workload cluster is managed or not + kcp_count=$(kubectl get kcp -n $CLUSTER_ID $CLUSTER_ID | wc -l) + awsmcp_count=$(kubectl get awsmcp -n $CLUSTER_ID $CLUSTER_ID | wc -l) + + if [ $kcp_count = 1 ]; then + kubectl wait --for=condition=Available --timeout=600s kcp -n $CLUSTER_ID $CLUSTER_ID + + KUBECONFIG_WORKLOAD=$(kubectl get secret -n $CLUSTER_ID $CLUSTER_ID-kubeconfig -o jsonpath="{.data.value}" | base64 -d) + elif [ $awsmcp_count = 1]; then + kubectl wait --for=condition=Available --timeout=600s awsmcp -n $CLUSTER_ID $CLUSTER_ID + + KUBECONFIG_WORKLOAD=$(kubectl get secret -n $CLUSTER_ID $CLUSTER_ID-user-kubeconfig -o jsonpath="{.data.value}" | base64 -d) + else + echo "Wrong AWS Cluster type!" + exit 1 + fi + + cat <<< "$KUBECONFIG_WORKLOAD" > kubeconfig_workload + echo "Wait for machinepool $CLUSTER_ID-mp-$TKS_NODE_NAME generated" while [ $(kubectl get machinepool -n $CLUSTER_ID $CLUSTER_ID-mp-$TKS_NODE_NAME --ignore-not-found | wc -l) == 0 ] do @@ -261,7 +275,10 @@ spec: ;; byoh) - echo "BYOH" + kubectl wait --for=condition=Available --timeout=600s kcp -n $CLUSTER_ID $CLUSTER_ID + + KUBECONFIG_WORKLOAD=$(kubectl get secret -n $CLUSTER_ID $CLUSTER_ID-kubeconfig -o jsonpath="{.data.value}" | base64 -d) + cat <<< "$KUBECONFIG_WORKLOAD" > kubeconfig_workload ;; *)