create-usercluster-wftpl.yaml

apiVersion: argoproj.io/v1alpha1
kind: WorkflowTemplate
metadata:
  name: create-tks-usercluster
  namespace: argo
spec:
  entrypoint: deploy
  arguments:
    parameters:
      - name: contract_id
        value: "P0010010a"
      - name: cluster_id
        value: "C011b88fa"
      - name: site_name
        value: "{{ workflow.parameters.cluster_id }}"
      - name: cloud_account_id # will be not NULL if the cluster is multitenancy
        value: "NULL"
      - name: template_name
        value: "aws-reference"
      - name: git_account
        value: "tks-management"
      - name: manifest_repo_url
        value: ""
      - name: revision
        value: "main"
      - name: app_prefix
        value: "{{workflow.parameters.cluster_id}}"
      - name: tks_api_url
        value: "http://tks-api.tks.svc:9110"
      - name: base_repo_branch
        value: "main"
      - name: keycloak_url
        value: "https://keycloak.yourdomain.org/auth"
      - name: policy_ids
        value: ""

  volumes:
    - name: kubeconfig-adm
      secret:
        secretName: tks-admin-kubeconfig-secret
    - name: awsconfig
      secret:
        secretName: awsconfig-secret
    - name: artifacts
      configMap:
        name: aws-artifacts
        defaultMode: 0555
    - name: tks-proto-vol
      configMap:
        name: tks-proto
    - name: awsconfig
      secret:
        secretName: awsconfig-secret

  templates:
    - name: deploy
      steps:
        - - name: tks-get-cluster-info
            templateRef:
              name: tks-get-cluster
              template: getTksCluster

        - - name: tks-create-cluster-repo
            templateRef:
              name: tks-create-cluster-repo
              template: main
            arguments:
              parameters:
                - name: cluster_info
                  value: "{{steps.tks-get-cluster-info.outputs.parameters.cluster_info}}"

        - - name: render-manifests
            templateRef:
              name: event-gitea-render-manifests
              template: main
            arguments:
              parameters:
                - name: decapod_site_repo
                  value: "{{ workflow.parameters.git_account }}/{{ workflow.parameters.cluster_id }}"
                - name: base_repo_branch
                  value: "{{ workflow.parameters.base_repo_branch }}"

        - - name: k8s-by-aws
            templateRef:
              name: create-application
              template: installApps
            arguments:
              parameters:
                - name: list
                  value: |
                    [
                      {
                        "app_group": "tks-cluster",
                        "path": "cluster-api-aws",
                        "namespace": "argo",
                        "target_cluster": "tks-admin"
                      }
                    ]
            when: "{{steps.tks-create-cluster-repo.outputs.parameters.infra_provider}} == aws"

        - - name: k8s-by-byoh
            templateRef:
              name: create-application
              template: installApps
            arguments:
              parameters:
                - name: list
                  value: |
                    [
                      {
                        "app_group": "tks-cluster",
                        "path": "cluster-api-byoh",
                        "namespace": "argo",
                        "target_cluster": "tks-admin"
                      }
                    ]
            when: "{{steps.tks-create-cluster-repo.outputs.parameters.infra_provider}} == byoh"

        - - name: tks-create-config-secret
            template: create-endpoint-secret
            arguments:
              parameters:
                - name: cluster_domains
                  value: "{{steps.tks-get-cluster-info.outputs.parameters.cluster_domains}}"
                - name: cluster_id
                  value: "{{ workflow.parameters.cluster_id }}"
            when: "{{steps.tks-create-cluster-repo.outputs.parameters.infra_provider}} == byoh"

        - - name: init-cluster-for-tks
            template: init-cluster-for-tks
            arguments:
              parameters:
                - name: cluster_id
                  value: "{{ workflow.parameters.cluster_id }}"
                - name: infra_provider
                  value: "{{steps.tks-create-cluster-repo.outputs.parameters.infra_provider}}"
                - name: cloud_account_id
                  value: "{{ workflow.parameters.cloud_account_id }}"
                - name: keycloak_url
                  value: "{{ workflow.parameters.keycloak_url }}"
                - name: contract_id
                  value: "{{ workflow.parameters.contract_id }}"

        - - name: create-eks-keycloak-oidc-provider
            templateRef:
              name: aws-eks-keycloak-oidc-provider
              template: createProvider
            arguments:
              parameters:
                - name: contract_id
                  value: "{{ workflow.parameters.contract_id }}"
                - name: cluster_id
                  value: "{{ workflow.parameters.cluster_id }}"
                - name: keycloak_url
                  value: "{{ workflow.parameters.keycloak_url }}"
                - name: cloud_account_id
                  value: "{{ workflow.parameters.cloud_account_id }}"
            when: >-
              {{steps.tks-create-cluster-repo.outputs.parameters.infra_provider}} == aws &&
                {{steps.tks-create-cluster-repo.outputs.parameters.managed_cluster}} == true

        - - name: create-aws-cluster-autoscaler-iam
            templateRef:
              name: aws-cluster-autoscaler-iam
              template: createIAMRole
            arguments:
              parameters:
                - name: cloud_account_id
                  value: "{{ workflow.parameters.cloud_account_id }}"
            when: >-
              {{steps.tks-create-cluster-repo.outputs.parameters.infra_provider}} == aws &&
                {{steps.tks-create-cluster-repo.outputs.parameters.managed_cluster}} == true

        - - name: install-cluster-autoscaler-rbac
            templateRef:
              name: create-application
              template: installApps
            arguments:
              parameters:
                - name: list
                  value: |
                    [
                      {
                        "app_group": "tks-cluster",
                        "path": "cluster-autoscaler-rbac",
                        "namespace": "argo",
                        "target_cluster": "tks-admin"
                      }
                    ]
            when: >-
              {{steps.tks-create-cluster-repo.outputs.parameters.infra_provider}} != byoh &&
                {{steps.tks-create-cluster-repo.outputs.parameters.managed_cluster}} == false

        - - name: prepare-cluster-autoscaler
            template: prepare-cluster-autoscaler
            arguments:
              parameters:
                - name: cluster_id
                  value: "{{ workflow.parameters.cluster_id }}"
                - name: infra_provider
                  value: "{{steps.tks-create-cluster-repo.outputs.parameters.infra_provider}}"
            when: >-
              {{steps.tks-create-cluster-repo.outputs.parameters.infra_provider}} != byoh &&
                {{steps.tks-create-cluster-repo.outputs.parameters.managed_cluster}} == false

        - - name: install-volumesnapshot-crds
            template: install-volumesnapshot-crds

        - - name: install-addons-cni-crds
            templateRef:
              name: create-application
              template: installApps
            arguments:
              parameters:
                - name: list
                  value: |
                    [
                      {
                        "app_group": "tks-cluster",
                        "path": "tigera-operator-crds",
                        "namespace": "taco-system",
                        "target_cluster": ""
                      }
                    ]
            when: "{{steps.tks-create-cluster-repo.outputs.parameters.managed_cluster}} == false"

        - - name: install-addons-cni
            templateRef:
              name: create-application
              template: installApps
            arguments:
              parameters:
                - name: list
                  value: |
                    [
                      {
                        "app_group": "tks-cluster",
                        "path": "tigera-operator",
                        "namespace": "taco-system",
                        "target_cluster": ""
                      }
                    ]
            when: "{{steps.tks-create-cluster-repo.outputs.parameters.managed_cluster}} == false"

        - - name: suspend
            template: suspend
            when: "{{steps.tks-create-cluster-repo.outputs.parameters.infra_provider}} == byoh"

        - - name: install-cluster-autoscaler
            templateRef:
              name: create-application
              template: installApps
            arguments:
              parameters:
                - name: list
                  value: |
                    [
                      {
                        "app_group": "tks-cluster",
                        "path": "cluster-autoscaler",
                        "namespace": "kube-system",
                        "target_cluster": ""
                      }
                    ]
            when: "{{steps.tks-create-cluster-repo.outputs.parameters.infra_provider}} != byoh"

        - - name: install-addons-common
            templateRef:
              name: create-application
              template: installApps
            arguments:
              parameters:
                - name: list
                  value: |
                    [
                      {
                        "app_group": "tks-cluster",
                        "path": "ingress-nginx",
                        "namespace": "taco-system",
                        "target_cluster": ""
                      },
                      {
                        "app_group": "tks-cluster",
                        "path": "argo-rollouts",
                        "namespace": "taco-system",
                        "target_cluster": ""
                      }
                    ]

        - - name: install-addons-aws
            templateRef:
              name: create-application
              template: installApps
            arguments:
              parameters:
                - name: list
                  value: |
                    [
                      {
                        "app_group": "tks-cluster",
                        "path": "s3-chart-crds",
                        "namespace": "taco-system",
                        "target_cluster": ""
                      },
                      {
                        "app_group": "tks-cluster",
                        "path": "metrics-server",
                        "namespace": "kube-system",
                        "target_cluster": ""
                      }, 
                      {
                        "app_group": "tks-cluster",
                        "path": "s3-chart",
                        "namespace": "taco-system",
                        "target_cluster": ""
                      }
                    ]
            when: >-
              {{steps.tks-create-cluster-repo.outputs.parameters.infra_provider}} == aws

        - - name: install-addons-aws-for-self-managed
            templateRef:
              name: create-application
              template: installApps
            arguments:
              parameters:
                - name: list
                  value: |
                    [
                      {
                        "app_group": "tks-cluster",
                        "path": "aws-ebs-csi-driver",
                        "namespace": "kube-system",
                        "target_cluster": ""
                      }
                    ]
            when: >-
              {{steps.tks-create-cluster-repo.outputs.parameters.infra_provider}} == aws &&
                {{steps.tks-create-cluster-repo.outputs.parameters.managed_cluster}} == false

        - - name: create-internal-communication
            templateRef:
              name: manage-internal-communication
              template: deploy
            when: >-
              {{steps.tks-create-cluster-repo.outputs.parameters.infra_provider}} == aws &&
                {{steps.tks-create-cluster-repo.outputs.parameters.managed_cluster}} == false

        - - name: install-addons-byoh
            templateRef:
              name: create-application
              template: installApps
            arguments:
              parameters:
                - name: list
                  value: |
                    [
                      {
                        "app_group": "tks-cluster",
                        "path": "local-path-provisioner",
                        "namespace": "taco-system",
                        "target_cluster": ""
                      },
                      {
                        "app_group": "tks-cluster",
                        "path": "metrics-server",
                        "namespace": "kube-system",
                        "target_cluster": ""
                      } 
                    ]
            when: "{{steps.tks-create-cluster-repo.outputs.parameters.infra_provider}} == byoh"

        - - name: create-default-rbac-resources
            template: k8s-rbac-setting

        - - name: install-policy-management
            templateRef:
              name: tks-policy
              template: deploy
            arguments:
              parameters:
                - name: cluster_id
                  value: "{{ workflow.parameters.cluster_id }}"
                - name: contract_id
                  value: "{{ workflow.parameters.contract_id }}"
                - name: policy_ids
                  value: "{{ workflow.parameters.policy_ids }}"
            # when: "{{steps.get-clusters-in-contract.outputs.parameters.primary_cluster}} != '' && {{workflow.parameters.cluster_id}} != {{steps.get-clusters-in-contract.outputs.parameters.primary_cluster}}"

    #######################
    # Template Definition #
    #######################
    - name: suspend
      suspend: {}

    - name: init-cluster-for-tks
      inputs:
        parameters:
          - name: cluster_id
          - name: infra_provider
          - name: cloud_account_id
          - name: keycloak_url
          - name: contract_id
      container:
        name: cluster-init
        image: harbor.taco-cat.xyz/tks/tks-cluster-init:v1.0.0
        command:
          - /bin/bash
          - "-exc"
          - |
            cp /kube/value kubeconfig_adm
            export KUBECONFIG=kubeconfig_adm

            ####### add tks info. on namespace ########
            kubectl label ns ${CLUSTER_ID} tks.io/organization=${CONTRACT_ID}
            if [ $(kubectl get ns -l tks.io/organization=${CONTRACT_ID} --ignore-not-found=true | grep -v NAME | awk '{print $1}' | wc -l ) -le 1 ]; then
              kubectl label ns ${CLUSTER_ID} tks.io/policy=${CLUSTER_ID}
            else
              POLICY_NS=$(kubectl get ns $(kubectl get ns -l tks.io/organization=${CONTRACT_ID} --ignore-not-found=true | grep -v NAME | awk '{print $1}' | head -n 1 )  --ignore-not-found=true -o jsonpath='{.metadata.labels.tks\.io\/policy}' )
              if [ -z "$POLICY_NS" ]; then
                kubectl label ns ${CLUSTER_ID} tks.io/policy=${CLUSTER_ID}
              else
                kubectl label ns ${CLUSTER_ID} tks.io/policy=${POLICY_NS}
              fi
            fi
            ###########################################

            mkdir ~/.aws
            cp /aws/* ~/.aws/

            case $INFRA_PROVIDER in
              aws)
                # check whether this workload cluster is managed or not
                kcp_count=$(kubectl get kcp -n $CLUSTER_ID $CLUSTER_ID | grep -v NAME | wc -l)
                awsmcp_count=$(kubectl get awsmcp -n $CLUSTER_ID $CLUSTER_ID | grep -v NAME | wc -l)

                if [ $kcp_count = 1 ]; then # Self-managed control plane cluster
                  kubectl wait --for=condition=Available --timeout=3600s kcp -n $CLUSTER_ID $CLUSTER_ID

                  KUBECONFIG_WORKLOAD=$(kubectl get secret -n $CLUSTER_ID $CLUSTER_ID-kubeconfig -o jsonpath="{.data.value}" | base64 -d)
                elif [ $awsmcp_count = 1 ]; then # EKS cluster
                  kubectl wait --for=condition=ready --timeout=3600s awsmcp -n $CLUSTER_ID $CLUSTER_ID
                  KUBECONFIG_WORKLOAD=$(kubectl get secret -n $CLUSTER_ID $CLUSTER_ID-user-kubeconfig -o jsonpath="{.data.value}" | base64 -d)
                  if [ "$CLOUD_ACCOUNT_ID" != "NULL" ]; then # multitenancy cluster
                    IDENTITY_ROLE_ARN=$(kubectl get awsri $CLOUD_ACCOUNT_ID-account-role -ojsonpath='{.spec.roleARN}')
                    cat <<< $KUBECONFIG_WORKLOAD | sed '/args/,+7d' > tmp_kubeconfig_workload
                    echo "      args:
                  - --region
                  - ap-northeast-2
                  - eks
                  - get-token
                  - --cluster-name
                  - ${CLUSTER_ID}
                  - --output
                  - json
                  - --role
                  - ${IDENTITY_ROLE_ARN}
                  command: aws" | envsubst >> tmp_kubeconfig_workload
                    KUBECONFIG_WORKLOAD=$(cat tmp_kubeconfig_workload)
                  fi
                else
                  echo "Wrong AWS Cluster type!"
                  exit 1
                fi

                cat <<< "$KUBECONFIG_WORKLOAD" > kubeconfig_workload

                echo "Wait for machinepool $CLUSTER_ID-mp-$TKS_NODE_NAME generated"
                while [ $(kubectl get machinepool -n $CLUSTER_ID $CLUSTER_ID-mp-$TKS_NODE_NAME --ignore-not-found | wc -l) == 0 ]
                do
                  echo "Wait for machinepools deployed (1s)"
                  sleep 1
                done

                desired_replicas=$(kubectl get machinepool -n $CLUSTER_ID $CLUSTER_ID-mp-$TKS_NODE_NAME -o=jsonpath='{.spec.replicas}' )
                while [ $(kubectl get machinepool -n $CLUSTER_ID $CLUSTER_ID-mp-$TKS_NODE_NAME -o=jsonpath='{.status.nodeRefs}' | jq '.[].uid' | wc -l) != $desired_replicas ]
                do
                  echo "Wait for instance is ready (1s)"
                  sleep 1
                done

                for node in $(kubectl get machinepool -n $CLUSTER_ID $CLUSTER_ID-mp-$TKS_NODE_NAME -o=jsonpath='{.status.nodeRefs}'| jq -r '.[].name')
                do
                  kubectl --kubeconfig=kubeconfig_workload label node $node taco-lma=enabled tks-ingressgateway=enabled tks-egressgateway=enabled tks-msa=enabled --overwrite
                done
                ;;

              byoh)
                kcp_count=$(kubectl get kcp -n $CLUSTER_ID $CLUSTER_ID | grep -v NAME | wc -l)
                kubectl wait --for=condition=Available --timeout=600s kcp -n $CLUSTER_ID $CLUSTER_ID

                KUBECONFIG_WORKLOAD=$(kubectl get secret -n $CLUSTER_ID $CLUSTER_ID-kubeconfig -o jsonpath="{.data.value}" | base64 -d)
                cat <<< "$KUBECONFIG_WORKLOAD" > kubeconfig_workload

                for machine in $(kubectl get machine -n $CLUSTER_ID -l cluster.x-k8s.io/deployment-name=$CLUSTER_ID-md-$TKS_NODE_NAME -oname)
                do
                  kubectl wait --for=condition=ready --timeout=3600s -n $CLUSTER_ID $machine
                done

                for node in $(kubectl get machine -n $CLUSTER_ID -l cluster.x-k8s.io/deployment-name=$CLUSTER_ID-md-$TKS_NODE_NAME -o=jsonpath='{.items[*].status.nodeRef.name}')
                do
                  kubectl --kubeconfig=kubeconfig_workload label node $node taco-lma=enabled tks-ingressgateway=enabled tks-egressgateway=enabled tks-msa=enabled --overwrite
                done
                ;;

              *)
                echo "Error: wrong infra provider"
                exit 1
                ;;
            esac

            export KUBECONFIG=kubeconfig_workload

            cat <<EOF > taco-system.yaml
            apiVersion: v1
            kind: Namespace
            metadata:
              labels:
                name: taco-system
              name: taco-system
            EOF
            kubectl apply -f taco-system.yaml

            argocd login --insecure --plaintext $ARGO_SERVER --username $ARGO_USERNAME --password $ARGO_PASSWORD
            CLUSTER_CONTEXT=$(kubectl config current-context)

            while [ $(kubectl get no | wc -l) == 0 ]
            do
                echo "Wait for cluster is ready (1s)"
                sleep 1
            done

            if [ $(argocd cluster list | grep \ $CLUSTER_ID\ | wc -l ) == 0 ]; then
                argocd cluster add $CLUSTER_CONTEXT --name $CLUSTER_ID --upsert
            else
                echo "Warning: $1 is already registered on argo-cd server. If unintended, it may occure woring operations."
            fi

            # Create a kubeconfig secret for TKS internal use from ArgoCD cluster secret and for TKS user
            export KUBECONFIG=kubeconfig_adm        
            if [ $kcp_count = 1 ]; then
              TKS_KUBECONFIG_WORKLOAD=$(kubectl get secret -n $CLUSTER_ID $CLUSTER_ID-kubeconfig -o jsonpath="{.data.value}" | base64 -d)
            elif [ $awsmcp_count = 1 ]; then
              CAPA_USER_KUBECONFIG_WORKLOAD=$(kubectl get secret -n $CLUSTER_ID $CLUSTER_ID-user-kubeconfig -o jsonpath="{.data.value}" | base64 -d)

              # tks-kubeconfig
              cat <<< $CAPA_USER_KUBECONFIG_WORKLOAD | sed '/exec:/,+15d' > tmp_tks_kubeconfig_workload
              API_SERVER=$(grep server tmp_tks_kubeconfig_workload | awk '{print tolower($2)}')
              ARGOCD_CLUSTER_SECRET=$(kubectl get secret -n argo | grep ${API_SERVER#*\/\/} | awk '{print $1}')
              CLIENT_TOKEN=$(kubectl get secret -n argo $ARGOCD_CLUSTER_SECRET -ojsonpath='{.data.config}' | base64 -d | jq -r .bearerToken)
              echo "    token: ${CLIENT_TOKEN}" >> tmp_tks_kubeconfig_workload
              TKS_KUBECONFIG_WORKLOAD=$(cat tmp_tks_kubeconfig_workload)

              cat <<EOF > sc-taco-storage.yaml
            apiVersion: storage.k8s.io/v1
            kind: StorageClass
            metadata:
              name: taco-storage
            parameters:
              fsType: ext4
              type: gp2
            provisioner: kubernetes.io/aws-ebs
            reclaimPolicy: Delete
            volumeBindingMode: WaitForFirstConsumer
            EOF
              kubectl --kubeconfig kubeconfig_workload apply -f sc-taco-storage.yaml
            else
              echo "Wrong Cluster type!"
              exit 1
            fi
            
            # generate kubeconfig for user
            ISSUER_URL=$KEYCLOAK_URL/realms/$CONTRACT_ID
            CLIENT_ID=$CLUSTER_ID-k8s-api
            OIDC_USER_NAME="oidc-user"
            EXISTING_USER_NAME=$CLUSTER_ID-admin
            
            kubectl get secret -n $CLUSTER_ID $CLUSTER_ID-kubeconfig -o jsonpath="{.data.value}" | base64 -d > tmp_user_kubeconfig
            kubectl --kubeconfig=tmp_user_kubeconfig config unset users.$EXISTING_USER_NAME
            kubectl --kubeconfig=tmp_user_kubeconfig config set-credentials $OIDC_USER_NAME \
              --exec-api-version=client.authentication.k8s.io/v1beta1 \
              --exec-command=kubectl \
              --exec-arg=oidc-login \
              --exec-arg=get-token \
              --exec-arg=--oidc-issuer-url=$ISSUER_URL \
              --exec-arg=--oidc-client-id=$CLIENT_ID \
              --exec-arg=--grant-type=password 
            
            CONTEXT_NAME=$(kubectl --kubeconfig=tmp_user_kubeconfig config current-context)
            kubectl --kubeconfig=tmp_user_kubeconfig config set-context $CONTEXT_NAME --user $OIDC_USER_NAME
            TKS_USER_KUBECONFIG_WORKLOAD=$(cat tmp_user_kubeconfig)
            
            cat <<< $TKS_KUBECONFIG_WORKLOAD > tks_kubeconfig_workload
            kubectl create secret generic -n $CLUSTER_ID $CLUSTER_ID-tks-kubeconfig --from-file=value=tks_kubeconfig_workload
            cat <<< $TKS_USER_KUBECONFIG_WORKLOAD > tks_user_kubeconfig_workload
            kubectl create secret generic -n $CLUSTER_ID $CLUSTER_ID-tks-user-kubeconfig --from-file=value=tks_user_kubeconfig_workload

        volumeMounts:
          - name: kubeconfig-adm
            mountPath: "/kube"
          - name: awsconfig
            mountPath: "/aws"
        envFrom:
          - secretRef:
              name: "decapod-argocd-config"
        env:
          - name: CLUSTER_ID
            value: "{{ inputs.parameters.cluster_id }}"
          - name: INFRA_PROVIDER
            value: "{{ inputs.parameters.infra_provider }}"
          - name: CLOUD_ACCOUNT_ID
            value: "{{ inputs.parameters.cloud_account_id }}"
          - name: TKS_NODE_NAME
            value: "taco"
          - name: KEYCLOAK_URL
            value: "{{ inputs.parameters.keycloak_url }}"
          - name: CONTRACT_ID
            value: "{{ inputs.parameters.contract_id }}"

    - name: prepare-cluster-autoscaler
      inputs:
        parameters:
          - name: cluster_id
          - name: infra_provider
      container:
        name: prepare-cluster-autoscaler
        image: harbor.taco-cat.xyz/tks/kubectl-shell:latest-v1.21.1-amd64
        command:
          - /bin/bash
          - "-exc"
          - |
            cp /kube/value kubeconfig_adm
            export KUBECONFIG=kubeconfig_adm

            cp /kube/value kubeconfig

            CLUSTER=$(kubectl get cl -ojsonpath='{.items[0].metadata.name}' -n default)

            create_cluster_autoscaler_kubeconfig() {
              ADMIN_CONTEXT=$1
              ADMIN_USER=$2

              TOKEN=$(kubectl get secret -n {{workflow.parameters.cluster_id}} cluster-autoscaler-token -ojsonpath={.data.token} | base64 -d)
              kubectl --kubeconfig kubeconfig config set-credentials cluster-autoscaler --token=$TOKEN
              kubectl --kubeconfig kubeconfig config set-context cluster-autoscaler --cluster=$CLUSTER --user=cluster-autoscaler
              kubectl --kubeconfig kubeconfig config use-context cluster-autoscaler

              kubectl --kubeconfig kubeconfig config delete-context "$ADMIN_CONTEXT"
              kubectl --kubeconfig kubeconfig config delete-user "$ADMIN_USER"
            }

            CURRENT_CONTEXT=$(kubectl --kubeconfig kubeconfig config current-context)
            case $INFRA_PROVIDER in
              aws)
                # check whether admin cluster is managed or not
                kcp_count=$(kubectl get kcp -n default $CLUSTER | grep -v NAME | wc -l)
                awsmcp_count=$(kubectl get awsmcp -n default $CLUSTER | grep -v NAME | wc -l)

                if [ $kcp_count = 1 ]; then # Self-managed control plane cluster
                  create_cluster_autoscaler_kubeconfig "$CURRENT_CONTEXT" "$CLUSTER-admin"
                elif [ $awsmcp_count = 1 ]; then # EKS cluster
                  create_cluster_autoscaler_kubeconfig "$CURRENT_CONTEXT" "$CLUSTER-user"
                else
                  echo "Wrong AWS Cluster type!"
                  exit 1
                fi
                ;;

               *)
                echo "Error: wrong infra provider"
                exit 1
                ;;
            esac

            KUBECONFIG_WORKLOAD=$(kubectl get secret -n {{workflow.parameters.cluster_id}} {{workflow.parameters.cluster_id}}-tks-kubeconfig -o jsonpath="{.data.value}" | base64 -d)
            echo -e "kubeconfig_workload:\n$KUBECONFIG_WORKLOAD" | head -n 5
            cat <<< "$KUBECONFIG_WORKLOAD" > kubeconfig_workload

            kubectl --kubeconfig kubeconfig_workload -n kube-system create secret generic mgmt-kubeconfig --from-file=kubeconfig
        volumeMounts:
          - name: kubeconfig-adm
            mountPath: "/kube"
        env:
          - name: CLUSTER_ID
            value: "{{ inputs.parameters.cluster_id }}"
          - name: INFRA_PROVIDER
            value: "{{ inputs.parameters.infra_provider }}"

    - name: install-volumesnapshot-crds
      container:
        name: install-volumesnapshot-crds
        image: harbor.taco-cat.xyz/tks/kubectl-shell:latest-v1.21.1-amd64
        command:
          - /bin/bash
          - "-exc"
          - |
            KUBECONFIG=$(kubectl get secret -n {{workflow.parameters.cluster_id}} {{workflow.parameters.cluster_id}}-tks-kubeconfig -o jsonpath="{.data.value}" | base64 -d)
            echo -e "kubeconfig:\n$KUBECONFIG" | head -n 5
            cat <<< "$KUBECONFIG" > kubeconfig_temp

            kubectl --kubeconfig kubeconfig_temp apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/master/client/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml -n kube-system
            kubectl --kubeconfig kubeconfig_temp apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/master/client/config/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml -n kube-system
            kubectl --kubeconfig kubeconfig_temp apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/master/client/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml -n kube-system


    - name: k8s-rbac-setting
      steps:
        - - name: create-cluster-role-for-create
            templateRef:
              name: k8s-client
              template: create-cluster-role
            arguments:
              parameters:
                - name: target_cluster_id
                  value: "{{workflow.parameters.cluster_id}}"
                - name: is_self_target
                  value: "false"
                - name: cluster_role_name
                  value: "cluster-admin-create"
                - name: api_group
                  value: '*'
                - name: resource_name
                  value: '*'
                - name: verbs
                  value: '["create"]'
                - name: ignore_exist
                  value: 'true'
        - - name: create-cluster-role-for-read
            templateRef:
              name: k8s-client
              template: create-cluster-role
            arguments:
              parameters:
                - name: target_cluster_id
                  value: "{{workflow.parameters.cluster_id}}"
                - name: is_self_target
                  value: "false"
                - name: cluster_role_name
                  value: "cluster-admin-read"
                - name: api_group
                  value: '*'
                - name: resource_name
                  value: '*'
                - name: verbs
                  value: '["get", "list", "watch"]'
                - name: ignore_exist
                  value: 'true'
        - - name: create-cluster-role-for-update
            templateRef:
              name: k8s-client
              template: create-cluster-role
            arguments:
              parameters:
                - name: target_cluster_id
                  value: "{{workflow.parameters.cluster_id}}"
                - name: is_self_target
                  value: "false"
                - name: cluster_role_name
                  value: "cluster-admin-update"
                - name: api_group
                  value: '*'
                - name: resource_name
                  value: '*'
                - name: verbs
                  value: '["update", "patch"]'
                - name: ignore_exist
                  value: 'true'
        - - name: create-cluster-role-for-delete
            templateRef:
              name: k8s-client
              template: create-cluster-role
            arguments:
              parameters:
                - name: target_cluster_id
                  value: "{{workflow.parameters.cluster_id}}"
                - name: is_self_target
                  value: "false"
                - name: cluster_role_name
                  value: "cluster-admin-delete"
                - name: api_group
                  value: '*'
                - name: resource_name
                  value: '*'
                - name: verbs
                  value: '["delete", "deletecollection"]'
                - name: ignore_exist
                  value: 'true'
        - - name: set-cluster-role-binding-create
            templateRef:
              name: k8s-client
              template: create-cluster-role-binding
            arguments:
              parameters:
                - name: target_cluster_id
                  value: "{{workflow.parameters.cluster_id}}"
                - name: is_self_target
                  value: "false"
                - name: rolebinding_name
                  value: "cluster-admin-create-rb"
                - name: role_name
                  value: "cluster-admin-create"
                - name: group_list
                  value: '["cluster-admin-create"]'
        - - name: set-cluster-role-binding-read
            templateRef:
              name: k8s-client
              template: create-cluster-role-binding
            arguments:
              parameters:
                - name: target_cluster_id
                  value: "{{workflow.parameters.cluster_id}}"
                - name: is_self_target
                  value: "false"
                - name: rolebinding_name
                  value: "cluster-admin-read-rb"
                - name: role_name
                  value: "cluster-admin-read"
                - name: group_list
                  value: '["cluster-admin-read"]'
        - - name: set-cluster-role-binding-update
            templateRef:
              name: k8s-client
              template: create-cluster-role-binding
            arguments:
              parameters:
                - name: target_cluster_id
                  value: "{{workflow.parameters.cluster_id}}"
                - name: is_self_target
                  value: "false"
                - name: rolebinding_name
                  value: "cluster-admin-update-rb"
                - name: role_name
                  value: "cluster-admin-update"
                - name: group_list
                  value: '["cluster-admin-update"]'
        - - name: set-cluster-role-binding-delete
            templateRef:
              name: k8s-client
              template: create-cluster-role-binding
            arguments:
              parameters:
                - name: target_cluster_id
                  value: "{{workflow.parameters.cluster_id}}"
                - name: is_self_target
                  value: "false"
                - name: rolebinding_name
                  value: "cluster-admin-delete-rb"
                - name: role_name
                  value: "cluster-admin-delete"
                - name: group_list
                  value: '["cluster-admin-delete"]'

    - name: create-endpoint-secret
      inputs:
        parameters:
          - name: cluster_domains
          - name: cluster_id
      container:
        name: create-namespace
        image: harbor.taco-cat.xyz/tks/hyperkube:v1.18.6
        command:
          - /bin/bash
          - '-c'
          - |
            CLUSTER_ID={{inputs.parameters.cluster_id}}

            GRAFANA_URL=$(echo $CLUSTER_DOMAINS | jq -r '. | map(select(.domainType | contains("grafana"))|.url)'[])
            LOKI_URL=$(echo $CLUSTER_DOMAINS | jq -r '. | map(select(.domainType | contains("loki"))|.url)'[])
            MINIO_URL=$(echo $CLUSTER_DOMAINS | jq -r '. | map(select(.domainType | contains("minio"))|.url)'[])
            PROMETHEUS_URL=$(echo $CLUSTER_DOMAINS | jq -r '. | map(select(.domainType | contains("prometheus"))|.url)'[])
            THANOS_URL=$(echo $CLUSTER_DOMAINS | jq -r '. | map(select(.domainType | contains("thanos"))|.url)'[])
            LOKI_USER_URL=$(echo $CLUSTER_DOMAINS | jq -r '. | map(select(.domainType | contains("loki_user"))|.url)'[])
            THANOS_RULER_URL=$(echo $CLUSTER_DOMAINS | jq -r '. | map(select(.domainType | contains("thanos_ruler"))|.url)'[])
            KIALI_URL=$(echo $CLUSTER_DOMAINS | jq -r '. | map(select(.domainType | contains("kiali"))|.url)'[])
            JAEGER_URL=$(echo $CLUSTER_DOMAINS | jq -r '. | map(select(.domainType | contains("jaeger"))|.url)'[])

            cat <<EOF > tks-endpoint-secret.yaml
            ---
            apiVersion: v1
            kind: Secret
            metadata:
              name: tks-endpoint-secret
              namespace: ${CLUSTER_ID}
            data:
              grafana: $(echo ${GRAFANA_URL} | base64)  # 30001
              loki: $(echo ${LOKI_URL} | base64)   # 30002
              minio: $(echo ${MINIO_URL} | base64)   # 30003
              prometheus: $(echo ${PROMETHEUS_URL} | base64)    # 30004
              thanos: $(echo ${THANOS_URL} | base64)  # 30005 (queryfrontend만 합시다...)
              loki_user: $(echo ${LOKI_USER_URL} | base64)   # 30006
              thanos_ruler: $(echo ${THANOS_RULER_URL} | base64)   # 30007
              kiali: $(echo ${KIALI_URL} | base64)    # 30011
              jaeger: $(echo ${JAEGER_URL} | base64)    # 30012
            EOF
            kubectl apply -f tks-endpoint-secret.yaml
        env:
        - name: CLUSTER_DOMAINS
          value: "{{inputs.parameters.cluster_domains}}"

      activeDeadlineSeconds: 30