From 9e594d7d00b30fe4910f98ed28647f9f9415d4fd Mon Sep 17 00:00:00 2001
From: Robert Choi <taeilchoi1@gmail.com>
Date: Mon, 27 Sep 2021 19:05:34 +0900
Subject: [PATCH 1/9] add sealed_secrets WFT

---
 sealed_secrets/README.md                      |  5 ++
 sealed_secrets/deploy-secrets.yaml            | 34 ++++++++++
 .../setup-sealed-secrets-infra.yaml           | 64 +++++++++++++++++++
 3 files changed, 103 insertions(+)
 create mode 100644 sealed_secrets/README.md
 create mode 100644 sealed_secrets/deploy-secrets.yaml
 create mode 100644 sealed_secrets/setup-sealed-secrets-infra.yaml

diff --git a/sealed_secrets/README.md b/sealed_secrets/README.md
new file mode 100644
index 00000000..eca41efe
--- /dev/null
+++ b/sealed_secrets/README.md
@@ -0,0 +1,5 @@
+## Install kubeseal binary
+```
+$ wget https://github.com/bitnami-labs/sealed-secrets/releases/download/v0.16.0/kubeseal-linux-amd64 -O kubeseal
+$ sudo install -m 755 kubeseal-linux-amd64 /usr/local/bin/kubeseal
+```
diff --git a/sealed_secrets/deploy-secrets.yaml b/sealed_secrets/deploy-secrets.yaml
new file mode 100644
index 00000000..fdde466e
--- /dev/null
+++ b/sealed_secrets/deploy-secrets.yaml
@@ -0,0 +1,34 @@
+apiVersion: argoproj.io/v1alpha1
+kind: WorkflowTemplate
+metadata:
+  name: deploy-secrets
+  namespace: argo
+spec:
+  templates:
+  - name: deploySecrets
+    activeDeadlineSeconds: 120
+    inputs:
+      parameters:
+      - name: repo_url
+        value: "" # Eg, "openinfradev/tks-admin-site"
+      - name: secret_path
+        value: "" # Eg, "directory/secret.yaml"
+    container:
+      image: k8s.gcr.io/hyperkube:v1.18.8
+      imagePullPolicy: IfNotPresent
+      command:
+      - /bin/bash
+      - -c
+      - |
+  
+        git clone https://$(echo $gittoken|xargs)@{{input.parameters.repo_url}}
+        repo_name=$(basename "{{input.parameters.repo_url}}")
+
+        kubectl apply -f $repo_name/{{input.parameters.secret_path}}
+
+        ## Check if workflow fails if this cmd fails. Otherwise, should add exit cmd ##
+        #kubectl get secret -n kube-system -l sealedsecrets.bitnami.com/sealed-secrets-key | grep sealed-secrets-key
+
+      envFrom:
+        - secretRef:
+            name: "gittoken"
diff --git a/sealed_secrets/setup-sealed-secrets-infra.yaml b/sealed_secrets/setup-sealed-secrets-infra.yaml
new file mode 100644
index 00000000..44b1af33
--- /dev/null
+++ b/sealed_secrets/setup-sealed-secrets-infra.yaml
@@ -0,0 +1,64 @@
+apiVersion: argoproj.io/v1alpha1
+kind: WorkflowTemplate
+metadata:
+  name: setup-sealed-secrets-infra
+  namespace: argo
+spec:
+  entrypoint: process
+  arguments:
+    parameters:
+    - name: site_name
+      value: "hanu-reference"
+    # TODO: This should be renamed to app_group_name
+    - name: app_name
+      value: "sealed-secrets"
+    - name: manifest_repo_url
+      value: "https://github.com/openinfradev/decapod-manifests"
+    - name: revision
+      value: "main"
+## Uncomment following lines if you need to customize these ##
+#    - name: master_key_repo_url
+#      value: "github.com/openinfradev/tks-admin-site"
+#    - name: master_key_secret_name
+#      value: "github.com/openinfradev/tks-admin-site"
+#    - name: sealed_secrets_repo_url
+#      value: "github.com/openinfradev/tks-admin-site"
+#      ...
+##############################################################
+  templates:
+  - name: process
+    steps:
+    - - name: deployMasterKey
+        templateRef:
+          name: deploy-secrets
+          template: deploySecrets
+        argument:
+          parameters:
+          - name: repo_url
+            value: "github.com/openinfradev/tks-admin-site"
+          - name: secret_path
+            value: "sealed-secret-key/master-key-secret.yaml"
+
+    - - name: installControllers
+        templateRef:
+          name: tks-create-application
+          template: AppGroupOnAdmin
+        arguments:
+          parameters:
+          - name: list
+            value: |
+              [
+                 { "path": “sealed-secret-controller", "namespace": “kube-system” },
+                 { "path": “kubed", "namespace": “kube-system” }
+              ]
+
+    - - name: deploySealedSecret
+        templateRef:
+          name: deploy-secrets
+          template: deploySecrets
+        argument:
+          parameters:
+          - name: repo_url
+            value: "github.com/openinfradev/tks-admin-site"
+          - name: secret_path
+            value: "sealed-certificates/taco-cat-tls-sealed.yaml"

From 71437e2bd976fcdba656f29fac2225eaa1683a01 Mon Sep 17 00:00:00 2001
From: Robert Choi <taeilchoi1@gmail.com>
Date: Tue, 28 Sep 2021 18:55:48 +0900
Subject: [PATCH 2/9] update workflow in consistent manner - sync variable
 names with other workflows - use 'steps' instead of 'dags' when there are
 only sequential tasks

---
 deploy_apps/tks-lma-federation-wftpl.yaml     |  7 ++---
 deploy_apps/tks-remove-servicemesh-wftpl.yaml |  6 ++--
 deploy_apps/tks-service-mesh-wftpl.yaml       |  2 +-
 sealed_secrets/deploy-secrets.yaml            |  3 +-
 .../setup-sealed-secrets-infra.yaml           |  4 +--
 tks-cluster/create-usercluster-wftpl.yaml     | 31 ++++++++-----------
 tks-cluster/tks-createapp-wftp.yaml           | 30 +++++++++---------
 7 files changed, 39 insertions(+), 44 deletions(-)

diff --git a/deploy_apps/tks-lma-federation-wftpl.yaml b/deploy_apps/tks-lma-federation-wftpl.yaml
index 52fcbb64..e30312b8 100644
--- a/deploy_apps/tks-lma-federation-wftpl.yaml
+++ b/deploy_apps/tks-lma-federation-wftpl.yaml
@@ -9,8 +9,7 @@ spec:
     parameters:
     - name: site_name
       value: "hanu-reference"
-    # TODO: This should be renamed to app_group_name
-    - name: app_name
+    - name: app_group
       value: "lma"
     # Replace these urls properly for your env #
     - name: site_repo_url
@@ -79,7 +78,7 @@ spec:
           - name: cluster_name
             value: "{{item.name}}"
           - name: app_group
-            value: "{{workflow.parameters.app_name}}"
+            value: "{{workflow.parameters.app_group}}"
           - name: chart
             value: "thanos"
           - name: kv_map_str 
@@ -99,7 +98,7 @@ spec:
           - name: cluster_name
             value: "{{steps.collectThanosScEndpoints.outputs.parameters.cur_cluster_name}}"
           - name: app_group
-            value: "{{workflow.parameters.app_name}}"
+            value: "{{workflow.parameters.app_group}}"
           - name: chart
             value: "thanos"
           - name: kv_map_str
diff --git a/deploy_apps/tks-remove-servicemesh-wftpl.yaml b/deploy_apps/tks-remove-servicemesh-wftpl.yaml
index acc90bd4..06ffb08e 100644
--- a/deploy_apps/tks-remove-servicemesh-wftpl.yaml
+++ b/deploy_apps/tks-remove-servicemesh-wftpl.yaml
@@ -26,7 +26,7 @@ spec:
             template: delete-argocd-app
             arguments:
               parameters:
-                - name: app_name
+                - name: app_group
                   value: service-mesh
                 - name: site_name
                   value: "{{workflow.parameters.site_name}}"
@@ -71,7 +71,7 @@ spec:
     - name: delete-argocd-app
       inputs:
         parameters:
-          - name: app_name
+          - name: app_group
           - name: site_name
       container:
         name: delete-argocd-app
@@ -98,7 +98,7 @@ spec:
               name: decapod-argocd-config
         env:
           - name: APP_NAME
-            value: "{{inputs.parameters.app_name}}"
+            value: "{{inputs.parameters.app_group}}"
           - name: SITE_NAME
             value: '{{inputs.parameters.site_name}}'
       activeDeadlineSeconds: 900
diff --git a/deploy_apps/tks-service-mesh-wftpl.yaml b/deploy_apps/tks-service-mesh-wftpl.yaml
index bfd50bee..fbb80b8c 100644
--- a/deploy_apps/tks-service-mesh-wftpl.yaml
+++ b/deploy_apps/tks-service-mesh-wftpl.yaml
@@ -9,7 +9,7 @@ spec:
     parameters:
       - name: site_name
         value: hanu-reference
-      - name: app_name
+      - name: app_group
         value: service-mesh
       - name: manifest_repo_url
         value: 'https://github.com/openinfradev/decapod-manifests'
diff --git a/sealed_secrets/deploy-secrets.yaml b/sealed_secrets/deploy-secrets.yaml
index fdde466e..8d2af0ad 100644
--- a/sealed_secrets/deploy-secrets.yaml
+++ b/sealed_secrets/deploy-secrets.yaml
@@ -26,8 +26,7 @@ spec:
 
         kubectl apply -f $repo_name/{{input.parameters.secret_path}}
 
-        ## Check if workflow fails if this cmd fails. Otherwise, should add exit cmd ##
-        #kubectl get secret -n kube-system -l sealedsecrets.bitnami.com/sealed-secrets-key | grep sealed-secrets-key
+        # TODO: need to add logic to check if the secret was successfully created?
 
       envFrom:
         - secretRef:
diff --git a/sealed_secrets/setup-sealed-secrets-infra.yaml b/sealed_secrets/setup-sealed-secrets-infra.yaml
index 44b1af33..573c3d5d 100644
--- a/sealed_secrets/setup-sealed-secrets-infra.yaml
+++ b/sealed_secrets/setup-sealed-secrets-infra.yaml
@@ -10,7 +10,7 @@ spec:
     - name: site_name
       value: "hanu-reference"
     # TODO: This should be renamed to app_group_name
-    - name: app_name
+    - name: app_group
       value: "sealed-secrets"
     - name: manifest_repo_url
       value: "https://github.com/openinfradev/decapod-manifests"
@@ -42,7 +42,7 @@ spec:
     - - name: installControllers
         templateRef:
           name: tks-create-application
-          template: AppGroupOnAdmin
+          template: installAppsOnAdmin
         arguments:
           parameters:
           - name: list
diff --git a/tks-cluster/create-usercluster-wftpl.yaml b/tks-cluster/create-usercluster-wftpl.yaml
index 16c5c9b0..7abedede 100755
--- a/tks-cluster/create-usercluster-wftpl.yaml
+++ b/tks-cluster/create-usercluster-wftpl.yaml
@@ -17,21 +17,19 @@ spec:
       value: main
     - name: tks_admin
       value: "tks-admin"
-    - name: app_name
+    - name: app_group
       value: "tks-cluster"
 
   templates:
   - name: deploy
-    dag:
-      tasks:
-      - name: tks-create-cluster-site
+    steps:
+    - - name: tks-create-cluster-site
         template: new-cluster-site
-        dependencies: []
 
-      - name: k8s-by-capi
+    - - name: k8s-by-capi
         templateRef:
           name: tks-create-application
-          template: AppGroupOnAdmin
+          template: installAppsOnAdmin
         arguments:
           parameters: 
           - name: list
@@ -39,28 +37,25 @@ spec:
               [
                 { "path": "cluster-api-aws", "namespace": "argo" }
               ]
-        dependencies: [tks-create-cluster-site]
 
-      - name: wait-for-clster-is-registered
-        template: wait-template
-        dependencies: [k8s-by-capi]
+    - - name: wait-for-clster-to-be-registered
+        template: wait-for-cluster-registration
 
-      - name: ready-for-cni-and-csi
+      # TODO: What does this name mean? Wait for CNI?
+    - - name: ready-for-cni-and-csi
         templateRef:
           name: tks-create-application
-          template: AppGroup
+          template: installApps
         arguments:
           parameters: 
           - name: list
             value: |
               [
                 { "path": "ingress-nginx", "namespace": "taco-system" },
-                { "path": "kubed", "namespace": "taco-system" },
                 { "path": "kubernetes-addons", "namespace": "taco-system" }
               ]
-        dependencies: [k8s-by-capi, wait-for-clster-is-registered ]
 
-  - name: wait-template
+  - name: wait-for-cluster-registration
     activeDeadlineSeconds: 1800
     container:
       image: ghcr.io/openinfradev/argocd-cli:v2.0.1
@@ -71,7 +66,7 @@ spec:
         yes | ./argocd login --insecure $ARGO_SERVER --username $ARGO_USERNAME --password $ARGO_PASSWORD
 
         while [ $(./argocd cluster list | grep \ $target\ | wc -l ) == 0 ]; do
-            echo "> Wait for cluster is registered"
+            echo "> Wait for cluster to be registered"
             sleep 30
         done
       envFrom:
@@ -121,4 +116,4 @@ spec:
         - name: git_account
           value: "{{workflow.parameters.git_account}}"
         - name: revision
-          value: "{{workflow.parameters.revision}}"
\ No newline at end of file
+          value: "{{workflow.parameters.revision}}"
diff --git a/tks-cluster/tks-createapp-wftp.yaml b/tks-cluster/tks-createapp-wftp.yaml
index a55e1f10..adadf782 100755
--- a/tks-cluster/tks-createapp-wftp.yaml
+++ b/tks-cluster/tks-createapp-wftp.yaml
@@ -8,12 +8,12 @@ spec:
     parameters:
     - name: cluster_id
       value: "hanu-deploy-apps"
-    - name: app_name
+    - name: app_group
       value: "lma"
-    - name: repository_url
+    - name: manifest_repo_url
       value: "https://github.com/openinfradev/decapod-manifests"
     - name: revision
-      value: main
+      value: "main"
   templates:
   - name: createApp
     inputs:
@@ -32,21 +32,23 @@ spec:
         #NAMESPACE=$SITE_NAME
         CD_APP=${SITE_NAME:0:8}-$PATH
         echo "argo-cd application name: $CD_APP"
+
         # log into Argo CD server
         ./argocd login $ARGO_SERVER --plaintext --insecure --username $ARGO_USERNAME \
         --password $ARGO_PASSWORD
 
-        # check if app already exists.
         REPO=https://$(echo $gittoken|xargs)@github.com/$git_account/${contract_id}-manifests.git
+
+        # check if app already exists.
         ./argocd app get $CD_APP 
         if [[ $? -ne 0 ]]; then
           echo "$CD_APP application is not in server"
           # create new application if not exists.
-          echo ./argocd app create $CD_APP --repo $REPO --revision $REVISION --path $SITE_NAME/$TACO_APP/$PATH --dest-namespace $NAMESPACE --dest-name $TARGET_CLUSTER --project $TACO_APP --label app=$TACO_APP --directory-recurse
+          echo ./argocd app create $CD_APP --repo $REPO --revision $REVISION --path $SITE_NAME/$APP_GROUP/$PATH --dest-namespace $NAMESPACE --dest-name $TARGET_CLUSTER --project $APP_GROUP --label app=$APP_GROUP --directory-recurse
           ./argocd app create $CD_APP --repo $REPO --revision $REVISION \
-            --path $SITE_NAME/$TACO_APP/$PATH \
+            --path $SITE_NAME/$APP_GROUP/$PATH \
             --dest-namespace $NAMESPACE --dest-name $TARGET_CLUSTER \
-            --project $TACO_APP --label app=$TACO_APP --directory-recurse
+            --project $APP_GROUP --label app=$APP_GROUP --directory-recurse
 
           if [[ $? -ne 0 ]]; then
             exit $?
@@ -68,8 +70,8 @@ spec:
           value: "{{workflow.parameters.cluster_id}}"
         - name: TARGET_CLUSTER
           value: "{{inputs.parameters.target_cluster}}"
-        - name: TACO_APP
-          value: "{{workflow.parameters.app_name}}"
+        - name: APP_GROUP
+          value: "{{workflow.parameters.app_group}}"
         - name: NAMESPACE
           value: "{{inputs.parameters.namespace}}"
         - name: REVISION
@@ -82,12 +84,12 @@ spec:
           value: "{{workflow.parameters.git_account}}"
 
 
-  - name: AppGroup
+  - name: installApps
     inputs:
       parameters:
       - name: list
     steps:
-    - - name: "InstallAppGroup"
+    - - name: "InstallApps"
         template: createApp
         arguments:
           parameters:
@@ -96,16 +98,16 @@ spec:
           - {name: target_cluster, value: "{{workflow.parameters.cluster_id}}"}
         withParam: "{{inputs.parameters.list}}"
 
-  - name: AppGroupOnAdmin
+  - name: installAppsOnAdmin
     inputs:
       parameters:
       - name: list
     steps:
-    - - name: "InstallAppGroup"
+    - - name: "InstallAppsOnAdmin"
         template: createApp
         arguments:
           parameters:
           - {name: path, value: "{{item.path}}"}
           - {name: namespace, value: "{{item.namespace}}"}
           - {name: target_cluster, value: "{{workflow.parameters.tks_admin}}"}
-        withParam: "{{inputs.parameters.list}}"
\ No newline at end of file
+        withParam: "{{inputs.parameters.list}}"

From 52067e1d9706b54c497a596c71fa129192dc5b64 Mon Sep 17 00:00:00 2001
From: Robert Choi <taeilchoi1@gmail.com>
Date: Wed, 29 Sep 2021 10:00:12 +0900
Subject: [PATCH 3/9] specify kubeconfig when deploying secrets

---
 sealed_secrets/deploy-secrets.yaml             | 17 ++++++++++++-----
 sealed_secrets/setup-sealed-secrets-infra.yaml |  4 ++++
 2 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/sealed_secrets/deploy-secrets.yaml b/sealed_secrets/deploy-secrets.yaml
index 8d2af0ad..b3211a0a 100644
--- a/sealed_secrets/deploy-secrets.yaml
+++ b/sealed_secrets/deploy-secrets.yaml
@@ -9,10 +9,9 @@ spec:
     activeDeadlineSeconds: 120
     inputs:
       parameters:
-      - name: repo_url
-        value: "" # Eg, "openinfradev/tks-admin-site"
-      - name: secret_path
-        value: "" # Eg, "directory/secret.yaml"
+      - name: repo_url   # Eg, "openinfradev/tks-admin-site"
+      - name: secret_path  # Eg, "directory/secret.yaml"
+      - name: kubeconfig_secret_name
     container:
       image: k8s.gcr.io/hyperkube:v1.18.8
       imagePullPolicy: IfNotPresent
@@ -21,13 +20,21 @@ spec:
       - -c
       - |
   
+        cat <<< "$KUBE_CONFIG" > /etc/kubeconfig
+
         git clone https://$(echo $gittoken|xargs)@{{input.parameters.repo_url}}
         repo_name=$(basename "{{input.parameters.repo_url}}")
 
-        kubectl apply -f $repo_name/{{input.parameters.secret_path}}
+        kubectl apply --kubeconfig=/etc/kubeconfig -f $repo_name/{{input.parameters.secret_path}}
 
         # TODO: need to add logic to check if the secret was successfully created?
 
       envFrom:
         - secretRef:
             name: "gittoken"
+      env:
+        - name: KUBE_CONFIG
+          valueFrom:
+            secretKeyRef:
+              name: "{{ inputs.parameters.kubeconfig_secret_name }}"
+              key: value
diff --git a/sealed_secrets/setup-sealed-secrets-infra.yaml b/sealed_secrets/setup-sealed-secrets-infra.yaml
index 573c3d5d..23217bb6 100644
--- a/sealed_secrets/setup-sealed-secrets-infra.yaml
+++ b/sealed_secrets/setup-sealed-secrets-infra.yaml
@@ -38,6 +38,8 @@ spec:
             value: "github.com/openinfradev/tks-admin-site"
           - name: secret_path
             value: "sealed-secret-key/master-key-secret.yaml"
+          - name: kubeconfig_secret_name
+            value: "{{workflow.parameters.site_name}}-kubeconfig"
 
     - - name: installControllers
         templateRef:
@@ -62,3 +64,5 @@ spec:
             value: "github.com/openinfradev/tks-admin-site"
           - name: secret_path
             value: "sealed-certificates/taco-cat-tls-sealed.yaml"
+          - name: kubeconfig_secret_name
+            value: "{{workflow.parameters.site_name}}-kubeconfig"

From 8cd91a506559c1a932eb5713209087709a54f534 Mon Sep 17 00:00:00 2001
From: Robert Choi <taeilchoi1@gmail.com>
Date: Wed, 29 Sep 2021 10:19:44 +0900
Subject: [PATCH 4/9] bugfix: add missing param

---
 sealed_secrets/setup-sealed-secrets-infra.yaml | 4 ++--
 tks-cluster/tks-createapp-wftp.yaml            | 8 ++++++--
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/sealed_secrets/setup-sealed-secrets-infra.yaml b/sealed_secrets/setup-sealed-secrets-infra.yaml
index 23217bb6..5cbb9afd 100644
--- a/sealed_secrets/setup-sealed-secrets-infra.yaml
+++ b/sealed_secrets/setup-sealed-secrets-infra.yaml
@@ -32,7 +32,7 @@ spec:
         templateRef:
           name: deploy-secrets
           template: deploySecrets
-        argument:
+        arguments:
           parameters:
           - name: repo_url
             value: "github.com/openinfradev/tks-admin-site"
@@ -58,7 +58,7 @@ spec:
         templateRef:
           name: deploy-secrets
           template: deploySecrets
-        argument:
+        arguments:
           parameters:
           - name: repo_url
             value: "github.com/openinfradev/tks-admin-site"
diff --git a/tks-cluster/tks-createapp-wftp.yaml b/tks-cluster/tks-createapp-wftp.yaml
index adadf782..145cd613 100755
--- a/tks-cluster/tks-createapp-wftp.yaml
+++ b/tks-cluster/tks-createapp-wftp.yaml
@@ -8,10 +8,14 @@ spec:
     parameters:
     - name: cluster_id
       value: "hanu-deploy-apps"
+    - name: contract_id
+      value: ""
+    - name: git_account
+      value: ""
+    - name: tks_admin
+      value: ""
     - name: app_group
       value: "lma"
-    - name: manifest_repo_url
-      value: "https://github.com/openinfradev/decapod-manifests"
     - name: revision
       value: "main"
   templates:

From 1727b4bf0623901ec806f76a2b813bcb480a4c6c Mon Sep 17 00:00:00 2001
From: Robert Choi <taeilchoi1@gmail.com>
Date: Wed, 29 Sep 2021 11:16:14 +0900
Subject: [PATCH 5/9] update workflow params to sync them with tks-createapp
 WFT

---
 .../setup-sealed-secrets-infra.yaml           | 20 ++++++++++++-------
 tks-cluster/tks-createapp-wftp.yaml           |  2 --
 2 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/sealed_secrets/setup-sealed-secrets-infra.yaml b/sealed_secrets/setup-sealed-secrets-infra.yaml
index 5cbb9afd..f43f3231 100644
--- a/sealed_secrets/setup-sealed-secrets-infra.yaml
+++ b/sealed_secrets/setup-sealed-secrets-infra.yaml
@@ -7,13 +7,17 @@ spec:
   entrypoint: process
   arguments:
     parameters:
-    - name: site_name
-      value: "hanu-reference"
-    # TODO: This should be renamed to app_group_name
+    - name: contract_id
+      value: "011b88fa-4d53-439f-9336-67845f994051"
+    - name: cluster_id
+      value: ""
+    - name: git_account
+      value: "tks-management"
+    # For cluster selection in Argo CD #
+    - name: tks_admin
+      value: "tks-admin"
     - name: app_group
       value: "sealed-secrets"
-    - name: manifest_repo_url
-      value: "https://github.com/openinfradev/decapod-manifests"
     - name: revision
       value: "main"
 ## Uncomment following lines if you need to customize these ##
@@ -34,12 +38,14 @@ spec:
           template: deploySecrets
         arguments:
           parameters:
+          # In case of user cluster, repo url should be constructed from parameters
+          # such as git_account, contract id & cluster id.
           - name: repo_url
             value: "github.com/openinfradev/tks-admin-site"
           - name: secret_path
             value: "sealed-secret-key/master-key-secret.yaml"
           - name: kubeconfig_secret_name
-            value: "{{workflow.parameters.site_name}}-kubeconfig"
+            value: "{{workflow.parameters.cluster_id}}-kubeconfig"
 
     - - name: installControllers
         templateRef:
@@ -65,4 +71,4 @@ spec:
           - name: secret_path
             value: "sealed-certificates/taco-cat-tls-sealed.yaml"
           - name: kubeconfig_secret_name
-            value: "{{workflow.parameters.site_name}}-kubeconfig"
+            value: "{{workflow.parameters.cluster_id}}-kubeconfig"
diff --git a/tks-cluster/tks-createapp-wftp.yaml b/tks-cluster/tks-createapp-wftp.yaml
index 145cd613..ae7e71a6 100755
--- a/tks-cluster/tks-createapp-wftp.yaml
+++ b/tks-cluster/tks-createapp-wftp.yaml
@@ -82,8 +82,6 @@ spec:
           value: "{{workflow.parameters.revision}}"
         - name: contract_id
           value: "{{workflow.parameters.contract_id}}"
-        - name: cluster_id
-          value: "{{workflow.parameters.cluster_id}}"
         - name: git_account
           value: "{{workflow.parameters.git_account}}"
 

From 85b8556b71989bdd1a24985f65a9de570108ce7b Mon Sep 17 00:00:00 2001
From: Robert Choi <taeilchoi1@gmail.com>
Date: Wed, 29 Sep 2021 14:40:33 +0900
Subject: [PATCH 6/9] separate deploySecrets template into two types

- one for admin cluster, the other for user cluster
---
 sealed_secrets/deploy-secrets.yaml            | 32 ++++++++++++++++---
 .../setup-sealed-secrets-infra.yaml           | 25 +++++++--------
 2 files changed, 40 insertions(+), 17 deletions(-)

diff --git a/sealed_secrets/deploy-secrets.yaml b/sealed_secrets/deploy-secrets.yaml
index b3211a0a..27f31c18 100644
--- a/sealed_secrets/deploy-secrets.yaml
+++ b/sealed_secrets/deploy-secrets.yaml
@@ -5,13 +5,13 @@ metadata:
   namespace: argo
 spec:
   templates:
-  - name: deploySecrets
+  - name: deploySecretsToUserCluster
     activeDeadlineSeconds: 120
     inputs:
       parameters:
-      - name: repo_url   # Eg, "openinfradev/tks-admin-site"
-      - name: secret_path  # Eg, "directory/secret.yaml"
-      - name: kubeconfig_secret_name
+      - name: repo_url   # Eg, "tks-management/011b88fa-4d53-439f-9336-67845f994051/25db54c6-d6cb-459b-9148-1b02ac545753"
+      - name: secret_path  # Eg, "sealed-cert/secret.yaml"
+      - name: kubeconfig_secret_name  # Eg, "25db54c6-d6cb-459b-9148-1b02ac545753-kubeconfig"
     container:
       image: k8s.gcr.io/hyperkube:v1.18.8
       imagePullPolicy: IfNotPresent
@@ -38,3 +38,27 @@ spec:
             secretKeyRef:
               name: "{{ inputs.parameters.kubeconfig_secret_name }}"
               key: value
+
+  - name: deploySecretsToAdminCluster
+    activeDeadlineSeconds: 120
+    inputs:
+      parameters:
+      - name: repo_url   # Eg, "openinfradev/tks-admin-site"
+      - name: secret_path  # Eg, "directory/secret.yaml"
+    container:
+      image: k8s.gcr.io/hyperkube:v1.18.8
+      imagePullPolicy: IfNotPresent
+      command:
+      - /bin/bash
+      - -c
+      - |
+
+        git clone https://$(echo $gittoken|xargs)@{{input.parameters.repo_url}}
+        repo_name=$(basename "{{input.parameters.repo_url}}")
+
+        kubectl apply -f $repo_name/{{input.parameters.secret_path}}
+
+        # TODO: need to add logic to check if the secret was successfully created?
+      envFrom:
+        - secretRef:
+            name: "gittoken"
diff --git a/sealed_secrets/setup-sealed-secrets-infra.yaml b/sealed_secrets/setup-sealed-secrets-infra.yaml
index f43f3231..757db652 100644
--- a/sealed_secrets/setup-sealed-secrets-infra.yaml
+++ b/sealed_secrets/setup-sealed-secrets-infra.yaml
@@ -7,20 +7,23 @@ spec:
   entrypoint: process
   arguments:
     parameters:
+    # Params for user cluster #
     - name: contract_id
       value: "011b88fa-4d53-439f-9336-67845f994051"
     - name: cluster_id
       value: ""
     - name: git_account
       value: "tks-management"
-    # For cluster selection in Argo CD #
-    - name: tks_admin
-      value: "tks-admin"
+    # For create-application task #
+    - name: manifest_repo_url
+      value: "https://github.com/openinfradev/decapod-manifests"
+    - name: site_name
+      value: "hanu-reference"
     - name: app_group
       value: "sealed-secrets"
     - name: revision
       value: "main"
-## Uncomment following lines if you need to customize these ##
+## Uncomment following lines and customize to fetch any secrets you want ##
 #    - name: master_key_repo_url
 #      value: "github.com/openinfradev/tks-admin-site"
 #    - name: master_key_secret_name
@@ -28,14 +31,14 @@ spec:
 #    - name: sealed_secrets_repo_url
 #      value: "github.com/openinfradev/tks-admin-site"
 #      ...
-##############################################################
+###########################################################################
   templates:
   - name: process
     steps:
     - - name: deployMasterKey
         templateRef:
           name: deploy-secrets
-          template: deploySecrets
+          template: deploySecretsToAdminCluster
         arguments:
           parameters:
           # In case of user cluster, repo url should be constructed from parameters
@@ -44,13 +47,11 @@ spec:
             value: "github.com/openinfradev/tks-admin-site"
           - name: secret_path
             value: "sealed-secret-key/master-key-secret.yaml"
-          - name: kubeconfig_secret_name
-            value: "{{workflow.parameters.cluster_id}}-kubeconfig"
 
     - - name: installControllers
         templateRef:
-          name: tks-create-application
-          template: installAppsOnAdmin
+          name: create-application
+          template: installApps
         arguments:
           parameters:
           - name: list
@@ -63,12 +64,10 @@ spec:
     - - name: deploySealedSecret
         templateRef:
           name: deploy-secrets
-          template: deploySecrets
+          template: deploySecretsToAdminCluster
         arguments:
           parameters:
           - name: repo_url
             value: "github.com/openinfradev/tks-admin-site"
           - name: secret_path
             value: "sealed-certificates/taco-cat-tls-sealed.yaml"
-          - name: kubeconfig_secret_name
-            value: "{{workflow.parameters.cluster_id}}-kubeconfig"

From 831f5ddbde9be5cd36cd85b91bdaded4560d4215 Mon Sep 17 00:00:00 2001
From: Robert Choi <taeilchoi1@gmail.com>
Date: Wed, 29 Sep 2021 16:50:29 +0900
Subject: [PATCH 7/9] fix typo

---
 sealed_secrets/deploy-secrets.yaml | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/sealed_secrets/deploy-secrets.yaml b/sealed_secrets/deploy-secrets.yaml
index 27f31c18..b1ef389b 100644
--- a/sealed_secrets/deploy-secrets.yaml
+++ b/sealed_secrets/deploy-secrets.yaml
@@ -22,10 +22,10 @@ spec:
   
         cat <<< "$KUBE_CONFIG" > /etc/kubeconfig
 
-        git clone https://$(echo $gittoken|xargs)@{{input.parameters.repo_url}}
-        repo_name=$(basename "{{input.parameters.repo_url}}")
+        git clone https://$(echo $gittoken|xargs)@$repo_url
+        repo_name=$(basename $repo_url)
 
-        kubectl apply --kubeconfig=/etc/kubeconfig -f $repo_name/{{input.parameters.secret_path}}
+        kubectl apply --kubeconfig=/etc/kubeconfig -f $repo_name/"{{inputs.parameters.secret_path}}"
 
         # TODO: need to add logic to check if the secret was successfully created?
 
@@ -33,6 +33,8 @@ spec:
         - secretRef:
             name: "gittoken"
       env:
+        - name: repo_url
+          value: "{{ inputs.parameters.repo_url }}"
         - name: KUBE_CONFIG
           valueFrom:
             secretKeyRef:
@@ -53,12 +55,15 @@ spec:
       - -c
       - |
 
-        git clone https://$(echo $gittoken|xargs)@{{input.parameters.repo_url}}
-        repo_name=$(basename "{{input.parameters.repo_url}}")
+        git clone https://$(echo $gittoken|xargs)@$repo_url
+        repo_name=$(basename $repo_url)
 
-        kubectl apply -f $repo_name/{{input.parameters.secret_path}}
+        kubectl apply -f $repo_name/"{{inputs.parameters.secret_path}}"
 
         # TODO: need to add logic to check if the secret was successfully created?
+      env:
+        - name: repo_url
+          value: "{{ inputs.parameters.repo_url }}"
       envFrom:
         - secretRef:
             name: "gittoken"

From 1cf70c5753c0eeb5210250597286142ee3788b61 Mon Sep 17 00:00:00 2001
From: Robert Choi <taeilchoi1@gmail.com>
Date: Thu, 30 Sep 2021 17:52:33 +0900
Subject: [PATCH 8/9] use different workflows for admin and user clusters for
 now

until tks-createapp-wf is refactored soon
---
 ...setup-sealed-secrets-on-admincluster.yaml} |  9 +--
 .../setup-sealed-secrets-on-usercluster.yaml  | 67 +++++++++++++++++++
 tks-cluster/tks-createapp-wftp.yaml           |  2 +-
 3 files changed, 69 insertions(+), 9 deletions(-)
 rename sealed_secrets/{setup-sealed-secrets-infra.yaml => setup-sealed-secrets-on-admincluster.yaml} (90%)
 create mode 100644 sealed_secrets/setup-sealed-secrets-on-usercluster.yaml

diff --git a/sealed_secrets/setup-sealed-secrets-infra.yaml b/sealed_secrets/setup-sealed-secrets-on-admincluster.yaml
similarity index 90%
rename from sealed_secrets/setup-sealed-secrets-infra.yaml
rename to sealed_secrets/setup-sealed-secrets-on-admincluster.yaml
index 757db652..727b7d64 100644
--- a/sealed_secrets/setup-sealed-secrets-infra.yaml
+++ b/sealed_secrets/setup-sealed-secrets-on-admincluster.yaml
@@ -1,19 +1,12 @@
 apiVersion: argoproj.io/v1alpha1
 kind: WorkflowTemplate
 metadata:
-  name: setup-sealed-secrets-infra
+  name: setup-sealed-secrets-on-admincluster
   namespace: argo
 spec:
   entrypoint: process
   arguments:
     parameters:
-    # Params for user cluster #
-    - name: contract_id
-      value: "011b88fa-4d53-439f-9336-67845f994051"
-    - name: cluster_id
-      value: ""
-    - name: git_account
-      value: "tks-management"
     # For create-application task #
     - name: manifest_repo_url
       value: "https://github.com/openinfradev/decapod-manifests"
diff --git a/sealed_secrets/setup-sealed-secrets-on-usercluster.yaml b/sealed_secrets/setup-sealed-secrets-on-usercluster.yaml
new file mode 100644
index 00000000..01d39f19
--- /dev/null
+++ b/sealed_secrets/setup-sealed-secrets-on-usercluster.yaml
@@ -0,0 +1,67 @@
+apiVersion: argoproj.io/v1alpha1
+kind: WorkflowTemplate
+metadata:
+  name: setup-sealed-secrets-on-usercluster
+  namespace: argo
+spec:
+  entrypoint: process
+  arguments:
+    parameters:
+    - name: git_account
+      value: "tks-management"
+    - name: contract_id
+      value: "011b88fa-4d53-439f-9336-67845f994051"
+    - name: cluster_id
+      value: ""
+    - name: app_group
+      value: "sealed-secrets"
+    - name: revision
+      value: "main"
+  templates:
+  - name: process
+    steps:
+    - - name: deployMasterKey
+        templateRef:
+          name: deploy-secrets
+          template: deploySecretsToUserCluster
+        arguments:
+          parameters:
+          - name: repo_url
+            value: "github.com/openinfradev/tks-admin-site"
+          - name: secret_path
+            value: "sealed-secret-key/master-key-secret.yaml"
+          - name: kubeconfig_secret_name
+            value: "{{workflow.parameters.cluster_id}}-kubeconfig"
+
+    - - name: installControllers
+        templateRef:
+          name: tks-create-application
+          template: installApps
+        arguments:
+          parameters:
+          - name: list
+            value: |
+              [
+                 { "path": "sealed-secret-controller", "namespace": "kube-system" },
+                 { "path": "kubed", "namespace": "kube-system" }
+              ]
+
+    - - name: deploySealedSecret
+        templateRef:
+          name: deploy-secrets
+          template: deploySecretsToUserCluster
+        arguments:
+          parameters:
+          ##########################################################################
+          # For real use case example 
+          #- name: repo_url
+          #  value: "github.com/tks-management/{{workflow.parameters.contract_id}}"
+          #- name: secret_path
+          #  value: "sealed-certificates/user-cat-tls-sealed.yaml"
+          ##########################################################################
+          - name: repo_url
+            value: "github.com/openinfradev/tks-admin-site"
+          - name: secret_path
+            value: "sealed-certificates/taco-cat-tls-sealed.yaml"
+          - name: kubeconfig_secret_name
+            value: "{{workflow.parameters.cluster_id}}-kubeconfig"
diff --git a/tks-cluster/tks-createapp-wftp.yaml b/tks-cluster/tks-createapp-wftp.yaml
index ae7e71a6..7853c8c8 100755
--- a/tks-cluster/tks-createapp-wftp.yaml
+++ b/tks-cluster/tks-createapp-wftp.yaml
@@ -7,7 +7,7 @@ spec:
   arguments:
     parameters:
     - name: cluster_id
-      value: "hanu-deploy-apps"
+      value: "hanu-reference"
     - name: contract_id
       value: ""
     - name: git_account

From 556f964f5effe22d326afcbf3200401ce2d79267 Mon Sep 17 00:00:00 2001
From: Robert Choi <taeilchoi1@gmail.com>
Date: Fri, 1 Oct 2021 11:29:37 +0900
Subject: [PATCH 9/9] make it work

---
 sealed_secrets/deploy-secrets.yaml                       | 4 ++--
 sealed_secrets/setup-sealed-secrets-on-admincluster.yaml | 4 ++--
 sealed_secrets/setup-sealed-secrets-on-usercluster.yaml  | 4 ++--
 tks-cluster/create-usercluster-wftpl.yaml                | 4 ++--
 tks-cluster/tks-createapp-wftp.yaml                      | 4 ++--
 5 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/sealed_secrets/deploy-secrets.yaml b/sealed_secrets/deploy-secrets.yaml
index b1ef389b..85dc6c5e 100644
--- a/sealed_secrets/deploy-secrets.yaml
+++ b/sealed_secrets/deploy-secrets.yaml
@@ -22,7 +22,7 @@ spec:
   
         cat <<< "$KUBE_CONFIG" > /etc/kubeconfig
 
-        git clone https://$(echo $gittoken|xargs)@$repo_url
+        git clone https://$(echo -n $gittoken)@$repo_url
         repo_name=$(basename $repo_url)
 
         kubectl apply --kubeconfig=/etc/kubeconfig -f $repo_name/"{{inputs.parameters.secret_path}}"
@@ -55,7 +55,7 @@ spec:
       - -c
       - |
 
-        git clone https://$(echo $gittoken|xargs)@$repo_url
+        git clone https://$(echo -n $gittoken)@$repo_url
         repo_name=$(basename $repo_url)
 
         kubectl apply -f $repo_name/"{{inputs.parameters.secret_path}}"
diff --git a/sealed_secrets/setup-sealed-secrets-on-admincluster.yaml b/sealed_secrets/setup-sealed-secrets-on-admincluster.yaml
index 727b7d64..30fe285f 100644
--- a/sealed_secrets/setup-sealed-secrets-on-admincluster.yaml
+++ b/sealed_secrets/setup-sealed-secrets-on-admincluster.yaml
@@ -50,8 +50,8 @@ spec:
           - name: list
             value: |
               [
-                 { "path": “sealed-secret-controller", "namespace": “kube-system” },
-                 { "path": “kubed", "namespace": “kube-system” }
+                 { "path": "sealed-secrets-controller", "namespace": "kube-system" },
+                 { "path": "kubed", "namespace": "kube-system" }
               ]
 
     - - name: deploySealedSecret
diff --git a/sealed_secrets/setup-sealed-secrets-on-usercluster.yaml b/sealed_secrets/setup-sealed-secrets-on-usercluster.yaml
index 01d39f19..730dd11b 100644
--- a/sealed_secrets/setup-sealed-secrets-on-usercluster.yaml
+++ b/sealed_secrets/setup-sealed-secrets-on-usercluster.yaml
@@ -35,14 +35,14 @@ spec:
 
     - - name: installControllers
         templateRef:
-          name: tks-create-application
+          name: tks-create-application-new
           template: installApps
         arguments:
           parameters:
           - name: list
             value: |
               [
-                 { "path": "sealed-secret-controller", "namespace": "kube-system" },
+                 { "path": "sealed-secrets-controller", "namespace": "kube-system" },
                  { "path": "kubed", "namespace": "kube-system" }
               ]
 
diff --git a/tks-cluster/create-usercluster-wftpl.yaml b/tks-cluster/create-usercluster-wftpl.yaml
index 7abedede..67da8a85 100755
--- a/tks-cluster/create-usercluster-wftpl.yaml
+++ b/tks-cluster/create-usercluster-wftpl.yaml
@@ -28,7 +28,7 @@ spec:
 
     - - name: k8s-by-capi
         templateRef:
-          name: tks-create-application
+          name: tks-create-application-new
           template: installAppsOnAdmin
         arguments:
           parameters: 
@@ -44,7 +44,7 @@ spec:
       # TODO: What does this name mean? Wait for CNI?
     - - name: ready-for-cni-and-csi
         templateRef:
-          name: tks-create-application
+          name: tks-create-application-new
           template: installApps
         arguments:
           parameters: 
diff --git a/tks-cluster/tks-createapp-wftp.yaml b/tks-cluster/tks-createapp-wftp.yaml
index 7853c8c8..320a9520 100755
--- a/tks-cluster/tks-createapp-wftp.yaml
+++ b/tks-cluster/tks-createapp-wftp.yaml
@@ -1,7 +1,7 @@
 apiVersion: argoproj.io/v1alpha1
 kind: WorkflowTemplate
 metadata:
-  name: tks-create-application
+  name: tks-create-application-new
   namespace: argo
 spec:
   arguments:
@@ -41,7 +41,7 @@ spec:
         ./argocd login $ARGO_SERVER --plaintext --insecure --username $ARGO_USERNAME \
         --password $ARGO_PASSWORD
 
-        REPO=https://$(echo $gittoken|xargs)@github.com/$git_account/${contract_id}-manifests.git
+        REPO=https://$(echo -n $gittoken)@github.com/$git_account/${contract_id}-manifests.git
 
         # check if app already exists.
         ./argocd app get $CD_APP