Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Supports template configuration with different infrastructure providers #100

Merged
merged 4 commits into from
Jun 30, 2022
Merged

Supports template configuration with different infrastructure providers #100

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
47efd9e
use seperated infra provider sites
zugwan Jun 22, 2022
ce63d74
add workarounds for current TKS cli
zugwan Jun 24, 2022
b6e5621
add workaroud for current TKS cluster LCM service
zugwan Jun 29, 2022
543e6cd
use the first group of the cluster UUID as the cluster name
zugwan Jun 29, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions deploy_apps/tks-lma-federation-wftpl.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,9 @@ spec:
echo "[$date] $level $msg"
}

kube_secret=$(kubectl get secret -n {{workflow.parameters.cluster_id}} {{workflow.parameters.cluster_id}}-kubeconfig -o jsonpath="{.data.value}" | base64 -d)
CLUSTER_NAME={{=sprig.substr(0, 8, workflow.parameters.cluster_id)}}
kube_secret=$(kubectl get secret -n $CLUSTER_NAME $CLUSTER_NAME-kubeconfig -o jsonpath="{.data.value}" | base64 -d)

echo -e "kube_secret:\n$kube_secret" | head -n 5
cat <<< "$kube_secret" > /etc/kubeconfig

Expand Down Expand Up @@ -183,7 +185,8 @@ spec:
- /bin/bash
- '-c'
- |
kube_secret=$(kubectl get secret -n {{workflow.parameters.cluster_id}} {{workflow.parameters.cluster_id}}-kubeconfig -o jsonpath="{.data.value}" | base64 -d)
CLUSTER_NAME={{=sprig.substr(0, 8, workflow.parameters.cluster_id)}}
kube_secret=$(kubectl get secret -n $CLUSTER_NAME $CLUSTER_NAME-kubeconfig -o jsonpath="{.data.value}" | base64 -d)
echo -e "kube_secret:\n$kube_secret" | head -n 5
cat <<< "$kube_secret" > /etc/kubeconfig

Expand Down
3 changes: 2 additions & 1 deletion deploy_apps/tks-remove-lma-federation-wftpl.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,8 @@ spec:
kube_params=""

if [[ -n "{{workflow.parameters.cluster_id}}" ]]; then
kube_secret=$(kubectl get secret -n {{workflow.parameters.cluster_id}} {{workflow.parameters.cluster_id}}-kubeconfig -o jsonpath="{.data.value}" | base64 -d)
CLUSTER_NAME={{=sprig.substr(0, 8, workflow.parameters.cluster_id)}}
kube_secret=$(kubectl get secret -n $CLUSTER_NAME $CLUSTER_NAME-kubeconfig -o jsonpath="{.data.value}" | base64 -d)
echo -e "kube_secret:\n$kube_secret" | head -n 5
cat <<< "$kube_secret" > /etc/kubeconfig

Expand Down
3 changes: 2 additions & 1 deletion deploy_apps/tks-remove-servicemesh-wftpl.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,8 @@ spec:

kube_params=""
if [[ -n "{{workflow.parameters.cluster_id}}" ]]; then
kube_secret=$(kubectl get secret -n {{workflow.parameters.cluster_id}} {{workflow.parameters.cluster_id}}-kubeconfig -o jsonpath="{.data.value}" | base64 -d)
CLUSTER_NAME={{=sprig.substr(0, 8, workflow.parameters.cluster_id)}}
kube_secret=$(kubectl get secret -n $CLUSTER_NAME $CLUSTER_NAME-kubeconfig -o jsonpath="{.data.value}" | base64 -d)
echo -e "kube_secret:\n$kube_secret" | head -n 5
cat <<< "$kube_secret" > /etc/kubeconfig
kube_params+="--kubeconfig=/etc/kubeconfig"
Expand Down
15 changes: 8 additions & 7 deletions deploy_apps/tks-service-mesh-dashboard-wftpl.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -828,7 +828,8 @@ spec:

kube_params=""
if [[ -n "{{workflow.parameters.cluster_id}}" ]]; then
kube_secret=$(kubectl get secret -n {{workflow.parameters.cluster_id}} {{workflow.parameters.cluster_id}}-kubeconfig -o jsonpath="{.data.value}" | base64 -d)
CLUSTER_NAME={{=sprig.substr(0, 8, workflow.parameters.cluster_id)}}
kube_secret=$(kubectl get secret -n $CLUSTER_NAME $CLUSTER_NAME-kubeconfig -o jsonpath="{.data.value}" | base64 -d)
echo -e "kube_secret:\n$kube_secret" | head -n 5
cat <<< "$kube_secret" > /tmp/kubeconfig
kube_params+="--kubeconfig=/tmp/kubeconfig"
Expand Down Expand Up @@ -936,7 +937,8 @@ spec:

kube_params=""
if [[ -n "{{workflow.parameters.cluster_id}}" ]]; then
kube_secret=$(kubectl get secret -n {{workflow.parameters.cluster_id}} {{workflow.parameters.cluster_id}}-kubeconfig -o jsonpath="{.data.value}" | base64 -d)
CLUSTER_NAME={{=sprig.substr(0, 8, workflow.parameters.cluster_id)}}
kube_secret=$(kubectl get secret -n $CLUSTER_NAME $CLUSTER_NAME-kubeconfig -o jsonpath="{.data.value}" | base64 -d)
echo -e "kube_secret:\n$kube_secret" | head -n 5
cat <<< "$kube_secret" > /tmp/kubeconfig
kube_params+="--kubeconfig=/tmp/kubeconfig"
Expand Down Expand Up @@ -1019,7 +1021,8 @@ spec:

kube_params=""
if [[ -n "{{workflow.parameters.cluster_id}}" ]]; then
kube_secret=$(kubectl get secret -n {{workflow.parameters.cluster_id}} {{workflow.parameters.cluster_id}}-kubeconfig -o jsonpath="{.data.value}" | base64 -d)
CLUSTER_NAME={{=sprig.substr(0, 8, workflow.parameters.cluster_id)}}
kube_secret=$(kubectl get secret -n $CLUSTER_NAME $CLUSTER_NAME-kubeconfig -o jsonpath="{.data.value}" | base64 -d)
echo -e "kube_secret:\n$kube_secret" | head -n 5
cat <<< "$kube_secret" > /tmp/kubeconfig
kube_params+="--kubeconfig=/tmp/kubeconfig"
Expand Down Expand Up @@ -1149,7 +1152,8 @@ spec:
- |
mkdir ~/.kube
cp /kube/value ~/.kube/config_adm
KUBECONFIG_USERCLUSTER=$(kubectl --kubeconfig ~/.kube/config_adm get secret -n ${CLUSTER_ID} ${CLUSTER_ID}-kubeconfig -o=jsonpath='{.data.value}' | base64 -d)
USER_CLUSTER_NAME={{=sprig.substr(0, 8, workflow.parameters.cluster_id)}}
KUBECONFIG_USERCLUSTER=$(kubectl --kubeconfig ~/.kube/config_adm get secret -n $CLUSTER_NAME $CLUSTER_NAME-kubeconfig -o jsonpath="{.data.value}" | base64 -d)
echo -e "kubeconfig_workload:\n$KUBECONFIG_USERCLUSTER" | head -n 5
cat <<< "$KUBECONFIG_USERCLUSTER" > ~/.kube/config_user

Expand Down Expand Up @@ -1203,9 +1207,6 @@ spec:
volumeMounts:
- name: kubeconfig-adm
mountPath: "/kube"
env:
- name: CLUSTER_ID
value: "{{workflow.parameters.cluster_id}}"
activeDeadlineSeconds: 900
retryStrategy:
limit: 2
Expand Down
42 changes: 33 additions & 9 deletions github_repo/create-cluster-repo.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ spec:
value: "cluster_uuid"
- name: template_name
value: "template-std"
- name: infra_provider
value: "aws"
- name: test_template_name
value: "aws-reference"

templates:
- name: createClusterRepo
Expand All @@ -37,13 +37,21 @@ spec:

git clone https://$(echo -n $TOKEN)@github.com/${USERNAME}/${CONTRACT_ID}.git

# Get cluster-api infra provider in the template
infra_provider_group="$(ls ${CONTRACT_ID}/$TEMPLATE_NAME | grep -v tks-cluster-common | grep tks-cluster)"
INFRA_PROVIDER=${infra_provider_group#tks-cluster-}
echo ${INFRA_PROVIDER} | tee /mnt/out/infra_provider.txt

# We se only the first UUID group for the cluster name and namespace.
CLUSTER_NAME=${CLUSTER_ID%%-*}

cp -r ${CONTRACT_ID}/${TEMPLATE_NAME} ${CLUSTER_ID}/${CLUSTER_ID}
cp -r ${CONTRACT_ID}/.github ${CLUSTER_ID}/.github
cp -r ${CONTRACT_ID}/_github ${CLUSTER_ID}/.github

echo $CLUSTER_INFO

## Replace site-values with fetched params ##
sed -i "s/clusterName:\ cluster.local/clusterName:\ $CLUSTER_ID/g" $CLUSTER_ID/$CLUSTER_ID/tks-cluster-common/site-values.yaml
sed -i "s/clusterName:\ cluster.local/clusterName:\ $CLUSTER_NAME/g" $CLUSTER_ID/$CLUSTER_ID/tks-cluster-common/site-values.yaml
Copy link
Contributor

@ktkfree ktkfree Jun 30, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

이 PR 에 대한 내용은 아닙니다.

clusterName 이
. 고객이 생성할때 입력하는 휴먼리더블 이름
. 지금 여기서 사용하는 프리픽스된 clusterId
. 어떤 곳에서는 full clusterId 사용
로 다양하게 사용되는것 같아 오개발할 위험성이 있어 보입니다.

애초에 ClusterId 를 생성하는 주체는 tks시스템이므로 발급시 unique 함만 보장하면 될 것 같습니다.
clusterId, contractId 등을 uuid 가 아닌 unique 가 보장되는 "8자리 uuid" 를 clusterId 로 사용하면 어떨까요? 위험한 생각일까요?

정리하면
. clusterName : 고객이 입력하는 human-readable 이름
. clusterId : 8자리 Cprefixed_uuid (유니크가 보장되는)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

저도 일단 기록을 남기는 용도로 기억을 더듬어 보면 clusterName은 human-readable 한 것 외에는 실제 TKS에서 다른 용도로 사용하지 않는 값으로 정의를 했었고 clusterId로 모두 관리하자는 게 의도였습니다. ID 형식을 무엇으로 할 것인가 얘기가 나왔을 때에는 uuid가 가장 무난하기 때문에 선정이 되었습니다.

case $INFRA_PROVIDER in
aws)
## Fetch cluster params from cluster_info file ##
Expand All @@ -56,7 +64,7 @@ spec:
val_max_size=$(echo $CLUSTER_INFO | sed 's/.*\(max_size_per_az:\ \S*\).*/\1/' | cut -d ':' -f2 | xargs)
echo "max_size_per_az: $val_max_size"

sed -i "s/clusterName:\ cluster.local/clusterName:\ $CLUSTER_ID/g" $CLUSTER_ID/$CLUSTER_ID/tks-cluster-aws/site-values.yaml
sed -i "s/clusterName:\ cluster.local/clusterName:\ $CLUSTER_NAME/g" $CLUSTER_ID/$CLUSTER_ID/tks-cluster-aws/site-values.yaml
sed -i "s/sshKeyName:\ CHANGEME/sshKeyName: $val_ssh_key/g" $CLUSTER_ID/$CLUSTER_ID/tks-cluster-aws/site-values.yaml
sed -i "s/clusterRegion:\ CHANGEME/clusterRegion: $val_region/g" $CLUSTER_ID/$CLUSTER_ID/tks-cluster-aws/site-values.yaml
sed -i "s/mdNumOfAz:\ CHANGEME/mdNumOfAz: $val_num_of_az/g" $CLUSTER_ID/$CLUSTER_ID/tks-cluster-aws/site-values.yaml
Expand All @@ -66,7 +74,7 @@ spec:
;;

byoh)
sed -i "s/clusterName:\ cluster.local/clusterName:\ $CLUSTER_ID/g" $CLUSTER_ID/$CLUSTER_ID/tks-cluster-byoh/site-values.yaml
sed -i "s/clusterName:\ cluster.local/clusterName:\ $CLUSTER_NAME/g" $CLUSTER_ID/$CLUSTER_ID/tks-cluster-byoh/site-values.yaml
echo "BYOH"
;;

Expand Down Expand Up @@ -119,11 +127,21 @@ spec:
- name: CLUSTER_ID
value: "{{workflow.parameters.cluster_id}}"
- name: TEMPLATE_NAME
value: "{{workflow.parameters.template_name}}"
- name: INFRA_PROVIDER
value: "{{workflow.parameters.infra_provider}}"
value: "{{workflow.parameters.test_template_name}}"
- name: CLUSTER_INFO
value: "{{inputs.parameters.cluster_info}}"
volumeMounts:
- name: out
mountPath: /mnt/out
volumes:
- name: out
emptyDir: { }
outputs:
parameters:
- name: infra_provider
valueFrom:
default: "Something wrong"
path: /mnt/out/infra_provider.txt

- name: createRepoCredential
activeDeadlineSeconds: 120
Expand Down Expand Up @@ -165,3 +183,9 @@ spec:
- - name: createRepoCredential
template: createRepoCredential
arguments: {}

outputs:
parameters:
- name: infra_provider
valueFrom:
parameter: "{{steps.createClusterRepo.outputs.parameters.infra_provider}}"
9 changes: 7 additions & 2 deletions github_repo/create-contract-repo.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,14 +37,19 @@ spec:
git clone -b ${REVISION} https://github.com/openinfradev/decapod-site

cd decapod-site

mv .github _github

# TODO: support to use decapod-reference-offline later?
mv decapod-reference template-std
rm -rf decapod-reference
rm -rf decapod-reference-offline

zugwan marked this conversation as resolved.
Show resolved Hide resolved
# Remove unnecessary app_group directory before commit.
# If these kinds of apps increases, then they might be defined as black list
# and then removed by FOR loop iteration.
# For now, this hardcoding seems enough.
rm -rf template-std/openstack template-std/decapod-controller
rm -rf aws-reference/openstack aws-reference/decapod-controller aws-reference/admin-tools
rm -rf byoh-reference/openstack byoh-reference/decapod-controller byoh-reference/admin-tools

sed -i "s/BRANCH=\"main\"/BRANCH=\"${REVISION}\"/g" .github/workflows/render-cd.sh

Expand Down
6 changes: 4 additions & 2 deletions tests/validate-service-wftpl.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@ spec:
- /bin/bash
- -c
- |
KUBECONFIG_=$(kubectl get secret -n {{inputs.parameters.cluster_id}} {{inputs.parameters.cluster_id}}-kubeconfig -o jsonpath="{.data.value}" | base64 -d)
CLUSTER_NAME={{=sprig.substr(0, 8, workflow.parameters.cluster_id)}}
KUBECONFIG_=$(kubectl get secret -n $CLUSTER_NAME $CLUSTER_NAME-kubeconfig -o jsonpath="{.data.value}" | base64 -d)
cat <<< "$KUBECONFIG_" > /etc/kubeconfig_temp
export KUBECONFIG='/etc/kubeconfig_temp'

Expand Down Expand Up @@ -80,7 +81,8 @@ spec:
- /bin/bash
- -c
- |
KUBECONFIG_=$(kubectl get secret -n {{inputs.parameters.cluster_id}} {{inputs.parameters.cluster_id}}-kubeconfig -o jsonpath="{.data.value}" | base64 -d)
CLUSTER_NAME={{=sprig.substr(0, 8, workflow.parameters.cluster_id)}}
KUBECONFIG_=$(kubectl get secret -n $CLUSTER_NAME $CLUSTER_NAME-kubeconfig -o jsonpath="{.data.value}" | base64 -d)
cat <<< "$KUBECONFIG_" > /etc/kubeconfig_temp
export KUBECONFIG='/etc/kubeconfig_temp'

Expand Down
4 changes: 3 additions & 1 deletion tests/validate-usercluster-wftpl.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,9 @@ spec:
- /bin/bash
- '-exc'
- |
KUBECONFIG_=$(kubectl get secret -n {{inputs.parameters.cluster_id}} {{inputs.parameters.cluster_id}}-kubeconfig -o jsonpath="{.data.value}" | base64 -d)
CLUSTER_NAME={{=sprig.substr(0, 8, workflow.parameters.cluster_id)}}
KUBECONFIG_=$(kubectl get secret -n $CLUSTER_NAME $CLUSTER_NAME-kubeconfig -o jsonpath="{.data.value}" | base64 -d)

cat <<< "$KUBECONFIG_" > /etc/kubeconfig_temp
export KUBECONFIG='/etc/kubeconfig_temp'

Expand Down
Loading