diff --git a/deploy_apps/tks-lma-federation-wftpl.yaml b/deploy_apps/tks-lma-federation-wftpl.yaml index 27b1dc0a..bc2f84b2 100644 --- a/deploy_apps/tks-lma-federation-wftpl.yaml +++ b/deploy_apps/tks-lma-federation-wftpl.yaml @@ -159,16 +159,15 @@ spec: - name: app_type value: GRAFANA - - - name: render-auth-oidc-grafana + - - name: wait-for-rendering-to-finish templateRef: - name: event-gitea-render-manifests + name: wait-for-rendering-to-finish template: main arguments: parameters: - - name: decapod_site_repo - value: "{{workflow.parameters.github_account}}/{{workflow.parameters.cluster_id}}" - - name: base_repo_branch - value: "{{ workflow.parameters.base_repo_branch }}" + - name: cluster_id + value: "{{ workflow.parameters.github_account }}/{{workflow.parameters.cluster_id}}" + when: "{{steps.update-auth-oidc-grafana.outputs.parameters.is_changed}} == YES" - - name: argocd-sync-wait template: argocd-sync-wait @@ -418,20 +417,35 @@ spec: yq -i e ".global.grafanaClientSecret=\"${keycloak_client_secret}\"" ${cluster_id}/lma/site-values.yaml yq -i e ".global.consoleUrl=\"${console_url}\"" ${cluster_id}/lma/site-values.yaml - git config --global user.name "tks" - git config --global user.email "tks@sktelecom.com" + if [[ `git status --porcelain` ]]; then + git config --global user.name "tks" + git config --global user.email "tks@sktelecom.com" - log "INFO" "##### commit changes grafana domain and root_url on ${cluster_id} to ${grafana_endpoint} and ${grafana_endpoint}/grafana" - cmessage="changes grafana domain and root_url on ${cluster_id} to ${grafana_endpoint} and ${grafana_endpoint}/grafana" - git add ${cluster_id}/lma/site-values.yaml - git commit -m "change values on grafana.ini.server." -m "$cmessage" - git push + log "INFO" "##### commit changes grafana domain and root_url on ${cluster_id} to ${grafana_endpoint} and ${grafana_endpoint}/grafana" + cmessage="changes grafana domain and root_url on ${cluster_id} to ${grafana_endpoint} and ${grafana_endpoint}/grafana" + git add ${cluster_id}/lma/site-values.yaml + git commit -m "change values on grafana.ini.server." -m "$cmessage" + git push + echo "YES" > /mnt/out/changed.txt + fi envFrom: - secretRef: name: "git-svc-token" - secretRef: name: "tks-api-secret" + volumeMounts: + - name: out + mountPath: /mnt/out + volumes: + - name: out + emptyDir: {} + outputs: + parameters: + - name: is_changed + valueFrom: + path: /mnt/out/changed.txt + default: "NO" - name: argocd-sync-wait inputs: diff --git a/deploy_apps/tks-primary-cluster.yaml b/deploy_apps/tks-primary-cluster.yaml index f269d577..c9eb20fa 100644 --- a/deploy_apps/tks-primary-cluster.yaml +++ b/deploy_apps/tks-primary-cluster.yaml @@ -22,6 +22,10 @@ spec: value: "decapod10" - name: object_store value: "s3" + - name: alert_tks + value: "NA" + - name: alert_slack + value: "NA" ########################## # For tks-info task # @@ -170,16 +174,14 @@ spec: - name: member_clusters value: '{{inputs.parameters.member_clusters}}' - - - name: render-modified-clusters + - - name: wait-for-rendering-to-finish-modified-cluster templateRef: - name: event-gitea-render-manifests + name: wait-for-rendering-to-finish template: main arguments: parameters: - - name: decapod_site_repo - value: "{{ workflow.parameters.github_account }}/{{item}}" - - name: base_repo_branch - value: "{{ workflow.parameters.base_repo_branch }}" + - name: cluster_id + value: "{{ workflow.parameters.github_account }}/{{ item }}" withParam: "{{ steps.change-target.outputs.parameters.modified_cluster_list}}" - name: loki-use-s3 @@ -200,16 +202,14 @@ spec: - name: member_clusters value: '{{inputs.parameters.member_clusters}}' - - - name: render-pre-modified-clusters + - - name: wait-for-rendering-to-finish-pre-modified-cluster templateRef: - name: event-gitea-render-manifests + name: wait-for-rendering-to-finish template: main arguments: parameters: - - name: decapod_site_repo - value: "{{ workflow.parameters.github_account }}/{{item}}" - - name: base_repo_branch - value: "{{ workflow.parameters.base_repo_branch }}" + - name: cluster_id + value: "{{ workflow.parameters.github_account }}/{{ item }}" withParam: "{{ steps.pre-change-target.outputs.parameters.modified_cluster_list}}" - - name: federation-components-preinstall-for-s3 @@ -237,20 +237,15 @@ spec: parameters: - name: primary_cluster value: '{{inputs.parameters.primary_cluster}}' - - name: member_clusters - value: '{{inputs.parameters.member_clusters}}' - - - name: render-current-cluster + - - name: wait-for-rendering-to-finish-this-cluster templateRef: - name: event-gitea-render-manifests + name: wait-for-rendering-to-finish template: main arguments: parameters: - - name: decapod_site_repo + - name: cluster_id value: "{{ workflow.parameters.github_account }}/{{ workflow.parameters.cluster_id }}" - - name: base_repo_branch - value: "{{ workflow.parameters.base_repo_branch }}" - when: "'{{steps.change-thanos-sidecar.outputs.parameters.changed}}' != 'NO_CHANGE_HERE'" # 이미 변경내역이 반영된 (한번 수행됐던) 클러스터라면 랜더링은 필요없음 - - name: sync-organization-changes template: sub-sync-organization-changes @@ -261,17 +256,15 @@ spec: - name: member_clusters value: '{{inputs.parameters.member_clusters}}' - - - name: render-primary-cluster + - - name: wait-for-rendering-to-finish-changed-clusters templateRef: - name: event-gitea-render-manifests + name: wait-for-rendering-to-finish template: main arguments: parameters: - - name: decapod_site_repo - value: "{{ workflow.parameters.github_account }}/{{steps.sync-organization-changes.outputs.parameters.changed}}" - - name: base_repo_branch - value: "{{ workflow.parameters.base_repo_branch }}" - when: "'{{steps.sync-organization-changes.outputs.parameters.changed}}' != 'NO_CHANGE_HERE'" + - name: cluster_id + value: "{{ workflow.parameters.github_account }}/{{steps.sync-organization-changes.outputs.parameters.changed_primary_id}}" + when: "{{steps.sync-organization-changes.outputs.parameters.changed_primary_id}} != NONE" ####################### # Template Definition # @@ -364,6 +357,7 @@ spec: yq -i e ".charts |= map(select(.name == \"thanos-config\").override.objectStorage.rawConfig.region=\"ap-northeast-2\")" ${member}/lma/site-values.yaml yq -i e ".charts |= map(select(.name == \"thanos-config\").override.objectStorage.rawConfig.bucket=\"${primary_cluster}-tks-thanos\")" ${member}/lma/site-values.yaml yq -i e ".charts |= map(select(.name == \"thanos-config\").override.objectStorage.rawConfig.signature_version2=false)" ${member}/lma/site-values.yaml + yq -i e ".charts |= map(select(.name == \"thanos-config\").override.objectStorage.rawConfig.aws_sdk_auth=true)" ${member}/lma/site-values.yaml yq -i e ".global.clusterName=\"${member}\"" ${member}/lma/site-values.yaml yq -i e "del(.charts[] | select(.name == \"loki\").override.loki.storageConfig.aws)" ${member}/lma/site-values.yaml @@ -591,7 +585,6 @@ spec: inputs: parameters: - name: primary_cluster - - name: member_clusters container: name: logging-target-changer image: harbor.taco-cat.xyz/tks/shyaml_jq_yq_kubectl_python:3.11 @@ -610,68 +603,55 @@ spec: echo "[$date] $level $msg" } + cp /kube/value ~/kubeconfig_adm + export KUBECONFIG=~/kubeconfig_adm + current_cluster={{workflow.parameters.cluster_id}} primary_cluster={{inputs.parameters.primary_cluster}} - member_clusters="{{inputs.parameters.member_clusters}}" empty_char= if [ -z ${primary_cluster} ] || [[ "${primary_cluster}" == "$empty_char" ]]; then primary_cluster=${current_cluster} fi - primary_kube_secret=$(kubectl get secret -n ${primary_cluster} ${primary_cluster}-tks-kubeconfig -o jsonpath="{.data.value}" | base64 -d) - # echo -e "primary_kube_secret:\n$primary_kube_secret" | head -n 5 - cat <<< "$primary_kube_secret" > kubeconfig S3_SERVICE=$(kubectl get secret -n ${primary_cluster} tks-endpoint-secret -o jsonpath='{.data.minio}'| base64 -d ) - if [[ "$S3_SERVICE" == "" ]]; then - - S3_SERVICE="s3://ap-northeast-2" - cp /kube/value ~/kubeconfig_adm - export KUBECONFIG=~/kubeconfig_adm + if [[ "$S3_SERVICE" != "" ]]; then + echo "This site uses the predefined loki and static object stores." + exit 0 + fi - ################# - # updates - ################# - GIT_ACCOUNT={{workflow.parameters.github_account}} - if [[ $GIT_SVC_URL == https://* ]]; then - repository_base=https://${TOKEN//[$'\t\r\n ']}@${GIT_SVC_URL/http:\/\//}/${GIT_ACCOUNT}/ - else - repository_base=http://${TOKEN//[$'\t\r\n ']}@${GIT_SVC_URL/http:\/\//}/${GIT_ACCOUNT}/ - fi + S3_SERVICE="s3://ap-northeast-2" - log "INFO" "##### change the loki target to $LOKI_HOST:$LOKI_PORT and $S3_SERVICE (the current target is ${current_cluster})" - [ -d ${current_cluster} ] || git clone ${repository_base}${current_cluster} - cd ${current_cluster} + GIT_ACCOUNT={{workflow.parameters.github_account}} + if [[ $GIT_SVC_URL == https://* ]]; then + repository_base=https://${TOKEN//[$'\t\r\n ']}@${GIT_SVC_URL/http:\/\//}/${GIT_ACCOUNT}/ + else + repository_base=http://${TOKEN//[$'\t\r\n ']}@${GIT_SVC_URL/http:\/\//}/${GIT_ACCOUNT}/ + fi - yq -i e "del(.charts[] | select(.name == \"thanos-config\").override.objectStorage)" ${current_cluster}/lma/site-values.yaml - yq -i e ".charts |= map(select(.name == \"thanos-config\").override.objectStorage.type=\"s3\")" ${current_cluster}/lma/site-values.yaml - yq -i e ".charts |= map(select(.name == \"thanos-config\").override.objectStorage.rawConfig.endpoint=\"s3.ap-northeast-2.amazonaws.com\")" ${current_cluster}/lma/site-values.yaml - yq -i e ".charts |= map(select(.name == \"thanos-config\").override.objectStorage.rawConfig.region=\"ap-northeast-2\")" ${current_cluster}/lma/site-values.yaml - yq -i e ".charts |= map(select(.name == \"thanos-config\").override.objectStorage.rawConfig.bucket=\"${primary_cluster}-tks-thanos\")" ${current_cluster}/lma/site-values.yaml - yq -i e ".charts |= map(select(.name == \"thanos-config\").override.objectStorage.rawConfig.signature_version2=false)" ${current_cluster}/lma/site-values.yaml + log "INFO" "##### configure thanos object storage (the current target is ${current_cluster})" + [ -d ${current_cluster} ] || git clone ${repository_base}${current_cluster} + cd ${current_cluster} - git config --global user.name "tks" - git config --global user.email "tks@sktelecom.com" + yq -i e "del(.charts[] | select(.name == \"thanos-config\").override.objectStorage)" ${current_cluster}/lma/site-values.yaml + yq -i e ".charts |= map(select(.name == \"thanos-config\").override.objectStorage.type=\"s3\")" ${current_cluster}/lma/site-values.yaml + yq -i e ".charts |= map(select(.name == \"thanos-config\").override.objectStorage.rawConfig.endpoint=\"s3.ap-northeast-2.amazonaws.com\")" ${current_cluster}/lma/site-values.yaml + yq -i e ".charts |= map(select(.name == \"thanos-config\").override.objectStorage.rawConfig.region=\"ap-northeast-2\")" ${current_cluster}/lma/site-values.yaml + yq -i e ".charts |= map(select(.name == \"thanos-config\").override.objectStorage.rawConfig.bucket=\"${primary_cluster}-tks-thanos\")" ${current_cluster}/lma/site-values.yaml + yq -i e ".charts |= map(select(.name == \"thanos-config\").override.objectStorage.rawConfig.signature_version2=false)" ${current_cluster}/lma/site-values.yaml + yq -i e ".charts |= map(select(.name == \"thanos-config\").override.objectStorage.rawConfig.aws_sdk_auth=true)" ${current_cluster}/lma/site-values.yaml - if [[ `git status --porcelain` ]]; then - log "INFO" "##### commit changes on ${current_cluster} to use s3" - cmessage="changes on ${current_cluster} to use s3" - git add ${current_cluster}/lma/site-values.yaml - git commit -m "change loki and thanos endpoints. (by set-primary workflow)" -m "$cmessage" - git push - modified_clusters=${current_cluster} - # echo -n "${current_cluster} " >> /mnt/out/modified_cluster_list.txt - else - log "INFO" "No change on the cluster ${current_cluster}" - echo NO_CHANGE_HERE > /mnt/out/modified_cluster_list.txt - fi - cd - - rm -rf ${current_cluster} + git config --global user.name "tks" + git config --global user.email "tks@sktelecom.com" - jq -n '$ARGS.positional' --args $modified_clusters > /mnt/out/modified_cluster_list.txt + if [[ `git status --porcelain` ]]; then + log "INFO" "##### commit changes on ${current_cluster} to use s3" + cmessage="changes on ${current_cluster} to use s3" + git add ${current_cluster}/lma/site-values.yaml + git commit -m "change loki and thanos endpoints. (by set-primary workflow)" -m "$cmessage" + git push else - echo "This site uses the predefined loki and static object stores." - echo NO_CHANGE_HERE > /mnt/out/modified_cluster_list.txt + log "INFO" "No change on the cluster ${current_cluster}" fi env: @@ -683,16 +663,7 @@ spec: volumeMounts: - name: kubeconfig-adm mountPath: "/kube" - - name: out - mountPath: /mnt/out volumes: - - name: out - emptyDir: {} - outputs: - parameters: - - name: changed - valueFrom: - path: /mnt/out/modified_cluster_list.txt activeDeadlineSeconds: 900 - name: sub-sync-organization-changes @@ -818,7 +789,6 @@ spec: echo ${primary_cluster} > /mnt/out/changed.txt else log "INFO" "No change on the cluster ${primary_cluster}" - echo NO_CHANGE_HERE > /mnt/out/changed.txt fi if [[ "$OBJECT_STORE" == "s3" ]]; then @@ -859,9 +829,10 @@ spec: emptyDir: {} outputs: parameters: - - name: changed + - name: changed_primary_id valueFrom: path: /mnt/out/changed.txt + default: "NONE" activeDeadlineSeconds: 900 - name: sub-remove-individual-loki-and-grafana diff --git a/git-repo/event-gitea-render-manifests.yaml b/git-repo/event-gitea-render-manifests.yaml index c4023005..34ac9bbd 100644 --- a/git-repo/event-gitea-render-manifests.yaml +++ b/git-repo/event-gitea-render-manifests.yaml @@ -11,6 +11,8 @@ spec: value: "org/cluster_id" - name: base_repo_branch value: "" + ttlStrategy: + secondsAfterSuccess: 5 templates: - name: main diff --git a/git-repo/render-manifests.yaml b/git-repo/render-manifests.yaml index 8bcb5fc0..7338e661 100644 --- a/git-repo/render-manifests.yaml +++ b/git-repo/render-manifests.yaml @@ -72,7 +72,7 @@ spec: - name: https_enabled container: name: render-manifests-template - image: harbor.taco-cat.xyz/tks/decapod-render:v3.2.0 + image: harbor.taco-cat.xyz/tks/decapod-render:v3.3.0 command: - /bin/bash - '-exc' diff --git a/git-repo/wait-for-redering-to-finish.yaml b/git-repo/wait-for-redering-to-finish.yaml new file mode 100644 index 00000000..8584289d --- /dev/null +++ b/git-repo/wait-for-redering-to-finish.yaml @@ -0,0 +1,81 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: wait-for-rendering-to-finish + namespace: argo +spec: + entrypoint: main + arguments: + parameters: + - name: cluster_id + value: "c011b88fa" + volumes: + - name: kubeconfig-adm + secret: + secretName: tks-admin-kubeconfig-secret + ttlStrategy: + secondsAfterSuccess: 5 + + templates: + #========================================================= + # Template Pipeline + #========================================================= + - name: main + steps: + - - name: wait-for-rendering-to-finish + template: wait-for-rendering + arguments: + parameters: + - name: cluster_id + value: "{{workflow.parameters.cluster_id}}" + + #========================================================= + # Template Definition + #========================================================= + - name: wait-for-rendering + inputs: + parameters: + - name: cluster_id + container: + name: wait + image: harbor.taco-cat.xyz/tks/python_kubectl_argo:v1.1.0 + command: + - /bin/bash + - '-exc' + - | + #!/bin/bash + + function log() { + level=$2 + msg=$3 + date=$(date '+%F %H:%M:%S') + if [ $1 -eq 0 ];then + echo "[$date] $level $msg" + else + level="ERROR" + echo "[$date] $level $msg failed" + exit $1 + fi + } + + cp /kube/value kubeconfig_adm + export KUBECONFIG=kubeconfig_adm + + TARGET_SITE_REPO="$USERNAME/{{inputs.parameters.cluster_id}}" + + for wf in $(kubectl get workflows -n argo -l workflows.argoproj.io/workflow-template=event-gitea-render-manifests -oname); do + wf_site_repo=$(kubectl get -n argo $wf -ojsonpath='{.spec.arguments.parameters[0].value}') + if [[ "$TARGET_SITE_REPO" == "$wf_site_repo" ]]; then + kubectl wait --for=condition=Completed -n argo $wf --timeout=600s || true + fi + done + + volumeMounts: + - name: kubeconfig-adm + mountPath: "/kube" + envFrom: + - secretRef: + name: "git-svc-token" + activeDeadlineSeconds: 900 + retryStrategy: + limit: 2