diff --git a/decapod-manifests/update-decapod-manifest-wftpl.yaml b/decapod-manifests/update-decapod-manifest-wftpl.yaml index 2d5996de..b1b1554f 100644 --- a/decapod-manifests/update-decapod-manifest-wftpl.yaml +++ b/decapod-manifests/update-decapod-manifest-wftpl.yaml @@ -42,7 +42,7 @@ spec: import json action = "{{inputs.parameters.action}}" - clusterName = "{{inputs.parameters.cluster_name}}" + clusterId = "{{inputs.parameters.cluster_name}}" appGroup = "{{inputs.parameters.app_group}}" chartName = "{{inputs.parameters.chart}}" kvMapStr = "{{inputs.parameters.kv_map_str}}" @@ -55,14 +55,19 @@ spec: numChanged = 0 siteDir = 'cluster-site' sitePath = "./{}".format(siteDir) - siteFileName = "{}/{}/site-values.yaml".format(clusterName,appGroup) + siteFileName = "{}/{}/site-values.yaml".format(clusterId,appGroup) siteFileNameFull = "{}/{}".format(sitePath, siteFileName) # Clone or re-use site repository # if not os.path.isdir(sitePath): print("Cloning repository...") - repoUrl = "https://" + os.environ['TOKEN'] + "@{{inputs.parameters.site_repo_url}}" + gitBaseUrl = os.environ['GIT_SVC_URL'].replace("http://","") + + #repoUrl = "https://" + os.environ['TOKEN'] + "@{{inputs.parameters.site_repo_url}}" + url = "@" + gitBaseUrl + "/" + os.environ['USERNAME'] + "/" + clusterId + repoUrl = "http://" + os.environ['TOKEN'] + url + try: repo = git.Repo.clone_from(repoUrl, siteDir) with repo.config_writer() as git_config: @@ -70,7 +75,7 @@ spec: git_config.set_value('user', 'name', 'TKS Argo') print("Cloned repo contents:") os.system("ls -al {}".format(sitePath)) - os.system("ls -al {}/{}".format(sitePath, clusterName)) + os.system("ls -al {}/{}".format(sitePath, clusterId)) except git.exc.GitCommandError as e: print("Repository {} doesn't exist. Skipping..".format(repoUrl)) print(e) @@ -94,7 +99,7 @@ spec: else: chartFound['override'][k] = v numChanged += 1 - commit_msg = "update items for {} chart in '{}' cluster".format(chartName, clusterName) + commit_msg = "update items for {} chart in '{}' cluster".format(chartName, clusterId) elif action == 'insert': for k,v in kvMap.items(): # If the value is list type, then append one by one iterating over the list. @@ -110,7 +115,7 @@ spec: else: chartFound['override'][k].append(v) numChanged += 1 - commit_msg = "insert items for {} chart in '{}' cluster".format(chartName, clusterName) + commit_msg = "insert items for {} chart in '{}' cluster".format(chartName, clusterId) # DeleteItemFromList elif action == 'delete': for k,v in kvMap.items(): @@ -120,7 +125,7 @@ spec: numChanged += 1 else: print("The value {} doesn't exist.".format(v)) - commit_msg = "delete items for {} chart in '{}' cluster".format(chartName, clusterName) + commit_msg = "delete items for {} chart in '{}' cluster".format(chartName, clusterId) else: sys.exit("Wrong action type") diff --git a/deploy_apps/tks-lma-federation-wftpl.yaml b/deploy_apps/tks-lma-federation-wftpl.yaml index 7d0806fb..137c885f 100644 --- a/deploy_apps/tks-lma-federation-wftpl.yaml +++ b/deploy_apps/tks-lma-federation-wftpl.yaml @@ -80,12 +80,10 @@ spec: template: updateManifest arguments: parameters: - - name: site_repo_url - value: "github.com/{{workflow.parameters.github_account}}/{{item.name}}" - name: action value: "insert" - name: cluster_name - value: "{{item.name}}" + value: "{{ workflow.parameters.cluster_id }}" - name: app_group value: "lma" - name: chart @@ -100,8 +98,6 @@ spec: template: updateManifest arguments: parameters: - - name: site_repo_url - value: "github.com/{{workflow.parameters.github_account}}/{{workflow.parameters.cluster_id}}" - name: action value: "insert" # TODO: modify this to reflect actual cluster name for all cases @@ -220,8 +216,7 @@ spec: script: name: 'collect' image: harbor-cicd.taco-cat.xyz/tks/centos-tks-api:v1.0 - command: - - python + command: ["python"] envFrom: - secretRef: name: "git-svc-token" @@ -233,18 +228,25 @@ spec: import os import git import requests + import json + + TKS_API_URL = "{{workflow.parameters.tks_info_host}}" + CLUSTER_ID = "{{workflow.parameters.cluster_id}}" + #CLUSTER_ID = "caldcde6u" + #CLUSTER_ID = "c6fk1w3dm" def getToken() : data = { + 'organizationId': 'master', 'accountId': 'admin', - 'password' : '1234' + 'password' : 'admin' } res = requests.post(TKS_API_URL+"/api/1.0/auth/login", json = data ) if res.status_code != 200 : return '' resJson = res.json() - return resJson['data']['user']['token'] + return resJson['user']['token'] output_cluster_list = [] temp_map = {} @@ -252,36 +254,46 @@ spec: inwards_endpoint_map = {} outwards_endpoint_map = {} + TOKEN=getToken() res = requests.get(TKS_API_URL+"/api/1.0/clusters/" + CLUSTER_ID, - headers={"Authorization": "Bearer " + getToken()} ) + headers={"Authorization": "Bearer " + TOKEN} ) if res.status_code != 200 : sys.exit('Failed to get cluster') - cluster = res.json()['data']['cluster'] + cluster = res.json()['cluster'] print( cluster ) - organizationId = res.cluster['organizationId'] - curClusterName = res.cluster['id'] + organizationId = cluster['organizationId'] + cur_cluster_name = cluster['id'] res = requests.get(TKS_API_URL+"/api/1.0/clusters?organizationId=" + organizationId, - headers={"Authorization": "Bearer " + getToken()} ) + headers={"Authorization": "Bearer " + TOKEN} ) if res.status_code != 200 : sys.exit('Failed to get clusters') + clusters = res.json()['clusters'] + print("Iterating over clusters in the same contract...") # Iterate over cluster list except current cluster # - for cluster in res.clusters: + for cluster in clusters: if cluster['status'] != "RUNNING": continue - if cluster['id'] != "{{workflow.parameters.cluster_id}}": + if cluster['id'] != CLUSTER_ID : print("*******************************************") - print("Checking cluster: {}".format(cluster.id)) + print("Checking cluster: {}".format(cluster['id'])) + + gitBaseUrl = os.environ['GIT_SVC_URL'].replace("http://","") + print( gitBaseUrl ) + print("Checking if corresponding cluster repo exists..") - url = "@github.com/{{workflow.parameters.github_account}}/{}".format(cluster['id']) - repoUrl = "https://" + os.environ['TOKEN'] + url + #url = "@github.com/{{workflow.parameters.github_account}}/{}".format(cluster['id']) + url = "@" + gitBaseUrl + "/" + os.environ['USERNAME'] + "/" + cluster['id'] + print( url ) + + repoUrl = "http://" + os.environ['TOKEN'] + url try: repo = git.Repo.clone_from(repoUrl, './tempcluster') @@ -290,106 +302,44 @@ spec: print("Repo {} doesn't exist. Skipping this cluster..".format(repoUrl)) continue - res = requests.get(TKS_API_URL+"/api/1.0/clusters?organizationId=" + organizationId, - headers={"Authorization": "Bearer " + getToken()} ) + res = requests.get(TKS_API_URL+"/api/1.0/app-groups?clusterId=" + cluster['id'], + headers={"Authorization": "Bearer " + TOKEN} ) if res.status_code != 200 : print( 'Failed to get appgroups for cluster ') continue - appGroups = res.json()['data']['appGroups'] + appGroups = res.json()['appGroups'] print( appGroups ) os.system("rm -rf ./tempcluster") # Check if LMA group exists. - for appGroup in iappGroups: - if appGroup['type'] == "LMA" : + for appGroup in appGroups: + if appGroup['appGroupType'] == "LMA" : print("Found LMA appGroup: {}".format(appGroup['name'])) - res = requests.get(TKS_API_URL+"/api/1.0/app-groups/" + appGroup['id'] + "/applications", - headers={"Authorization": "Bearer " + getToken()} ) + res = requests.get(TKS_API_URL+"/api/1.0/app-groups/" + appGroup['id'] + "/applications?applicationType=PROMETHEUS", + headers={"Authorization": "Bearer " + TOKEN} ) if res.status_code != 200 : print( 'Failed to get applications for appgroup') - continue + continue + applications = res.json()['applications'] - res = app_stub.GetApps(info_pb2.GetAppsRequest(app_group_id=app_group.app_group_id, type=common_pb2.PROMETHEUS)) - - if res.apps: + if applications : # This is based on the premise that there's only one prometheus per appGroup. - endpoint = res.apps[0].endpoint + endpoint = applications[0]['endpoint'] print("Get Thanos-sc endpoint: {}. Appending it to inward list.".format(endpoint)) # Add this cluster's endpoint to endpoint map inwards_endpoint_list.append(endpoint) # Add this cluster to outward list so that current ep is updated to this cluster - temp_map["name"] = cluster.id + temp_map["name"] = cluster['id'] str_json = json.dumps(temp_map) output_cluster_list.append(str_json) - with grpc.insecure_channel(addr) as channel: - cl_stub = info_pb2_grpc.ClusterInfoServiceStub(channel) - app_stub = info_pb2_grpc.AppInfoServiceStub(channel) - - res = cl_stub.GetCluster(info_pb2.GetClusterRequest(cluster_id="{{workflow.parameters.cluster_id}}")) - - print("Response from GetCluster: ") - print(res.cluster) - contract_id = res.cluster.contract_id - csp_id = res.cluster.csp_id - cur_cluster_name = res.cluster.id - - res = cl_stub.GetClusters(info_pb2.GetClustersRequest(contract_id=contract_id, csp_id='')) - - print("Iterating over clusters in the same contract...") - - # Iterate over cluster list except current cluster # - for cluster in res.clusters: - if cluster.status != common_pb2.RUNNING: - continue - - if cluster.id != "{{workflow.parameters.cluster_id}}": - print("*******************************************") - print("Checking cluster: {}".format(cluster.id)) - - print("Checking if corresponding cluster repo exists..") - url = "@github.com/{{workflow.parameters.github_account}}/{}".format(cluster.id) - repoUrl = "https://" + os.environ['TOKEN'] + url - try: - repo = git.Repo.clone_from(repoUrl, './tempcluster') - - except git.exc.GitCommandError as e: - print(e) - print("Repo {} doesn't exist. Skipping this cluster..".format(repoUrl)) - continue - - res = app_stub.GetAppGroupsByClusterID(common_pb2.IDRequest(id=cluster.id)) - print("Response from GetAppGroupsByClusterID:") - print(res.app_groups) - - os.system("rm -rf ./tempcluster") - - # Check if LMA group exists. - for app_group in res.app_groups: - if app_group.type == common_pb2.LMA: - print("Found LMA app_group: {}".format(app_group.app_group_name)) - res = app_stub.GetApps(info_pb2.GetAppsRequest(app_group_id=app_group.app_group_id, type=common_pb2.PROMETHEUS)) - - if res.apps: - # This is based on the premise that there's only one prometheus per appGroup. - endpoint = res.apps[0].endpoint - print("Get Thanos-sc endpoint: {}. Appending it to inward list.".format(endpoint)) - - # Add this cluster's endpoint to endpoint map - inwards_endpoint_list.append(endpoint) - - # Add this cluster to outward list so that current ep is updated to this cluster - temp_map["name"] = cluster.id - str_json = json.dumps(temp_map) - output_cluster_list.append(str_json) - # Compose profer format to be used as input on next step inwards_endpoint_map['querier.stores'] = inwards_endpoint_list diff --git a/git-repo/create-cluster-repo.yaml b/git-repo/create-cluster-repo.yaml index 1b41f18f..41a5ef06 100644 --- a/git-repo/create-cluster-repo.yaml +++ b/git-repo/create-cluster-repo.yaml @@ -90,7 +90,7 @@ spec: sed -i "s/mdMinSizePerAz:\ CHANGEME/mdMinSizePerAz: $val_min_size/g" $CLUSTER_ID/$CLUSTER_ID/tks-cluster/site-values.yaml sed -i "s/mdMaxSizePerAz:\ CHANGEME/mdMaxSizePerAz: $val_max_size/g" $CLUSTER_ID/$CLUSTER_ID/tks-cluster/site-values.yaml sed -i "s/mdMachineType:\ CHANGEME/mdMachineType: $val_machine_type/g" $CLUSTER_ID/$CLUSTER_ID/tks-cluster/site-values.yaml - sed -i "s/cpReplicas:\ CHANGEME/cpReplicas: 1/g" $CLUSTER_ID/$CLUSTER_ID/tks-cluster/site-values.yaml + sed -i "s/cpReplicas:\ CHANGEME/cpReplicas: 3/g" $CLUSTER_ID/$CLUSTER_ID/tks-cluster/site-values.yaml ;; byoh) diff --git a/git-repo/render-manifests.yaml b/git-repo/render-manifests.yaml index 2df56a46..5b27ed4d 100644 --- a/git-repo/render-manifests.yaml +++ b/git-repo/render-manifests.yaml @@ -72,7 +72,7 @@ spec: - name: https_enabled container: name: render-manifests-template - image: harbor-cicd.taco-cat.xyz/tks/decapod-render:v2.1.0 + image: harbor-cicd.taco-cat.xyz/tks/decapod-render:v3.1.2 command: - /bin/bash - '-exc' diff --git a/tks-cluster/remove-usercluster-wftpl.yaml b/tks-cluster/remove-usercluster-wftpl.yaml index 7facfb6a..b6d61ca9 100644 --- a/tks-cluster/remove-usercluster-wftpl.yaml +++ b/tks-cluster/remove-usercluster-wftpl.yaml @@ -271,6 +271,7 @@ spec: else echo "Error: installations.v1.operator.tigera.io 'default' was NOT found." exit 1 + fi kubectl $kube_params get installations.v1.operator.tigera.io | grep default if [[ $? =~ 0 ]]; then diff --git a/tks_info/update-tks-app-group-wftpl.yaml b/tks_info/update-tks-app-group-wftpl.yaml index d9ed874c..8d00d73e 100644 --- a/tks_info/update-tks-app-group-wftpl.yaml +++ b/tks_info/update-tks-app-group-wftpl.yaml @@ -32,7 +32,7 @@ spec: def getToken() : data = { - 'organizationId': 'master' + 'organizationId': 'master', 'accountId': 'admin', 'password' : 'admin' } @@ -41,7 +41,7 @@ spec: if res.status_code != 200 : return '' resJson = res.json() - return resJson['data']['user']['token'] + return resJson['user']['token'] print( ENDPOINTS ) for appType, endpoint in {{inputs.parameters.endpoints}}.items() :