Skip to content

Commit

Permalink
Merge branch 'tks-api-migration' of https://github.com/openinfradev/t…
Browse files Browse the repository at this point in the history 
…ks-flow into tks-api-migration
  • Loading branch information
Your Name committed Mar 30, 2023
2 parents 2f90607 + 70d35cf commit 485c13a
Show file tree
Hide file tree
Showing 6 changed files with 60 additions and 104 deletions.
19 changes: 12 additions & 7 deletions decapod-manifests/update-decapod-manifest-wftpl.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ spec:
import json
action = "{{inputs.parameters.action}}"
clusterName = "{{inputs.parameters.cluster_name}}"
clusterId = "{{inputs.parameters.cluster_name}}"
appGroup = "{{inputs.parameters.app_group}}"
chartName = "{{inputs.parameters.chart}}"
kvMapStr = "{{inputs.parameters.kv_map_str}}"
Expand All @@ -55,22 +55,27 @@ spec:
numChanged = 0
siteDir = 'cluster-site'
sitePath = "./{}".format(siteDir)
siteFileName = "{}/{}/site-values.yaml".format(clusterName,appGroup)
siteFileName = "{}/{}/site-values.yaml".format(clusterId,appGroup)
siteFileNameFull = "{}/{}".format(sitePath, siteFileName)
# Clone or re-use site repository #
if not os.path.isdir(sitePath):
print("Cloning repository...")
repoUrl = "https://" + os.environ['TOKEN'] + "@{{inputs.parameters.site_repo_url}}"
gitBaseUrl = os.environ['GIT_SVC_URL'].replace("http://","")
#repoUrl = "https://" + os.environ['TOKEN'] + "@{{inputs.parameters.site_repo_url}}"
url = "@" + gitBaseUrl + "/" + os.environ['USERNAME'] + "/" + clusterId
repoUrl = "http://" + os.environ['TOKEN'] + url
try:
repo = git.Repo.clone_from(repoUrl, siteDir)
with repo.config_writer() as git_config:
git_config.set_value('user', 'email', '[email protected]')
git_config.set_value('user', 'name', 'TKS Argo')
print("Cloned repo contents:")
os.system("ls -al {}".format(sitePath))
os.system("ls -al {}/{}".format(sitePath, clusterName))
os.system("ls -al {}/{}".format(sitePath, clusterId))
except git.exc.GitCommandError as e:
print("Repository {} doesn't exist. Skipping..".format(repoUrl))
print(e)
Expand All @@ -94,7 +99,7 @@ spec:
else:
chartFound['override'][k] = v
numChanged += 1
commit_msg = "update items for {} chart in '{}' cluster".format(chartName, clusterName)
commit_msg = "update items for {} chart in '{}' cluster".format(chartName, clusterId)
elif action == 'insert':
for k,v in kvMap.items():
# If the value is list type, then append one by one iterating over the list.
Expand All @@ -110,7 +115,7 @@ spec:
else:
chartFound['override'][k].append(v)
numChanged += 1
commit_msg = "insert items for {} chart in '{}' cluster".format(chartName, clusterName)
commit_msg = "insert items for {} chart in '{}' cluster".format(chartName, clusterId)
# DeleteItemFromList
elif action == 'delete':
for k,v in kvMap.items():
Expand All @@ -120,7 +125,7 @@ spec:
numChanged += 1
else:
print("The value {} doesn't exist.".format(v))
commit_msg = "delete items for {} chart in '{}' cluster".format(chartName, clusterName)
commit_msg = "delete items for {} chart in '{}' cluster".format(chartName, clusterId)
else:
sys.exit("Wrong action type")
Expand Down
136 changes: 43 additions & 93 deletions deploy_apps/tks-lma-federation-wftpl.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,12 +80,10 @@ spec:
template: updateManifest
arguments:
parameters:
- name: site_repo_url
value: "github.com/{{workflow.parameters.github_account}}/{{item.name}}"
- name: action
value: "insert"
- name: cluster_name
value: "{{item.name}}"
value: "{{ workflow.parameters.cluster_id }}"
- name: app_group
value: "lma"
- name: chart
Expand All @@ -100,8 +98,6 @@ spec:
template: updateManifest
arguments:
parameters:
- name: site_repo_url
value: "github.com/{{workflow.parameters.github_account}}/{{workflow.parameters.cluster_id}}"
- name: action
value: "insert"
# TODO: modify this to reflect actual cluster name for all cases
Expand Down Expand Up @@ -220,8 +216,7 @@ spec:
script:
name: 'collect'
image: harbor-cicd.taco-cat.xyz/tks/centos-tks-api:v1.0
command:
- python
command: ["python"]
envFrom:
- secretRef:
name: "git-svc-token"
Expand All @@ -233,55 +228,72 @@ spec:
import os
import git
import requests
import json
TKS_API_URL = "{{workflow.parameters.tks_info_host}}"
CLUSTER_ID = "{{workflow.parameters.cluster_id}}"
#CLUSTER_ID = "caldcde6u"
#CLUSTER_ID = "c6fk1w3dm"
def getToken() :
data = {
'organizationId': 'master',
'accountId': 'admin',
'password' : '1234'
'password' : 'admin'
}
res = requests.post(TKS_API_URL+"/api/1.0/auth/login", json = data )
if res.status_code != 200 :
return ''
resJson = res.json()
return resJson['data']['user']['token']
return resJson['user']['token']
output_cluster_list = []
temp_map = {}
inwards_endpoint_list = []
inwards_endpoint_map = {}
outwards_endpoint_map = {}
TOKEN=getToken()
res = requests.get(TKS_API_URL+"/api/1.0/clusters/" + CLUSTER_ID,
headers={"Authorization": "Bearer " + getToken()} )
headers={"Authorization": "Bearer " + TOKEN} )
if res.status_code != 200 :
sys.exit('Failed to get cluster')
cluster = res.json()['data']['cluster']
cluster = res.json()['cluster']
print( cluster )
organizationId = res.cluster['organizationId']
curClusterName = res.cluster['id']
organizationId = cluster['organizationId']
cur_cluster_name = cluster['id']
res = requests.get(TKS_API_URL+"/api/1.0/clusters?organizationId=" + organizationId,
headers={"Authorization": "Bearer " + getToken()} )
headers={"Authorization": "Bearer " + TOKEN} )
if res.status_code != 200 :
sys.exit('Failed to get clusters')
clusters = res.json()['clusters']
print("Iterating over clusters in the same contract...")
# Iterate over cluster list except current cluster #
for cluster in res.clusters:
for cluster in clusters:
if cluster['status'] != "RUNNING":
continue
if cluster['id'] != "{{workflow.parameters.cluster_id}}":
if cluster['id'] != CLUSTER_ID :
print("*******************************************")
print("Checking cluster: {}".format(cluster.id))
print("Checking cluster: {}".format(cluster['id']))
gitBaseUrl = os.environ['GIT_SVC_URL'].replace("http://","")
print( gitBaseUrl )
print("Checking if corresponding cluster repo exists..")
url = "@github.com/{{workflow.parameters.github_account}}/{}".format(cluster['id'])
repoUrl = "https://" + os.environ['TOKEN'] + url
#url = "@github.com/{{workflow.parameters.github_account}}/{}".format(cluster['id'])
url = "@" + gitBaseUrl + "/" + os.environ['USERNAME'] + "/" + cluster['id']
print( url )
repoUrl = "http://" + os.environ['TOKEN'] + url
try:
repo = git.Repo.clone_from(repoUrl, './tempcluster')
Expand All @@ -290,106 +302,44 @@ spec:
print("Repo {} doesn't exist. Skipping this cluster..".format(repoUrl))
continue
res = requests.get(TKS_API_URL+"/api/1.0/clusters?organizationId=" + organizationId,
headers={"Authorization": "Bearer " + getToken()} )
res = requests.get(TKS_API_URL+"/api/1.0/app-groups?clusterId=" + cluster['id'],
headers={"Authorization": "Bearer " + TOKEN} )
if res.status_code != 200 :
print( 'Failed to get appgroups for cluster ')
continue
appGroups = res.json()['data']['appGroups']
appGroups = res.json()['appGroups']
print( appGroups )
os.system("rm -rf ./tempcluster")
# Check if LMA group exists.
for appGroup in iappGroups:
if appGroup['type'] == "LMA" :
for appGroup in appGroups:
if appGroup['appGroupType'] == "LMA" :
print("Found LMA appGroup: {}".format(appGroup['name']))
res = requests.get(TKS_API_URL+"/api/1.0/app-groups/" + appGroup['id'] + "/applications",
headers={"Authorization": "Bearer " + getToken()} )
res = requests.get(TKS_API_URL+"/api/1.0/app-groups/" + appGroup['id'] + "/applications?applicationType=PROMETHEUS",
headers={"Authorization": "Bearer " + TOKEN} )
if res.status_code != 200 :
print( 'Failed to get applications for appgroup')
continue
continue
applications = res.json()['applications']
res = app_stub.GetApps(info_pb2.GetAppsRequest(app_group_id=app_group.app_group_id, type=common_pb2.PROMETHEUS))
if res.apps:
if applications :
# This is based on the premise that there's only one prometheus per appGroup.
endpoint = res.apps[0].endpoint
endpoint = applications[0]['endpoint']
print("Get Thanos-sc endpoint: {}. Appending it to inward list.".format(endpoint))
# Add this cluster's endpoint to endpoint map
inwards_endpoint_list.append(endpoint)
# Add this cluster to outward list so that current ep is updated to this cluster
temp_map["name"] = cluster.id
temp_map["name"] = cluster['id']
str_json = json.dumps(temp_map)
output_cluster_list.append(str_json)
with grpc.insecure_channel(addr) as channel:
cl_stub = info_pb2_grpc.ClusterInfoServiceStub(channel)
app_stub = info_pb2_grpc.AppInfoServiceStub(channel)
res = cl_stub.GetCluster(info_pb2.GetClusterRequest(cluster_id="{{workflow.parameters.cluster_id}}"))
print("Response from GetCluster: ")
print(res.cluster)
contract_id = res.cluster.contract_id
csp_id = res.cluster.csp_id
cur_cluster_name = res.cluster.id
res = cl_stub.GetClusters(info_pb2.GetClustersRequest(contract_id=contract_id, csp_id=''))
print("Iterating over clusters in the same contract...")
# Iterate over cluster list except current cluster #
for cluster in res.clusters:
if cluster.status != common_pb2.RUNNING:
continue
if cluster.id != "{{workflow.parameters.cluster_id}}":
print("*******************************************")
print("Checking cluster: {}".format(cluster.id))
print("Checking if corresponding cluster repo exists..")
url = "@github.com/{{workflow.parameters.github_account}}/{}".format(cluster.id)
repoUrl = "https://" + os.environ['TOKEN'] + url
try:
repo = git.Repo.clone_from(repoUrl, './tempcluster')
except git.exc.GitCommandError as e:
print(e)
print("Repo {} doesn't exist. Skipping this cluster..".format(repoUrl))
continue
res = app_stub.GetAppGroupsByClusterID(common_pb2.IDRequest(id=cluster.id))
print("Response from GetAppGroupsByClusterID:")
print(res.app_groups)
os.system("rm -rf ./tempcluster")
# Check if LMA group exists.
for app_group in res.app_groups:
if app_group.type == common_pb2.LMA:
print("Found LMA app_group: {}".format(app_group.app_group_name))
res = app_stub.GetApps(info_pb2.GetAppsRequest(app_group_id=app_group.app_group_id, type=common_pb2.PROMETHEUS))
if res.apps:
# This is based on the premise that there's only one prometheus per appGroup.
endpoint = res.apps[0].endpoint
print("Get Thanos-sc endpoint: {}. Appending it to inward list.".format(endpoint))
# Add this cluster's endpoint to endpoint map
inwards_endpoint_list.append(endpoint)
# Add this cluster to outward list so that current ep is updated to this cluster
temp_map["name"] = cluster.id
str_json = json.dumps(temp_map)
output_cluster_list.append(str_json)
# Compose profer format to be used as input on next step
inwards_endpoint_map['querier.stores'] = inwards_endpoint_list
Expand Down
2 changes: 1 addition & 1 deletion git-repo/create-cluster-repo.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ spec:
sed -i "s/mdMinSizePerAz:\ CHANGEME/mdMinSizePerAz: $val_min_size/g" $CLUSTER_ID/$CLUSTER_ID/tks-cluster/site-values.yaml
sed -i "s/mdMaxSizePerAz:\ CHANGEME/mdMaxSizePerAz: $val_max_size/g" $CLUSTER_ID/$CLUSTER_ID/tks-cluster/site-values.yaml
sed -i "s/mdMachineType:\ CHANGEME/mdMachineType: $val_machine_type/g" $CLUSTER_ID/$CLUSTER_ID/tks-cluster/site-values.yaml
sed -i "s/cpReplicas:\ CHANGEME/cpReplicas: 1/g" $CLUSTER_ID/$CLUSTER_ID/tks-cluster/site-values.yaml
sed -i "s/cpReplicas:\ CHANGEME/cpReplicas: 3/g" $CLUSTER_ID/$CLUSTER_ID/tks-cluster/site-values.yaml
;;
byoh)
Expand Down
2 changes: 1 addition & 1 deletion git-repo/render-manifests.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ spec:
- name: https_enabled
container:
name: render-manifests-template
image: harbor-cicd.taco-cat.xyz/tks/decapod-render:v2.1.0
image: harbor-cicd.taco-cat.xyz/tks/decapod-render:v3.1.2
command:
- /bin/bash
- '-exc'
Expand Down
1 change: 1 addition & 0 deletions tks-cluster/remove-usercluster-wftpl.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -271,6 +271,7 @@ spec:
else
echo "Error: installations.v1.operator.tigera.io 'default' was NOT found."
exit 1
fi
kubectl $kube_params get installations.v1.operator.tigera.io | grep default
if [[ $? =~ 0 ]]; then
Expand Down
4 changes: 2 additions & 2 deletions tks_info/update-tks-app-group-wftpl.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ spec:
def getToken() :
data = {
'organizationId': 'master'
'organizationId': 'master',
'accountId': 'admin',
'password' : 'admin'
}
Expand All @@ -41,7 +41,7 @@ spec:
if res.status_code != 200 :
return ''
resJson = res.json()
return resJson['data']['user']['token']
return resJson['user']['token']
print( ENDPOINTS )
for appType, endpoint in {{inputs.parameters.endpoints}}.items() :
Expand Down

0 comments on commit 485c13a

Please sign in to comment.