Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

handle all corner cases when updating thanos-sc endpoints #78

Merged
merged 1 commit into from
Apr 11, 2022
Merged

handle all corner cases when updating thanos-sc endpoints #78

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 12 additions & 7 deletions decapod-manifests/update-decapod-manifest-wftpl.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,13 +63,18 @@ spec:
print("Cloning repository...")

repoUrl = "https://" + os.environ['TOKEN'] + "@{{inputs.parameters.site_repo_url}}"
repo = git.Repo.clone_from(repoUrl, siteDir)
with repo.config_writer() as git_config:
git_config.set_value('user', 'email', '[email protected]')
git_config.set_value('user', 'name', 'TKS Argo')
print("Cloned repo contents:")
os.system("ls -al {}".format(sitePath))
os.system("ls -al {}/{}".format(sitePath, clusterName))
try:
repo = git.Repo.clone_from(repoUrl, siteDir)
with repo.config_writer() as git_config:
git_config.set_value('user', 'email', '[email protected]')
git_config.set_value('user', 'name', 'TKS Argo')
print("Cloned repo contents:")
os.system("ls -al {}".format(sitePath))
os.system("ls -al {}/{}".format(sitePath, clusterName))
except git.exc.GitCommandError as e:
print("Repository {} doesn't exist. Skipping..".format(repoUrl))
print(e)
sys.exit(0)
else:
print("The repo already exists. Pulling latest updates..")

Expand Down
49 changes: 37 additions & 12 deletions deploy_apps/tks-lma-federation-wftpl.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,15 +47,15 @@ spec:
name: lma-federation
template: deploy

- - name: updateTksInfo
- - name: updateEndpointToTksInfo
templateRef:
name: update-tks-app-group-info
template: updateTksAppGroup
arguments:
parameters:
# TODO: Can this be pre-determined? Or composed dynamically on deployment?
- name: endpoints
value: "{'PROMETHEUS': 'thanos-sidecar.cluster_xy'}"
value: "{'PROMETHEUS': '{{workflow.parameters.cluster_id}}-thanos-ep'}"

# LMA appGroup specific task #
- - name: collectThanosScEndpoints
Expand All @@ -71,7 +71,7 @@ spec:
value: "{{ workflow.parameters.app_group_id }}"
# Again, how can this be determined?
- name: cur_endpoint
value: "thanos-sidecar.cluster_xy"
value: "{{workflow.parameters.cluster_id}}-thanos-ep"

- - name: updateDecapodManifestOutwards
templateRef:
Expand Down Expand Up @@ -184,6 +184,9 @@ spec:
image: sktdev/python-centos-wf-worker:v1.0
command:
- python
envFrom:
- secretRef:
name: "github-tks-mgmt-token"
env:
- name: PYTHONPATH
value: "/opt/protobuf/:/opt/rh/rh-python38/root/lib/python3.8/site-packages/:/opt/app-root/lib/python3.8/site-packages/"
Expand All @@ -195,13 +198,15 @@ spec:
mountPath: /mnt/out
source: |
import sys
import os
import google.protobuf
import grpc
import info_pb2
import info_pb2_grpc
import common_pb2
import common_pb2_grpc
import json
import git

output_cluster_list = []
temp_map = {}
Expand Down Expand Up @@ -238,24 +243,44 @@ spec:
continue

if cluster.id != "{{inputs.parameters.cluster_id}}":
print("Found cluster: {}".format(cluster.id))
temp_map["name"] = cluster.id
str_json = json.dumps(temp_map)
output_cluster_list.append(str_json)
print("*******************************************")
print("Checking cluster: {}".format(cluster.id))

print("Checking if corresponding cluster repo exists..")
url = "@github.com/{{workflow.parameters.github_account}}/{}".format(cluster.id)
repoUrl = "https://" + os.environ['TOKEN'] + url
try:
repo = git.Repo.clone_from(repoUrl, './tempcluster')

except git.exc.GitCommandError as e:
print(e)
print("Repo {} doesn't exist. Skipping this cluster..".format(repoUrl))
continue

res = app_stub.GetAppGroupsByClusterID(common_pb2.IDRequest(id=cluster.id))
print("Response from GetAppGroupsByClusterID:")
print(res.app_groups)

os.system("rm -rf ./tempcluster")

# Check if LMA group exists.
for app_group in res.app_groups:
if app_group.type == common_pb2.LMA:
res = app_stub.GetApps(info_pb2.GetAppsRequest(app_group_id="{{inputs.parameters.app_group_id}}", type=common_pb2.PROMETHEUS))
print("Found LMA app_group: {}".format(app_group.app_group_name))
res = app_stub.GetApps(info_pb2.GetAppsRequest(app_group_id=app_group.app_group_id, type=common_pb2.PROMETHEUS))
ktkfree marked this conversation as resolved.
Show resolved Hide resolved

if res.apps:
# This is based on the premise that there's only one prometheus per appGroup.
endpoint = res.apps[0].endpoint
print("Get Thanos-sc endpoint: {}. Appending it to inward list.".format(endpoint))

# This is based on the premise that there's only one prometheus per appGroup.
endpoint = res.apps[0].endpoint
# Add this cluster's endpoint to endpoint map
inwards_endpoint_list.append(endpoint)

# Add this cluster's endpoint to endpoint map
inwards_endpoint_list.append(endpoint)
# Add this cluster to outward list so that current ep is updated to this cluster
temp_map["name"] = cluster.id
str_json = json.dumps(temp_map)
output_cluster_list.append(str_json)

# Update current endpoint to other cluster's site-yaml #
outwards_endpoint_map['querier.stores'] = "{{inputs.parameters.cur_endpoint}}"
Expand Down
41 changes: 36 additions & 5 deletions deploy_apps/tks-remove-lma-federation-wftpl.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ spec:
value: "{{ workflow.parameters.app_group_id }}"
# Again, how can this be determined?
- name: cur_endpoint
value: "thanos-sidecar.cluster_xy"
value: "{{workflow.parameters.cluster_id}}-thanos-ep"

# Delete endpoint from other cluster's manifests
- - name: updateDecapodManifestOutwards
Expand Down Expand Up @@ -152,6 +152,9 @@ spec:
image: sktdev/python-centos-wf-worker:v1.0
command:
- python
envFrom:
- secretRef:
name: "github-tks-mgmt-token"
env:
- name: PYTHONPATH
value: "/opt/protobuf/:/opt/rh/rh-python38/root/lib/python3.8/site-packages/:/opt/app-root/lib/python3.8/site-packages/"
Expand All @@ -163,13 +166,15 @@ spec:
mountPath: /mnt/out
source: |
import sys
import os
import google.protobuf
import grpc
import info_pb2
import info_pb2_grpc
import common_pb2
import common_pb2_grpc
import json
import git

output_cluster_list = []
temp_map = {}
Expand Down Expand Up @@ -200,10 +205,36 @@ spec:
# Iterate over cluster list except current cluster #
for cluster in res.clusters:
if cluster.id != "{{inputs.parameters.cluster_id}}":
print("Found cluster: {}".format(cluster.id))
temp_map["name"] = cluster.id
str_json = json.dumps(temp_map)
output_cluster_list.append(str_json)
print("*******************************************")
print("Checking cluster: {}".format(cluster.id))

print("Checking if corresponding cluster repo exists..")
url = "@github.com/{{workflow.parameters.github_account}}/{}".format(cluster.id)
repoUrl = "https://" + os.environ['TOKEN'] + url
try:
repo = git.Repo.clone_from(repoUrl, './tempcluster')

except git.exc.GitCommandError as e:
print(e)
print("Repo {} doesn't exist. Skipping this cluster..".format(repoUrl))
continue

res = app_stub.GetAppGroupsByClusterID(common_pb2.IDRequest(id=cluster.id))
print("Response from GetAppGroupsByClusterID:")
print(res.app_groups)

os.system("rm -rf ./tempcluster")

# Check if LMA group exists.
for app_group in res.app_groups:
if app_group.type == common_pb2.LMA:
print("Found LMA app_group: {}".format(app_group.app_group_name))
res = app_stub.GetApps(info_pb2.GetAppsRequest(app_group_id=app_group.app_group_id, type=common_pb2.PROMETHEUS))

# Add this cluster to outward list so that current ep is deleted from this cluster
temp_map["name"] = cluster.id
str_json = json.dumps(temp_map)
output_cluster_list.append(str_json)

# Update current endpoint to other cluster's site-yaml #
outwards_endpoint_map['querier.stores'] = "{{inputs.parameters.cur_endpoint}}"
Expand Down