From 0d7ea913111400a7c0e77bb21cd1f967d94619ab Mon Sep 17 00:00:00 2001 From: Robert Choi Date: Mon, 13 Sep 2021 14:52:07 +0900 Subject: [PATCH 1/5] add new workflow templates - tks-lma-fed - update-tks-info --- deploy_apps/tks-lma-federation-wftpl.yaml | 232 ++++++++++++++++++++++ tks_info/update-tks-info-wftpl.yaml | 64 ++++++ 2 files changed, 296 insertions(+) create mode 100644 deploy_apps/tks-lma-federation-wftpl.yaml create mode 100644 tks_info/update-tks-info-wftpl.yaml diff --git a/deploy_apps/tks-lma-federation-wftpl.yaml b/deploy_apps/tks-lma-federation-wftpl.yaml new file mode 100644 index 00000000..5eb5ef5e --- /dev/null +++ b/deploy_apps/tks-lma-federation-wftpl.yaml @@ -0,0 +1,232 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: tks-lma-federation + namespace: argo +spec: + entrypoint: deploy + arguments: + parameters: + - name: site_name + value: "hanu-reference" + # This should be renamed to app_group_name + - name: app_name + value: "lma" + - name: repository_url + value: "https://github.com/openinfradev/decapod-manifests" + ########################## + # For updateTksInfo task # + ########################## + - name: tks_info_host + value: "127.0.0.1" + - name: cluster_id + value: "abbead61-ff2a-4af4-8f41-d2c44c745de7" + - name: app_group_id + value: "abbead61-ff2a-4af4-8f41-d2c44c745de7" + volumes: + - name: tks-proto-vol + configMap: + name: tks-proto + templates: + - name: deployLMA + steps: + - - name: installApps + templateRef: + name: lma-federation + template: deploy + + - - name: updateTksInfo + templateRef: + name: update-tks-info + template: updateTksApp + arguments: + parameters: + # TODO: Can this be pre-determined? Or composed dynamically on deployment? + - name: endpoints + value: "{'PROMETHEUS': 'thanos-sidecar.cluster_xy'}" + - name: app_group_status + value: "APP_GROUP_RUNNING" + + # LMA appGroup specific task # + - - name: collectThanosScEndpoints + template: collectThanosScEndpoints + arguments: + # These params should be moved to global argument? # + parameters: + - name: tks_info_host + value: "{{ workflow.parameters.tks_info_host }}" + - name: cluster_id + value: "{{ workflow.parameters.cluster_id }}" + - name: app_group_id + value: "{{ workflow.parameters.app_group_id }}" + # Again, how can this be determined? + - name: cur_endpoint + value: "thanos-sidecar.cluster_xy" + + - - name: updateDecapodManifestOutwards + templateRef: + name: update-decapod-manifest + template: updateManifest + arguments: + parameters: + - name: action + value: "insert" + - name: cluster_name + value: "{{item.name}}" + - name: app_group + value: "{{workflow.parameters.app_name}}" + - name: chart + value: "thanos" + - name: kv_map_str + value: "{{steps.collectThanosScEndpoints.outputs.parameters.outwards_endpoint_map}}" + ## {"querier.stores": "CURRENT_ENDPOINT"} + withParam: "{{steps.collectThanosScEndpoints.outputs.parameters.cluster_list}}" + + - - name: updateDecapodManifestInwards + templateRef: + name: update-decapod-manifest + template: updateManifest + arguments: + parameters: + - name: action + value: "insert" + # TODO: modify this to reflect actual cluster name for all cases + - name: cluster_name + value: "{{workflow.parameters.cluster_id}}" + - name: app_group + value: "{{workflow.parameters.app_name}}" + - name: chart + value: "thanos" + - name: kv_map_str + value: "{{steps.collectThanosScEndpoints.outputs.parameters.inwards_endpoint_map}}" + ## {"querier.stores": ['endpointA', 'endpointB']} + + - name: collectThanosScEndpoints + inputs: + parameters: + - name: tks_info_host + - name: cluster_id + - name: app_group_id + - name: cur_endpoint + outputs: + parameters: + - name: cluster_list + valueFrom: + path: /mnt/out/cluster_list.txt + - name: inwards_endpoint_map + valueFrom: + path: /mnt/out/inwards_endpoint.txt + - name: outwards_endpoint_map + valueFrom: + path: /mnt/out/outwards_endpoint.txt + volumes: + - name: out + emptyDir: {} + script: + name: 'collect' + image: centos/python-38-centos7 + command: + - python + env: + - name: PYTHONPATH + value: "/opt/protobuf/:/opt/rh/rh-python38/root/lib/python3.8/site-packages/:/opt/app-root/lib/python3.8/site-packages/" + volumeMounts: + - name: tks-proto-vol + mountPath: "/opt/protobuf" + readOnly: true + - name: out + mountPath: /mnt/out + source: | + import subprocess + import sys + + # TODO: bake custom image that includes these packages + subprocess.check_call([sys.executable, "-m", "pip", "install", + "grpcio", "protobuf"]) + + import google.protobuf + import grpc + import info_pb2 + import info_pb2_grpc + import common_pb2 + import common_pb2_grpc + import json + + output_cluster_list = [] + temp_map = {} + inwards_endpoint_list = [] + inwards_endpoint_map = {} + outwards_endpoint_map = {} + + ip = "{{inputs.parameters.tks_info_host}}" + # TODO: Make port workflow param? + port = 9111 + addr = "%s:%d" % (ip, port) + print("tks-info addr: %s" % addr) + + with grpc.insecure_channel(addr) as channel: + cl_stub = info_pb2_grpc.ClusterInfoServiceStub(channel) + app_stub = info_pb2_grpc.AppInfoServiceStub(channel) + + res = cl_stub.GetCluster(info_pb2.GetClusterRequest(cluster_id="{{inputs.parameters.cluster_id}}")) + + print("Response from GetCluster: ") + print(res.cluster) + contract_id = res.cluster.contract_id + csp_id = res.cluster.csp_id + # TODO: export this as step output to use it on next step + cur_cluster_name = res.cluster.name + + res = cl_stub.GetClusters(info_pb2.GetClustersRequest(contract_id=contract_id, csp_id=csp_id)) + print("Response from GetClusters: ") + print(res.clusters) + + # Iterate over cluster list except current cluster # + for cluster in res.clusters: + if cluster.id != "{{inputs.parameters.cluster_id}}": + temp_map["name"] = cluster.name + str_json = json.dumps(temp_map) + output_cluster_list.append(str_json) + + res = app_stub.GetAppGroupsByClusterID(common_pb2.IDRequest(id=cluster.id)) + print("Response from GetAppGroupsByClusterID:") + print(res.app_groups) + + for app_group in res.app_groups: + if app_group.type == common_pb2.LMA: + res = app_stub.GetApps(info_pb2.GetAppsRequest(app_group_id="{{inputs.parameters.app_group_id}}", type=common_pb2.PROMETHEUS)) + + # This is based on the premise that there's only one prometheus per appGroup. + endpoint = res.apps[0].endpoint + + # Add this cluster's endpoint to endpoint map + inwards_endpoint_list.append(endpoint) + + # Update current endpoint to other cluster's site-yaml # + outwards_endpoint_map['querier.stores'] = "{{inputs.parameters.cur_endpoint}}" + + # Compose profer format to be used as input on next step + inwards_endpoint_map['querier.stores'] = inwards_endpoint_list + + ########################### + # Construct output params # + ########################### + len_list = len(output_cluster_list) + + with open("/mnt/out/cluster_list.txt", "w") as f: + f.write('[') + + for idx, item in enumerate(output_cluster_list, start=1): + print("item {}: {}".format(idx, item)) + f.write(item.strip("'")) + if idx < len_list: + f.write(',') + f.write(']') + + with open("/mnt/out/inwards_endpoint.txt", "w") as f: + str_inwards_endpoint = repr(inwards_endpoint_map) + f.write(str_inwards_endpoint) + + with open("/mnt/out/outwards_endpoint.txt", "w") as f: + str_outwards_endpoint = repr(outwards_endpoint_map) + f.write(str_outwards_endpoint) diff --git a/tks_info/update-tks-info-wftpl.yaml b/tks_info/update-tks-info-wftpl.yaml new file mode 100644 index 00000000..3970bf97 --- /dev/null +++ b/tks_info/update-tks-info-wftpl.yaml @@ -0,0 +1,64 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: update-tks-info + namespace: argo +spec: + entrypoint: updateTksApp + arguments: + parameters: + - name: tks_info_host + value: "127.0.0.1" + - name: app_group_id + value: "abbead61-ff2a-4af4-8f41-d2c44c745de7" + volumes: + - name: tks-proto-vol + configMap: + name: tks-proto + templates: + - name: updateTksApp + inputs: + parameters: + - name: endpoints # Dict type + - name: app_group_status + value: APP_GROUP_RUNNING + script: + image: centos/python-38-centos7 + command: ["python"] + env: + - name: PYTHONPATH + value: "/opt/protobuf/:/opt/rh/rh-python38/root/lib/python3.8/site-packages/:/opt/app-root/lib/python3.8/site-packages/" + volumeMounts: + - name: tks-proto-vol + mountPath: "/opt/protobuf" + readOnly: true + source: | + import subprocess + import sys + + # TODO: bake custom image that includes these packages + subprocess.check_call([sys.executable, "-m", "pip", "install", + "grpcio", "protobuf"]) + + import google.protobuf + import grpc + import info_pb2 + import info_pb2_grpc + import common_pb2 + import common_pb2_grpc + + ip = "{{workflow.parameters.tks_info_host}}" + port = 9111 # if not specified + addr = "%s:%d" % (ip, port) + print("tks-info addr: %s" % addr) + + with grpc.insecure_channel(addr) as channel: + app_stub = info_pb2_grpc.AppInfoServiceStub(channel) + + res = app_stub.UpdateAppGroupStatus(info_pb2.UpdateAppGroupStatusRequest(app_group_id="{{workflow.parameters.app_group_id}}", status=common_pb2.{{inputs.parameters.app_group_status}})) + print("Response code from UpdateAppGroupStaus: %d" % res.code) + + # TODO: how to dynamically construct app_type param as enum format here? + for app_type_, ep in {{inputs.parameters.endpoints}}: + res = app_stub.UpdateApp(info_pb2.UpdateAppRequest(app_group_id="{{workflow.parameters.app_group_id}}", app_type=common_pb2.{{app_type_}}, endpoint=ep, metadata="{}")) + print("Response code from UpdateApp: %d" % res.code) From 257564ff149314edbb587e7e59cac6a3553f2812 Mon Sep 17 00:00:00 2001 From: Robert Choi Date: Wed, 15 Sep 2021 11:41:28 +0900 Subject: [PATCH 2/5] export cluster_name as output so that it can be used by update-decapod-manifest workflow --- deploy_apps/tks-lma-federation-wftpl.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/deploy_apps/tks-lma-federation-wftpl.yaml b/deploy_apps/tks-lma-federation-wftpl.yaml index 5eb5ef5e..e0a530c6 100644 --- a/deploy_apps/tks-lma-federation-wftpl.yaml +++ b/deploy_apps/tks-lma-federation-wftpl.yaml @@ -119,12 +119,15 @@ spec: - name: outwards_endpoint_map valueFrom: path: /mnt/out/outwards_endpoint.txt + - name: cur_cluster_name + valueFrom: + path: /mnt/out/cur_cluster_name.txt volumes: - name: out emptyDir: {} script: name: 'collect' - image: centos/python-38-centos7 + image: sktdev/python-centos-wf-worker:v1.0 command: - python env: @@ -137,13 +140,7 @@ spec: - name: out mountPath: /mnt/out source: | - import subprocess import sys - - # TODO: bake custom image that includes these packages - subprocess.check_call([sys.executable, "-m", "pip", "install", - "grpcio", "protobuf"]) - import google.protobuf import grpc import info_pb2 @@ -230,3 +227,6 @@ spec: with open("/mnt/out/outwards_endpoint.txt", "w") as f: str_outwards_endpoint = repr(outwards_endpoint_map) f.write(str_outwards_endpoint) + + with open("/mnt/out/cur_cluster_name.txt", "w") as f: + f.write(cur_cluster_name) From f8ecfcbcb39f58b29706d68eaef9a60b67bcdd05 Mon Sep 17 00:00:00 2001 From: Robert Choi Date: Wed, 15 Sep 2021 15:39:32 +0900 Subject: [PATCH 3/5] minor grammer fix --- deploy_apps/tks-lma-federation-wftpl.yaml | 6 ++++-- tks_info/update-tks-info-wftpl.yaml | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/deploy_apps/tks-lma-federation-wftpl.yaml b/deploy_apps/tks-lma-federation-wftpl.yaml index e0a530c6..83167abc 100644 --- a/deploy_apps/tks-lma-federation-wftpl.yaml +++ b/deploy_apps/tks-lma-federation-wftpl.yaml @@ -4,7 +4,7 @@ metadata: name: tks-lma-federation namespace: argo spec: - entrypoint: deploy + entrypoint: deployLMA arguments: parameters: - name: site_name @@ -14,6 +14,8 @@ spec: value: "lma" - name: repository_url value: "https://github.com/openinfradev/decapod-manifests" + - name: revision + value: "main" ########################## # For updateTksInfo task # ########################## @@ -92,7 +94,7 @@ spec: value: "insert" # TODO: modify this to reflect actual cluster name for all cases - name: cluster_name - value: "{{workflow.parameters.cluster_id}}" + value: "{{steps.collectThanosScEndpoints.outputs.parameters.cur_cluster_name}}" - name: app_group value: "{{workflow.parameters.app_name}}" - name: chart diff --git a/tks_info/update-tks-info-wftpl.yaml b/tks_info/update-tks-info-wftpl.yaml index 3970bf97..b726f901 100644 --- a/tks_info/update-tks-info-wftpl.yaml +++ b/tks_info/update-tks-info-wftpl.yaml @@ -59,6 +59,6 @@ spec: print("Response code from UpdateAppGroupStaus: %d" % res.code) # TODO: how to dynamically construct app_type param as enum format here? - for app_type_, ep in {{inputs.parameters.endpoints}}: + for app_type_, ep in {{inputs.parameters.endpoints}}.items(): res = app_stub.UpdateApp(info_pb2.UpdateAppRequest(app_group_id="{{workflow.parameters.app_group_id}}", app_type=common_pb2.{{app_type_}}, endpoint=ep, metadata="{}")) print("Response code from UpdateApp: %d" % res.code) From da48f5a8532e95de2d73536169bc54d9173791e9 Mon Sep 17 00:00:00 2001 From: Robert Choi Date: Thu, 16 Sep 2021 10:06:43 +0900 Subject: [PATCH 4/5] add site_repo_url param --- deploy_apps/tks-lma-federation-wftpl.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/deploy_apps/tks-lma-federation-wftpl.yaml b/deploy_apps/tks-lma-federation-wftpl.yaml index 83167abc..38b07fcd 100644 --- a/deploy_apps/tks-lma-federation-wftpl.yaml +++ b/deploy_apps/tks-lma-federation-wftpl.yaml @@ -9,10 +9,13 @@ spec: parameters: - name: site_name value: "hanu-reference" - # This should be renamed to app_group_name + # TODO: This should be renamed to app_group_name - name: app_name value: "lma" - - name: repository_url + # Replace these urls properly for your env # + - name: site_repo_url + value: "https://github.com/openinfradev/decapod-site" + - name: manifest_repo_url value: "https://github.com/openinfradev/decapod-manifests" - name: revision value: "main" From f24f95b33ee393f8926efd22e403f1aab5713dc5 Mon Sep 17 00:00:00 2001 From: Robert Choi Date: Thu, 16 Sep 2021 10:37:19 +0900 Subject: [PATCH 5/5] fix RPC param --- tks_info/update-tks-info-wftpl.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tks_info/update-tks-info-wftpl.yaml b/tks_info/update-tks-info-wftpl.yaml index b726f901..78df0250 100644 --- a/tks_info/update-tks-info-wftpl.yaml +++ b/tks_info/update-tks-info-wftpl.yaml @@ -60,5 +60,5 @@ spec: # TODO: how to dynamically construct app_type param as enum format here? for app_type_, ep in {{inputs.parameters.endpoints}}.items(): - res = app_stub.UpdateApp(info_pb2.UpdateAppRequest(app_group_id="{{workflow.parameters.app_group_id}}", app_type=common_pb2.{{app_type_}}, endpoint=ep, metadata="{}")) + res = app_stub.UpdateApp(info_pb2.UpdateAppRequest(app_group_id="{{workflow.parameters.app_group_id}}", app_type=app_type_, endpoint=ep, metadata="{}")) print("Response code from UpdateApp: %d" % res.code)