From 0b90026a9422bf475c4c204945f584d050480809 Mon Sep 17 00:00:00 2001 From: YANGDB Date: Wed, 21 Jun 2023 14:04:50 -0700 Subject: [PATCH] add integration dependency on the catalog repository for getting all the schema related index and component templates mapping (#29) Signed-off-by: YANGDB --- src/integrations/src/data.ini | 1169 ------------------------------- src/integrations/src/install.py | 45 +- 2 files changed, 35 insertions(+), 1179 deletions(-) diff --git a/src/integrations/src/data.ini b/src/integrations/src/data.ini index 2e62882289..9408ecd76b 100644 --- a/src/integrations/src/data.ini +++ b/src/integrations/src/data.ini @@ -2,1175 +2,6 @@ # SPDX-License-Identifier: Apache-2.0 [DEFAULT] - -[component-templates] -cloud_template = { - "template": { - "mappings": { - "properties": { - "cloud": { - "properties": { - "provider": { - "type": "keyword" - }, - "availability_zone": { - "type": "keyword" - }, - "region": { - "type": "keyword" - }, - "machine": { - "type": "object", - "properties": { - "type": { - "type": "keyword" - } - } - }, - "account": { - "type": "object", - "properties": { - "id": { - "type": "keyword" - }, - "name": { - "type": "keyword" - } - } - }, - "service": { - "type": "object", - "properties": { - "name": { - "type": "keyword" - } - } - }, - "project": { - "type": "object", - "properties": { - "id": { - "type": "keyword" - }, - "name": { - "type": "keyword" - } - } - }, - "instance": { - "type": "object", - "properties": { - "id": { - "type": "keyword" - }, - "name": { - "type": "keyword" - } - } - } - } - } - } - } - } - } - -communication_template = { - "template": { - "mappings": { - "properties": { - "communication": { - "properties": { - "sock.family": { - "type": "keyword", - "ignore_above": 256 - }, - "source": { - "type": "object", - "properties": { - "address": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "domain": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "bytes": { - "type": "long" - }, - "ip": { - "type": "ip" - }, - "port": { - "type": "long" - }, - "mac": { - "type": "keyword", - "ignore_above": 1024 - }, - "packets": { - "type": "long" - } - } - }, - "destination": { - "type": "object", - "properties": { - "address": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "domain": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "bytes": { - "type": "long" - }, - "ip": { - "type": "ip" - }, - "port": { - "type": "long" - }, - "mac": { - "type": "keyword", - "ignore_above": 1024 - }, - "packets": { - "type": "long" - } - } - } - } - } - } - } - } - } - -http_template = { - "template": { - "mappings": { - "dynamic_templates": [ - { - "request_header_map": { - "mapping": { - "type": "keyword" - }, - "path_match": "request.header.*" - } - }, - { - "response_header_map": { - "mapping": { - "type": "keyword" - }, - "path_match": "response.header.*" - } - } - ], - "properties": { - "http": { - "properties": { - "flavor": { - "type": "keyword", - "ignore_above": 256 - }, - "user_agent": { - "type": "keyword", - "ignore_above": 2048 - }, - "url": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 2048 - } - } - }, - "schema": { - "type": "keyword", - "ignore_above": 1024 - }, - "target": { - "type": "keyword", - "ignore_above": 1024 - }, - "route": { - "type": "keyword", - "ignore_above": 1024 - }, - "client.ip": { - "type": "ip" - }, - "resent_count": { - "type": "integer" - }, - "request": { - "type": "object", - "properties": { - "id": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "body.content": { - "type": "text" - }, - "bytes": { - "type": "long" - }, - "method": { - "type": "keyword", - "ignore_above": 256 - }, - "referrer": { - "type": "keyword", - "ignore_above": 1024 - }, - "mime_type": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "response": { - "type": "object", - "properties": { - "id": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "body.content": { - "type": "text" - }, - "bytes": { - "type": "long" - }, - "status_code": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - } - } - } - } - } - -tracegroup_template = { - "template": { - "mappings": { - "properties": { - "traceGroup": { - "ignore_above": 1024, - "type": "keyword" - }, - "traceGroupFields": { - "properties": { - "endTime": { - "type": "date_nanos" - }, - "durationInNanos": { - "type": "long" - }, - "statusCode": { - "type": "integer" - } - } - } - } - } - } - } - -[index-templates] -logs_template = { - "index_patterns": [ - "sso_logs-*-*" - ], - "data_stream": {}, - "template": { - "mappings": { - "_meta": { - "version": "1.0.0", - "catalog": "observability", - "type": "logs", - "component" : "log" - }, - "_source": { - "enabled": true - }, - "dynamic_templates": [ - { - "resources_map": { - "mapping": { - "type": "keyword" - }, - "path_match": "resource.*" - } - }, - { - "attributes_map": { - "mapping": { - "type": "keyword" - }, - "path_match": "attributes.*" - } - }, - { - "instrumentation_scope_attributes_map": { - "mapping": { - "type": "keyword" - }, - "path_match": "instrumentationScope.attributes.*" - } - } - ], - "properties": { - "severity": { - "properties": { - "number": { - "type": "long" - }, - "text": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - }, - "attributes": { - "type": "object", - "properties": { - "data_stream": { - "properties": { - "dataset": { - "ignore_above": 128, - "type": "keyword" - }, - "namespace": { - "ignore_above": 128, - "type": "keyword" - }, - "type": { - "ignore_above": 56, - "type": "keyword" - } - } - } - } - }, - "body": { - "type": "text" - }, - "@timestamp": { - "type": "date" - }, - "observedTimestamp": { - "type": "date" - }, - "traceId": { - "ignore_above": 256, - "type": "keyword" - }, - "spanId": { - "ignore_above": 256, - "type": "keyword" - }, - "schemaUrl": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "instrumentationScope": { - "properties": { - "name": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 128 - } - } - }, - "version": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "dropped_attributes_count": { - "type": "integer" - }, - "schemaUrl": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - }, - "event": { - "properties": { - "domain": { - "ignore_above": 256, - "type": "keyword" - }, - "name": { - "ignore_above": 256, - "type": "keyword" - }, - "category": { - "ignore_above": 256, - "type": "keyword" - }, - "type": { - "ignore_above": 256, - "type": "keyword" - }, - "kind": { - "ignore_above": 256, - "type": "keyword" - }, - "result": { - "ignore_above": 256, - "type": "keyword" - }, - "exception": { - "properties": { - "message": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 256, - "type": "keyword" - }, - "stacktrace": { - "type": "text" - } - } - } - } - } - } - }, - "settings": { - "index": { - "mapping": { - "total_fields": { - "limit": 10000 - } - }, - "refresh_interval": "5s" - } - } - }, - "composed_of": [ - "http_template", - "communication_template", - "cloud_template" - ], - "version": 1, - "_meta": { - "description": "Simple Schema For Observability", - "catalog": "observability", - "type": "logs" - } - } - -metrics_template = { - "index_patterns": [ - "sso_metrics-*-*" - ], - "data_stream": {}, - "template": { - "mappings": { - "_meta": { - "version": "1.0.0", - "catalog": "observability", - "type": "metrics", - "component": "metrics", - "correlations" : [ - { - "field": "spanId", - "foreign-schema" : "traces", - "foreign-field" : "spanId" - }, - { - "field": "traceId", - "foreign-schema" : "traces", - "foreign-field" : "traceId" - } - ] - }, - "_source": { - "enabled": true - }, - "dynamic_templates": [ - { - "attributes_map": { - "mapping": { - "type": "keyword" - }, - "path_match": "attributes.*" - } - }, - { - "resources_map": { - "mapping": { - "type": "keyword" - }, - "path_match": "resource.*" - } - }, - { - "exemplar_attributes_map": { - "mapping": { - "type": "keyword" - }, - "path_match": "exemplar.attributes.*" - } - }, - { - "instrumentation_scope_attributes_map": { - "mapping": { - "type": "keyword" - }, - "path_match": "instrumentationScope.attributes.*" - } - } - ], - "properties": { - "name": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "attributes": { - "type": "object", - "properties": { - "data_stream": { - "properties": { - "dataset": { - "ignore_above": 128, - "type": "keyword" - }, - "namespace": { - "ignore_above": 128, - "type": "keyword" - }, - "type": { - "ignore_above": 56, - "type": "keyword" - } - } - } - } - }, - "description": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "unit": { - "type": "keyword", - "ignore_above": 128 - }, - "kind": { - "type": "keyword", - "ignore_above": 128 - }, - "aggregationTemporality": { - "type": "keyword", - "ignore_above": 128 - }, - "monotonic": { - "type": "boolean" - }, - "startTime": { - "type": "date" - }, - "@timestamp": { - "type": "date" - }, - "observedTimestamp": { - "type": "date_nanos" - }, - "value": { - "properties": { - "int": { - "type": "integer" - }, - "double": { - "type": "double" - } - } - }, - "buckets": { - "properties": { - "count": { - "type": "long" - }, - "sum": { - "type": "double" - }, - "max": { - "type": "float" - }, - "min": { - "type": "float" - } - } - }, - "bucketCount": { - "type": "long" - }, - "bucketCountsList": { - "type": "long" - }, - "explicitBoundsList": { - "type": "float" - }, - "explicitBoundsCount": { - "type": "float" - }, - "quantiles": { - "properties": { - "quantile": { - "type": "double" - }, - "value": { - "type": "double" - } - } - }, - "quantileValuesCount": { - "type": "long" - }, - "positiveBuckets": { - "properties": { - "count": { - "type": "long" - }, - "max": { - "type": "float" - }, - "min": { - "type": "float" - } - } - }, - "negativeBuckets": { - "properties": { - "count": { - "type": "long" - }, - "max": { - "type": "float" - }, - "min": { - "type": "float" - } - } - }, - "negativeOffset": { - "type": "integer" - }, - "positiveOffset": { - "type": "integer" - }, - "zeroCount": { - "type": "long" - }, - "scale": { - "type": "long" - }, - "max": { - "type": "float" - }, - "min": { - "type": "float" - }, - "sum": { - "type": "float" - }, - "count": { - "type": "long" - }, - "exemplar": { - "properties": { - "time": { - "type": "date" - }, - "traceId": { - "ignore_above": 256, - "type": "keyword" - }, - "spanId": { - "ignore_above": 256, - "type": "keyword" - } - } - }, - "instrumentationScope": { - "properties": { - "name": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 128 - } - } - }, - "version": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "droppedAttributesCount": { - "type": "integer" - }, - "schemaUrl": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - }, - "schemaUrl": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - }, - "settings": { - "index": { - "mapping": { - "total_fields": { - "limit": 10000 - } - }, - "refresh_interval": "5s" - } - } - }, - "version": 1, - "_meta": { - "description": "Observability Metrics Mapping Template", - "catalog": "observability", - "type": "metrics", - "correlations" : [ - { - "field": "spanId", - "foreign-schema" : "traces", - "foreign-field" : "spanId" - }, - { - "field": "traceId", - "foreign-schema" : "traces", - "foreign-field" : "traceId" - } - ] - } - } - -traces_template = { - "index_patterns": [ - "sso_traces-*-*" - ], - "data_stream": {}, - "template": { - "mappings": { - "_meta": { - "version": "1.0.0", - "catalog": "observability", - "type": "traces", - "component": "trace", - "correlations": [ - { - "field": "serviceName", - "foreign-schema": "services", - "foreign-field": "spanId" - } - ] - }, - "dynamic_templates": [ - { - "attributes_map": { - "mapping": { - "type": "keyword" - }, - "path_match": "attributes.*" - } - }, - { - "events_attributes_map": { - "mapping": { - "type": "keyword" - }, - "path_match": "events.attributes.*" - } - }, - { - "links_attributes_map": { - "mapping": { - "type": "keyword" - }, - "path_match": "links.attributes.*" - } - }, - { - "instrumentation_scope_attributes_map": { - "mapping": { - "type": "keyword" - }, - "path_match": "instrumentationScope.attributes.*" - } - }, - { - "resources_map": { - "mapping": { - "type": "keyword" - }, - "path_match": "resource.*" - } - } - ], - "_source": { - "enabled": true - }, - "properties": { - "traceId": { - "ignore_above": 256, - "type": "keyword" - }, - "spanId": { - "ignore_above": 256, - "type": "keyword" - }, - "traceState": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "parentSpanId": { - "ignore_above": 256, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "kind": { - "ignore_above": 128, - "type": "keyword" - }, - "startTime": { - "type": "date_nanos" - }, - "endTime": { - "type": "date_nanos" - }, - "droppedAttributesCount": { - "type": "long" - }, - "droppedEventsCount": { - "type": "long" - }, - "droppedLinksCount": { - "type": "long" - }, - "status": { - "properties": { - "code": { - "type": "long" - }, - "message": { - "ignore_above": 128, - "type": "keyword" - } - } - }, - "attributes": { - "type": "object", - "properties": { - "serviceName": { - "type": "keyword" - }, - "data_stream": { - "properties": { - "dataset": { - "ignore_above": 128, - "type": "keyword" - }, - "namespace": { - "ignore_above": 128, - "type": "keyword" - }, - "type": { - "ignore_above": 56, - "type": "keyword" - } - } - } - } - }, - "events": { - "type": "nested", - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "@timestamp": { - "type": "date_nanos" - }, - "observedTimestamp": { - "type": "date_nanos" - } - } - }, - "links": { - "type": "nested", - "properties": { - "traceId": { - "ignore_above": 256, - "type": "keyword" - }, - "spanId": { - "ignore_above": 256, - "type": "keyword" - }, - "traceState": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - }, - "instrumentationScope": { - "properties": { - "name": { - "type": "keyword" - }, - "version": { - "type": "keyword" - }, - "droppedAttributesCount": { - "type": "integer" - }, - "schemaUrl": { - "type": "keyword" - } - } - }, - "schemaUrl": { - "type": "keyword" - } - } - } - }, - "composed_of": [ - "tracegroup_template" - ], - "version": 1, - "_meta": { - "description": "Observability Traces Mapping Template", - "catalog": "observability", - "type": "traces", - "correlations": [ - { - "field": "serviceName", - "foreign-schema": "services", - "foreign-field": "spanId" - } - ] - } - } - -services_template = { - "index_patterns": [ - "sso_services-*-*" - ], - "template": { - "mappings": { - "_meta": { - "version": "1.0.0", - "catalog": "observability", - "type": "traces", - "component": "services", - "correlations": [ - { - "field": "traceGroupName", - "foreign-schema": "traceGroups", - "foreign-field": "traceGroup" - } - ] - }, - "_source": { - "enabled": true - }, - "dynamic_templates": [ - { - "attributes_map": { - "mapping": { - "type": "keyword" - }, - "path_match": "attributes.*" - } - } - ], - "properties": { - "destination": { - "properties": { - "domain": { - "type": "keyword", - "ignore_above": 1024 - }, - "resource": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "hashId": { - "type": "keyword", - "ignore_above": 1024 - }, - "serviceName": { - "type": "keyword", - "ignore_above": 1024 - }, - "kind": { - "type": "keyword", - "ignore_above": 1024 - }, - "target": { - "properties": { - "domain": { - "type": "keyword", - "ignore_above": 1024 - }, - "resource": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "traceGroupName": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "settings": { - "index": { - "mapping": { - "total_fields": { - "limit": 10000 - } - }, - "refresh_interval": "5s" - } - } - }, - "composed_of": [ - ], - "version": 1, - "_meta": { - "description": "Simple Schema For Observability Service", - "catalog": "observability", - "type": "services", - "correlations": [ - { - "field": "traceGroupName", - "foreign-schema": "traceGroups", - "foreign-field": "traceGroup" - } - ] - } - } - [datastreams] observability = sso_logs-nginx-prod diff --git a/src/integrations/src/install.py b/src/integrations/src/install.py index 51a20dae26..1b420f4d2e 100644 --- a/src/integrations/src/install.py +++ b/src/integrations/src/install.py @@ -162,7 +162,7 @@ def get_dist_version(auth): # Generic upsert api using general endpoint def upsert_obj(auth, items, api): for key in items: - payload = json.loads(items[key]) + payload = items[key] logger.info(f' calling: https://{opensearch_host}:9200/{api}/{key}') res = requests.put( url=f'https://{opensearch_host}:9200/{api}/{key}', @@ -180,6 +180,31 @@ def upsert_obj(auth, items, api): def output_message(key, res): return f'{key}: status={res.status_code}, message={res.text}' +# load component templates from opensearch-catalog and post them in the cluster +def fetch_and_post_component_templates(api, folder, mode='include', fileNames=[]): + url = f'https://api.github.com/repos/opensearch-project/opensearch-catalog/contents/schema/observability/{folder}' + headers = {'Accept': 'application/vnd.github.v3+json'} + response = requests.get(url, headers=headers) + files = response.json() + + for file in files: + base_name = file['name'].split('.')[0] # get the base filename without the suffix + logger.info(f' file: {base_name}') + if file['name'].endswith('.mapping'): + if mode == 'exclude' and base_name not in fileNames: + process_file(file, api) + elif mode == 'include' and base_name in fileNames: + process_file(file, api) + +def process_file(file,api): + file_url = file['download_url'] + file_content = requests.get(file_url).text + logger.info(f' loading json: {file_url}') + mapping = json.loads(file_content) + base_name = file['name'].split('.')[0] + template_name = f'{base_name}_template' + upsert_obj(auth, {template_name: mapping}, api) + # ---------------------------------------------------------------------------------------------------------------------- if __name__ == '__main__': # attempt connection to cluster @@ -188,19 +213,19 @@ def output_message(key, res): dist_name, domain_version = get_dist_version(auth) logger.info(f'dist_name: {dist_name}, domain_version: {domain_version}') - # load composable index template + # load composable template logger.info('Create/Update component index templates') - upsert_obj(auth, app_data['component-templates'], - api='_component_template') + fetch_and_post_component_templates("_component_template","logs","exclude",["logs"]) + fetch_and_post_component_templates("_component_template","traces","exclude",["traces","services"]) + fetch_and_post_component_templates("_component_template","metrics","exclude",["metrics"]) + # load index template logger.info('Create/Update index templates') - upsert_obj(auth, app_data['index-templates'], - api='_index_template') + fetch_and_post_component_templates("_index_template","logs","include",["logs"]) + fetch_and_post_component_templates("_index_template","traces","include",["traces","services"]) + fetch_and_post_component_templates("_index_template","metrics","include",["metrics"]) + # load data stream - logger.info('Create/Update index templates') - upsert_obj(auth, app_data['index-templates'], - api='_index_template') - # load data-streams logger.info('Create datastreams ') create_data_streams(auth, app_data['datastreams'], domain='observability')