From b6f8d89ccaf6cc7b8587c3bf3a587f1326e35705 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Wed, 9 Aug 2023 13:43:58 -0700 Subject: [PATCH] Backport ELB and VPC changes to 2.9 (#878) * Copy ELB fixes from main Signed-off-by: Simeon Widdis * Copy VPC fixes from main Signed-off-by: Simeon Widdis --------- Signed-off-by: Simeon Widdis --- .../repository/aws_elb/aws_elb-1.0.0.json | 6 +- .../schemas/logs_elb-1.0.0.mapping.json | 249 ++++++++++++++++++ .../aws_vpc_flow/aws_vpc_flow-1.0.0.json | 2 +- .../repository/aws_vpc_flow/info/README.md | 2 + .../schemas/aws_s3-1.0.0.mapping.json | 171 ++++++++++++ .../schemas/logs_vpc-1.0.0.mapping.json | 9 +- 6 files changed, 432 insertions(+), 7 deletions(-) create mode 100644 server/adaptors/integrations/__data__/repository/aws_elb/schemas/logs_elb-1.0.0.mapping.json create mode 100644 server/adaptors/integrations/__data__/repository/aws_vpc_flow/schemas/aws_s3-1.0.0.mapping.json diff --git a/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json index c7508d94a3..2f9a07f77e 100644 --- a/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json @@ -4,7 +4,7 @@ "displayName": "AWS ELB", "description": "AWS Elastic Load Balancer collector", "license": "Apache-2.0", - "type": "logs", + "type": "logs_elb", "author": "OpenSearch", "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_elb/info", "statics": { @@ -41,7 +41,7 @@ "version": "1.0.0" }, { - "name": "logs", + "name": "logs_elb", "version": "1.0.0" } ], @@ -54,4 +54,4 @@ "sampleData": { "path": "sample.json" } -} \ No newline at end of file +} diff --git a/server/adaptors/integrations/__data__/repository/aws_elb/schemas/logs_elb-1.0.0.mapping.json b/server/adaptors/integrations/__data__/repository/aws_elb/schemas/logs_elb-1.0.0.mapping.json new file mode 100644 index 0000000000..c4a75762f7 --- /dev/null +++ b/server/adaptors/integrations/__data__/repository/aws_elb/schemas/logs_elb-1.0.0.mapping.json @@ -0,0 +1,249 @@ +{ + "index_patterns": [ + "ss4o_logs-aws_elb-*" + ], + "data_stream": {}, + "template": { + "aliases": { + "logs-elb": {} + }, + "mappings": { + "_meta": { + "version": "1.0.0", + "catalog": "observability", + "type": "logs", + "component": "log", + "correlations": [ + { + "field": "spanId", + "foreign-schema": "traces", + "foreign-field": "spanId" + }, + { + "field": "traceId", + "foreign-schema": "traces", + "foreign-field": "traceId" + } + ] + }, + "_source": { + "enabled": true + }, + "dynamic_templates": [ + { + "resources_map": { + "mapping": { + "type": "keyword" + }, + "path_match": "resource.*" + } + }, + { + "attributes_map": { + "mapping": { + "type": "keyword" + }, + "path_match": "attributes.*" + } + }, + { + "instrumentation_scope_attributes_map": { + "mapping": { + "type": "keyword" + }, + "path_match": "instrumentationScope.attributes.*" + } + } + ], + "properties": { + "severity": { + "properties": { + "number": { + "type": "long" + }, + "text": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "attributes": { + "type": "object", + "properties": { + "data_stream": { + "properties": { + "dataset": { + "ignore_above": 128, + "type": "keyword" + }, + "namespace": { + "ignore_above": 128, + "type": "keyword" + }, + "type": { + "ignore_above": 56, + "type": "keyword" + } + } + } + } + }, + "body": { + "type": "text" + }, + "@message": { + "type": "alias", + "path": "body" + }, + "@timestamp": { + "type": "date" + }, + "observedTimestamp": { + "type": "date" + }, + "observerTime": { + "type": "alias", + "path": "observedTimestamp" + }, + "traceId": { + "ignore_above": 256, + "type": "keyword" + }, + "spanId": { + "ignore_above": 256, + "type": "keyword" + }, + "schemaUrl": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "instrumentationScope": { + "properties": { + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 128 + } + } + }, + "version": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "dropped_attributes_count": { + "type": "integer" + }, + "schemaUrl": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "event": { + "properties": { + "domain": { + "ignore_above": 256, + "type": "keyword" + }, + "name": { + "ignore_above": 256, + "type": "keyword" + }, + "source": { + "ignore_above": 256, + "type": "keyword" + }, + "category": { + "ignore_above": 256, + "type": "keyword" + }, + "type": { + "ignore_above": 256, + "type": "keyword" + }, + "kind": { + "ignore_above": 256, + "type": "keyword" + }, + "result": { + "ignore_above": 256, + "type": "keyword" + }, + "exception": { + "properties": { + "message": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 256, + "type": "keyword" + }, + "stacktrace": { + "type": "text" + } + } + } + } + } + } + }, + "settings": { + "index": { + "mapping": { + "total_fields": { + "limit": 10000 + } + }, + "refresh_interval": "5s" + } + } + }, + "composed_of": [ + "communication", + "http", + "cloud", + "aws_elb", + "url" + ], + "version": 1, + "_meta": { + "description": "Simple Schema For Observability", + "catalog": "observability", + "type": "logs", + "correlations": [ + { + "field": "spanId", + "foreign-schema": "traces", + "foreign-field": "spanId" + }, + { + "field": "traceId", + "foreign-schema": "traces", + "foreign-field": "traceId" + } + ] + } + } diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json index 86e8a9b030..cffc317efb 100644 --- a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json @@ -37,7 +37,7 @@ "version": "1.0.0" }, { - "name": "s3", + "name": "aws_s3", "version": "1.0.0" } ], diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/info/README.md b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/info/README.md index 6e55804662..af6635934a 100644 --- a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/info/README.md +++ b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/info/README.md @@ -21,4 +21,6 @@ An integration is a bundle of pre-canned assets which are bundled togather in a AWS VPC flow logs integration includes dashboards, visualisations, queries and an index mapping. ### Dashboards +The Dashboard uses the index alias `logs-vpc` for shortening the index name - be advised. + ![](../static/dashboard1.png) diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/schemas/aws_s3-1.0.0.mapping.json b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/schemas/aws_s3-1.0.0.mapping.json new file mode 100644 index 0000000000..60e519f369 --- /dev/null +++ b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/schemas/aws_s3-1.0.0.mapping.json @@ -0,0 +1,171 @@ +{ + "template": { + "mappings": { + "_meta": { + "version": "1.0.0", + "catalog": "observability", + "type": "logs", + "component": "aws_s3" + }, + "properties": { + "aws": { + "type" : "object", + "properties": { + "s3": { + "type" : "object", + "properties": { + "bucket_owner": { + "type": "keyword" + }, + "bucket": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "remote_ip": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "requester": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "request_id": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "operation": { + "type": "keyword" + }, + "key": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "copy_source": { + "type": "keyword" + }, + "upload_id": { + "type": "keyword" + }, + "delete": { + "type": "keyword" + }, + "part_number": { + "type": "keyword" + }, + "request_uri": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "http_status": { + "type": "keyword" + }, + "error_code": { + "type": "keyword" + }, + "bytes_sent": { + "type": "long" + }, + "object_size": { + "type": "long" + }, + "total_time": { + "type": "integer" + }, + "turn_around_time": { + "type": "integer" + }, + "referrer": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "user_agent": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "version_id": { + "type": "keyword" + }, + "host_id": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "signature_version": { + "type": "keyword" + }, + "cipher_suite": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "authentication_type": { + "type": "keyword" + }, + "host_header": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "tls_version": { + "type": "keyword" + } + } + } + } + } + } + } + } +} diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/schemas/logs_vpc-1.0.0.mapping.json b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/schemas/logs_vpc-1.0.0.mapping.json index 1ef1f7b00c..618fd27737 100644 --- a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/schemas/logs_vpc-1.0.0.mapping.json +++ b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/schemas/logs_vpc-1.0.0.mapping.json @@ -1,10 +1,13 @@ { "index_patterns": [ - "ss4o_logs-*-*" + "ss4o_logs-aws_vpc-*" ], "priority": 900, "data_stream": {}, "template": { + "aliases": { + "logs-vpc": {} + }, "mappings": { "_meta": { "version": "1.0.0", @@ -221,9 +224,9 @@ }, "composed_of": [ "aws_vpc_flow", + "aws_s3", "cloud", - "communication", - "s3" + "communication" ], "version": 1, "_meta": {