From 5506eb8e88057f5d73b56108ba5b7748f5b5cfea Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Tue, 15 Aug 2023 14:17:14 -0700 Subject: [PATCH 1/3] Update label handling on frontend + Refactors (#931) * Update label display on frontend Signed-off-by: Simeon Widdis * Clear type errors in added_integration.tsx Signed-off-by: Simeon Widdis * Remove unused imports Signed-off-by: Simeon Widdis * Add missing labels Signed-off-by: Simeon Widdis * Autoformat the whole repository Signed-off-by: Simeon Widdis * Revert large sample data changes Signed-off-by: Simeon Widdis * Undo _import autoescape Signed-off-by: Simeon Widdis --------- Signed-off-by: Simeon Widdis --- .../components/added_integration.tsx | 11 +- .../available_integration_card_view.tsx | 12 +- .../available_integration_overview_page.tsx | 4 +- .../available_integration_table.tsx | 9 +- .../integration_category_badge_group.tsx | 16 +- .../components/integration_details_panel.tsx | 4 +- .../repository/apache/apache-1.0.0.json | 83 +++-- .../repository/apache/info/INGESTION.md | 11 +- .../__data__/repository/apache/info/README.md | 3 + .../schemas/communication-1.0.0.mapping.json | 254 +++++++-------- .../apache/schemas/http-1.0.0.mapping.json | 308 +++++++++--------- .../schemas/logs_apache-1.0.0.mapping.json | 4 +- .../aws_cloudfront/aws_cloudfront-1.0.0.json | 2 +- .../repository/aws_cloudfront/info/README.md | 1 + .../schemas/aws_cloudfront-1.0.0.mapping.json | 2 +- .../schemas/aws_s3-1.0.0.mapping.json | 2 +- .../logs-aws_cloudfront-1.0.0.mapping.json | 12 +- .../aws_cloudtrail/aws_cloudtrail-1.0.0.json | 2 +- .../aws_cloudtrail/data/samples.json | 8 +- .../repository/aws_cloudtrail/info/README.md | 3 +- .../schemas/aws_cloudtrail-1.0.0.mapping.json | 2 +- .../schemas/aws_s3-1.0.0.mapping.json | 2 +- .../logs-aws_cloudtrail-1.0.0.mapping.json | 12 +- .../repository/aws_elb/aws_elb-1.0.0.json | 114 +++---- .../repository/aws_elb/info/INGESTION.md | 8 +- .../repository/aws_elb/info/README.md | 10 +- .../schemas/aws_elb-1.0.0.mapping.json | 2 +- .../schemas/communication-1.0.0.mapping.json | 254 +++++++-------- .../aws_elb/schemas/http-1.0.0.mapping.json | 308 +++++++++--------- .../schemas/logs_elb-1.0.0.mapping.json | 12 +- .../repository/aws_rds/assets/README.md | 14 +- .../repository/aws_rds/aws_rds-1.0.0.json | 2 +- .../repository/aws_rds/info/README.md | 9 +- .../schemas/aws_rds-1.0.0.mapping.json | 2 +- .../aws_rds/schemas/aws_s3-1.0.0.mapping.json | 2 +- .../schemas/logs_rds-1.0.0.mapping.json | 16 +- .../repository/aws_s3/aws_s3-1.0.0.json | 78 ++--- .../__data__/repository/aws_s3/info/README.md | 3 +- .../aws_s3/schemas/aws_s3-1.0.0.mapping.json | 2 +- .../aws_s3/schemas/logs_s3-1.0.0.mapping.json | 11 +- .../repository/aws_vpc_flow/assets/README.md | 43 +-- .../aws_vpc_flow/aws_vpc_flow-1.0.0.json | 2 +- .../repository/aws_vpc_flow/info/README.md | 9 +- .../schemas/aws_s3-1.0.0.mapping.json | 2 +- .../schemas/aws_vpc_flow-1.0.0.mapping.json | 2 +- .../schemas/communication-1.0.0.mapping.json | 4 +- .../schemas/logs_vpc-1.0.0.mapping.json | 13 +- .../repository/aws_waf/aws_waf-1.0.0.json | 2 +- .../repository/aws_waf/info/README.md | 1 + .../aws_waf/schemas/aws_s3-1.0.0.mapping.json | 2 +- .../schemas/aws_waf-1.0.0.mapping.json | 2 +- .../schemas/logs_waf-1.0.0.mapping.json | 12 +- .../__data__/repository/k8s/assets/README.md | 34 +- .../__data__/repository/k8s/info/README.md | 42 ++- .../repository/k8s/ingestion/README.md | 12 +- .../__data__/repository/k8s/k8s-1.0.0.json | 90 ++--- .../k8s/schemas/container-1.0.0.mapping.json | 2 +- .../k8s/schemas/k8s-1.0.0.mapping.json | 4 +- .../k8s/schemas/logs-k8s-1.0.0.mapping.json | 4 +- .../__data__/repository/nginx/info/README.md | 3 + .../repository/nginx/nginx-1.0.0.json | 83 +++-- .../schemas/communication-1.0.0.mapping.json | 254 +++++++-------- .../nginx/schemas/http-1.0.0.mapping.json | 308 +++++++++--------- .../nginx/schemas/logs-1.0.0.mapping.json | 11 +- server/adaptors/integrations/types.ts | 2 +- server/adaptors/integrations/validators.ts | 2 +- 66 files changed, 1249 insertions(+), 1310 deletions(-) diff --git a/public/components/integrations/components/added_integration.tsx b/public/components/integrations/components/added_integration.tsx index 2c466c9f19..48cc15160b 100644 --- a/public/components/integrations/components/added_integration.tsx +++ b/public/components/integrations/components/added_integration.tsx @@ -5,14 +5,10 @@ /* eslint-disable react-hooks/exhaustive-deps */ import { - EuiBadge, - EuiButton, EuiButtonIcon, EuiFlexGroup, EuiFlexItem, - EuiGlobalToastList, EuiHealth, - EuiIcon, EuiInMemoryTable, EuiLink, EuiOverlayMask, @@ -64,7 +60,7 @@ export function AddedIntegration(props: AddedIntegrationProps) { const [isModalVisible, setIsModalVisible] = useState(false); const [modalLayout, setModalLayout] = useState(); - const badge = (status) => { + const badge = (status: string) => { switch (status) { case 'available': return Active; @@ -136,7 +132,6 @@ export function AddedIntegration(props: AddedIntegrationProps) { { getModal(); @@ -173,7 +168,7 @@ export function AddedIntegration(props: AddedIntegrationProps) { const assets = data?.assets || []; - const renderAsset = (record) => { + const renderAsset = (record: any) => { switch (record.assetType) { case 'dashboard': return ( @@ -234,7 +229,7 @@ export function AddedIntegration(props: AddedIntegrationProps) { }, filters: [ { - type: 'field_value_selection', + type: 'field_value_selection' as const, field: 'assetType', name: 'Type', multiSelect: false, diff --git a/public/components/integrations/components/available_integration_card_view.tsx b/public/components/integrations/components/available_integration_card_view.tsx index b3e3880d6a..5c0ff6fd2c 100644 --- a/public/components/integrations/components/available_integration_card_view.tsx +++ b/public/components/integrations/components/available_integration_card_view.tsx @@ -9,17 +9,11 @@ import { EuiFlexGroup, EuiFlexItem, EuiSpacer, - EuiSearchBar, - EuiButton, EuiFieldSearch, - EuiSwitch, EuiButtonGroup, - EuiBadgeGroup, - EuiBadge, - EuiToolTip, } from '@elastic/eui'; import _ from 'lodash'; -import React, { useRef, useState } from 'react'; +import React, { useState } from 'react'; import { AvailableIntegrationsCardViewProps, AvailableIntegrationType, @@ -54,7 +48,7 @@ export function AvailableIntegrationsCardView(props: AvailableIntegrationsCardVi }, ]; - const onChangeIcons = (optionId) => { + const onChangeIcons = (optionId: string) => { setToggleIconIdSelected(optionId); if (optionId === '0') { props.setCardView(false); @@ -82,7 +76,7 @@ export function AvailableIntegrationsCardView(props: AvailableIntegrationsCardVi data-test-subj={`integration_card_${i.name.toLowerCase()}`} titleElement="span" onClick={() => (window.location.hash = `#/available/${i.name}`)} - footer={badges(i.components)} + footer={badges(i.labels ?? [])} /> ); diff --git a/public/components/integrations/components/available_integration_overview_page.tsx b/public/components/integrations/components/available_integration_overview_page.tsx index 6e8e4fe56d..e858daa549 100644 --- a/public/components/integrations/components/available_integration_overview_page.tsx +++ b/public/components/integrations/components/available_integration_overview_page.tsx @@ -25,6 +25,7 @@ import { useToast } from '../../../../public/components/common/toast'; import { HttpStart } from '../../../../../../src/core/public'; export interface AvailableIntegrationType { + labels?: string[]; name: string; description: string; assetUrl?: string | undefined; @@ -117,8 +118,7 @@ export function AvailableIntegrationOverviewPage(props: AvailableIntegrationOver setData(exists.data); let newItems = exists.data.hits - .flatMap((hit: { components: Array<{ name: string }> }) => hit.components) - .map((component: { name: string }) => component.name); + .flatMap((hit: { labels?: string[] }) => hit.labels ?? []); newItems = [...new Set(newItems)].sort().map((newItem) => { return { name: newItem, diff --git a/public/components/integrations/components/available_integration_table.tsx b/public/components/integrations/components/available_integration_table.tsx index 1d336b8bb8..57bb24978e 100644 --- a/public/components/integrations/components/available_integration_table.tsx +++ b/public/components/integrations/components/available_integration_table.tsx @@ -5,18 +5,11 @@ import { EuiButtonGroup, - EuiFieldSearch, - EuiFilterButton, - EuiFilterGroup, - EuiFilterSelectItem, EuiFlexGroup, EuiFlexItem, - EuiIcon, EuiInMemoryTable, EuiLink, EuiPageContent, - EuiPopover, - EuiPopoverTitle, EuiSpacer, EuiTableFieldDataColumnType, EuiText, @@ -44,7 +37,7 @@ export function AvailableIntegrationsTable(props: AvailableIntegrationsTableProp const [toggleIconIdSelected, setToggleIconIdSelected] = useState('0'); - const onChangeIcons = (optionId) => { + const onChangeIcons = (optionId: string) => { setToggleIconIdSelected(optionId); if (optionId === '0') { props.setCardView(false); diff --git a/public/components/integrations/components/integration_category_badge_group.tsx b/public/components/integrations/components/integration_category_badge_group.tsx index 8635c6270e..dd94cb3e3d 100644 --- a/public/components/integrations/components/integration_category_badge_group.tsx +++ b/public/components/integrations/components/integration_category_badge_group.tsx @@ -6,24 +6,24 @@ import { EuiBadge, EuiBadgeGroup, EuiToolTip } from '@elastic/eui'; import React from 'react'; -export const badges = (categories) => { - if (categories.length <= 3) { +export const badges = (labels: string[]) => { + if (labels.length <= 3) { return ( - {categories.map((cateogry) => { - return {cateogry.name}; + {labels.map((label) => { + return {label}; })} ); } else { - const tooltip = `+${categories.length - 2} more`; + const tooltip = `+${labels.length - 2} more`; return ( - {categories[0].name} - {categories[1].name} + {labels[0]} + {labels[1]} (index ? ', ' : '') + item.name)} + content={labels.slice(2).map((item: string, index: number) => (index ? ', ' : '') + item)} >

{tooltip}

diff --git a/public/components/integrations/components/integration_details_panel.tsx b/public/components/integrations/components/integration_details_panel.tsx index 0f116a8a21..704e22f5d6 100644 --- a/public/components/integrations/components/integration_details_panel.tsx +++ b/public/components/integrations/components/integration_details_panel.tsx @@ -43,8 +43,8 @@ export function IntegrationDetails(props: any) { - {config.components.map((cateogry) => { - return {cateogry.name}; + {config.labels?.map((label: string) => { + return {label}; })} diff --git a/server/adaptors/integrations/__data__/repository/apache/apache-1.0.0.json b/server/adaptors/integrations/__data__/repository/apache/apache-1.0.0.json index 53c3c4829c..80cbfc9061 100644 --- a/server/adaptors/integrations/__data__/repository/apache/apache-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/apache/apache-1.0.0.json @@ -1,49 +1,46 @@ { - "name": "apache", - "version": "1.0.0", - "displayName": "Apache Dashboard", - "description": "Apache web logs collector", - "license": "Apache-2.0", - "type": "logs_apache", - "labels": [ - "log", - "communication", - "http"], + "name": "apache", + "version": "1.0.0", + "displayName": "Apache Dashboard", + "description": "Apache web logs collector", + "license": "Apache-2.0", + "type": "logs_apache", + "labels": ["log", "communication", "http"], "author": "OpenSearch", - "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/apache/info", - "statics": { - "logo": { - "annotation": "Apache Logo", - "path": "logo.png" - }, - "gallery": [ - { - "annotation": "Apache Dashboard", - "path": "dashboard1.png" - } - ] + "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/apache/info", + "statics": { + "logo": { + "annotation": "Apache Logo", + "path": "logo.png" }, - "components": [ - { - "name": "communication", - "version": "1.0.0" - }, - { - "name": "http", - "version": "1.0.0" - }, - { - "name": "logs_apache", - "version": "1.0.0" - } - ], - "assets": { - "savedObjects": { - "name": "apache", - "version": "1.0.0" - } + "gallery": [ + { + "annotation": "Apache Dashboard", + "path": "dashboard1.png" + } + ] + }, + "components": [ + { + "name": "communication", + "version": "1.0.0" }, - "sampleData": { - "path": "sample.json" + { + "name": "http", + "version": "1.0.0" + }, + { + "name": "logs_apache", + "version": "1.0.0" + } + ], + "assets": { + "savedObjects": { + "name": "apache", + "version": "1.0.0" } + }, + "sampleData": { + "path": "sample.json" + } } diff --git a/server/adaptors/integrations/__data__/repository/apache/info/INGESTION.md b/server/adaptors/integrations/__data__/repository/apache/info/INGESTION.md index f9f64bcdd6..27ff56ea92 100644 --- a/server/adaptors/integrations/__data__/repository/apache/info/INGESTION.md +++ b/server/adaptors/integrations/__data__/repository/apache/info/INGESTION.md @@ -1,10 +1,13 @@ # Ingestion Pipeline -To set up an ingestion pipeline, I used Docker to run Fluent-bit and an Apache fake log generator, along with an instance of OpenSearch. + +To set up an ingestion pipeline, I used Docker to run Fluent-bit and an Apache fake log generator, along with an instance of OpenSearch. ## FluentBit and OpenSearch Setup + First, create a `docker-compose.yml` instance like [this](https://github.com/opensearch-project/data-prepper/blob/93d06db5cad280e2e4c53e12dfb47c7cbaa7b364/examples/log-ingestion/docker-compose.yaml). This will pull FluentBit and OpenSearch Docker images and run them in `opensearch-net` Docker network. Next, use an Apache log generator to generate sample logs. I used `mingrammer/flog` in my `docker-compose.yml` file: + ``` apache: image: mingrammer/flog @@ -23,6 +26,7 @@ Next, use an Apache log generator to generate sample logs. I used `mingrammer/fl ``` Then, create a `fluent-bit.conf` as follows: + ``` [SERVICE] Parsers_File parsers.conf @@ -51,9 +55,11 @@ Then, create a `fluent-bit.conf` as follows: Index ss4o_logs-apache-prod-sample Suppress_Type_Name On ``` + This creates a Fluent-Bit pipeline that forwards your generated apache logs through a parser to OpenSearch. Create a `parsers.conf` file as follows: + ``` [PARSER] Name apache @@ -66,6 +72,7 @@ Create a `parsers.conf` file as follows: You can also use a [GeoIP2 Filter](https://docs.fluentbit.io/manual/pipeline/filters/geoip2-filter) to enrich the data with geolocation data. Finally, I used a `otel-converter.lua` script to convert the parsed data into schema-compliant data: + ```lua local hexCharset = "0123456789abcdef" local function randHex(length) @@ -96,7 +103,7 @@ function convert_to_otel(tag, timestamp, record) if tag=="apache.access" then record.remote=record.host end - + local data = { traceId=randHex(32), spanId=randHex(16), diff --git a/server/adaptors/integrations/__data__/repository/apache/info/README.md b/server/adaptors/integrations/__data__/repository/apache/info/README.md index 1bc564fac2..eb85135a5f 100644 --- a/server/adaptors/integrations/__data__/repository/apache/info/README.md +++ b/server/adaptors/integrations/__data__/repository/apache/info/README.md @@ -1,14 +1,17 @@ # Apache Integration ## What is Apache ? + Apache is an open source web server software for modern operating systems including UNIX and Windows. See additional details [here](https://httpd.apache.org/). ## What is Apache Integration ? + An integration is a bundle of pre-canned assets which are bundled togather in a meaningful manner. Apache integration includes dashboards, visualisations, queries and an index mapping. ### Dashboards + ![](../static/dashboard1.png) diff --git a/server/adaptors/integrations/__data__/repository/apache/schemas/communication-1.0.0.mapping.json b/server/adaptors/integrations/__data__/repository/apache/schemas/communication-1.0.0.mapping.json index 97263ad264..d9af5d7193 100644 --- a/server/adaptors/integrations/__data__/repository/apache/schemas/communication-1.0.0.mapping.json +++ b/server/adaptors/integrations/__data__/repository/apache/schemas/communication-1.0.0.mapping.json @@ -1,135 +1,135 @@ { - "template": { - "mappings": { - "_meta": { - "version": "1.0.0", - "catalog": "observability", - "type": "logs", - "component": "communication", - "labels": ["communication"] + "template": { + "mappings": { + "_meta": { + "version": "1.0.0", + "catalog": "observability", + "type": "logs", + "component": "communication", + "labels": ["communication"] + }, + "properties": { + "communication": { + "properties": { + "sock.family": { + "type": "keyword", + "ignore_above": 256 }, - "properties": { - "communication": { - "properties": { - "sock.family": { - "type": "keyword", - "ignore_above": 256 - }, - "source": { - "type": "object", - "properties": { - "address": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "domain": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "bytes": { - "type": "long" - }, - "ip": { - "type": "ip" - }, - "port": { - "type": "long" - }, - "mac": { - "type": "keyword", - "ignore_above": 1024 - }, - "packets": { - "type": "long" - }, - "geo": { - "type": "object", - "properties": { - "city_name": { - "type": "keyword" - }, - "country_iso_code": { - "type": "keyword" - }, - "country_name": { - "type": "keyword" - }, - "location": { - "type": "geo_point" - } - } - } - } - }, - "destination": { - "type": "object", - "properties": { - "address": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "domain": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "bytes": { - "type": "long" - }, - "ip": { - "type": "ip" - }, - "port": { - "type": "long" - }, - "mac": { - "type": "keyword", - "ignore_above": 1024 - }, - "packets": { - "type": "long" - }, - "geo": { - "type": "object", - "properties": { - "city_name": { - "type": "keyword" - }, - "country_iso_code": { - "type": "keyword" - }, - "country_name": { - "type": "keyword" - }, - "location": { - "type": "geo_point" - } - } - } - } - } + "source": { + "type": "object", + "properties": { + "address": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 1024 } + } + }, + "domain": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "bytes": { + "type": "long" + }, + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + }, + "mac": { + "type": "keyword", + "ignore_above": 1024 + }, + "packets": { + "type": "long" + }, + "geo": { + "type": "object", + "properties": { + "city_name": { + "type": "keyword" + }, + "country_iso_code": { + "type": "keyword" + }, + "country_name": { + "type": "keyword" + }, + "location": { + "type": "geo_point" + } + } + } + } + }, + "destination": { + "type": "object", + "properties": { + "address": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "domain": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "bytes": { + "type": "long" + }, + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + }, + "mac": { + "type": "keyword", + "ignore_above": 1024 + }, + "packets": { + "type": "long" + }, + "geo": { + "type": "object", + "properties": { + "city_name": { + "type": "keyword" + }, + "country_iso_code": { + "type": "keyword" + }, + "country_name": { + "type": "keyword" + }, + "location": { + "type": "geo_point" + } + } } + } } + } } + } } + } } diff --git a/server/adaptors/integrations/__data__/repository/apache/schemas/http-1.0.0.mapping.json b/server/adaptors/integrations/__data__/repository/apache/schemas/http-1.0.0.mapping.json index 8a1e18f04a..5fec510cb8 100644 --- a/server/adaptors/integrations/__data__/repository/apache/schemas/http-1.0.0.mapping.json +++ b/server/adaptors/integrations/__data__/repository/apache/schemas/http-1.0.0.mapping.json @@ -1,166 +1,166 @@ { - "template": { - "mappings": { - "_meta": { - "version": "1.0.0", - "catalog": "observability", - "type": "logs", - "component": "http", - "labels": ["http"] + "template": { + "mappings": { + "_meta": { + "version": "1.0.0", + "catalog": "observability", + "type": "logs", + "component": "http", + "labels": ["http"] + }, + "dynamic_templates": [ + { + "request_header_map": { + "mapping": { + "type": "keyword" }, - "dynamic_templates": [ - { - "request_header_map": { - "mapping": { - "type": "keyword" - }, - "path_match": "request.header.*" + "path_match": "request.header.*" + } + }, + { + "response_header_map": { + "mapping": { + "type": "keyword" + }, + "path_match": "response.header.*" + } + } + ], + "properties": { + "http": { + "properties": { + "flavor": { + "type": "keyword", + "ignore_above": 256 + }, + "user_agent": { + "type": "object", + "properties": { + "original": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "version": { + "type": "keyword" + }, + "device": { + "type": "object", + "properties": { + "name": { + "type": "keyword" } + } }, - { - "response_header_map": { - "mapping": { - "type": "keyword" - }, - "path_match": "response.header.*" + "os": { + "type": "object", + "properties": { + "type": { + "type": "keyword" + }, + "platform": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "full": { + "type": "keyword" + }, + "family": { + "type": "keyword" + }, + "version": { + "type": "keyword" + }, + "kernel": { + "type": "keyword" + } + } + } + } + }, + "url": { + "type": "keyword", + "ignore_above": 2048 + }, + "schema": { + "type": "keyword", + "ignore_above": 1024 + }, + "target": { + "type": "keyword", + "ignore_above": 1024 + }, + "route": { + "type": "keyword", + "ignore_above": 1024 + }, + "client.ip": { + "type": "ip" + }, + "resent_count": { + "type": "integer" + }, + "request": { + "type": "object", + "properties": { + "id": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 } + } + }, + "body.content": { + "type": "text" + }, + "bytes": { + "type": "long" + }, + "method": { + "type": "keyword", + "ignore_above": 256 + }, + "referrer": { + "type": "keyword", + "ignore_above": 1024 + }, + "mime_type": { + "type": "keyword", + "ignore_above": 1024 } - ], - "properties": { - "http": { - "properties": { - "flavor": { - "type": "keyword", - "ignore_above": 256 - }, - "user_agent": { - "type": "object", - "properties": { - "original": { - "type": "keyword" - }, - "name": { - "type": "keyword" - }, - "version": { - "type": "keyword" - }, - "device": { - "type": "object", - "properties": { - "name": { - "type": "keyword" - } - } - }, - "os": { - "type": "object", - "properties": { - "type": { - "type": "keyword" - }, - "platform": { - "type": "keyword" - }, - "name": { - "type": "keyword" - }, - "full": { - "type": "keyword" - }, - "family": { - "type": "keyword" - }, - "version": { - "type": "keyword" - }, - "kernel": { - "type": "keyword" - } - } - } - } - }, - "url": { - "type": "keyword", - "ignore_above": 2048 - }, - "schema": { - "type": "keyword", - "ignore_above": 1024 - }, - "target": { - "type": "keyword", - "ignore_above": 1024 - }, - "route": { - "type": "keyword", - "ignore_above": 1024 - }, - "client.ip": { - "type": "ip" - }, - "resent_count": { - "type": "integer" - }, - "request": { - "type": "object", - "properties": { - "id": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "body.content": { - "type": "text" - }, - "bytes": { - "type": "long" - }, - "method": { - "type": "keyword", - "ignore_above": 256 - }, - "referrer": { - "type": "keyword", - "ignore_above": 1024 - }, - "mime_type": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "response": { - "type": "object", - "properties": { - "id": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "body.content": { - "type": "text" - }, - "bytes": { - "type": "long" - }, - "status_code": { - "type": "integer" - } - } - } + } + }, + "response": { + "type": "object", + "properties": { + "id": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 } + } + }, + "body.content": { + "type": "text" + }, + "bytes": { + "type": "long" + }, + "status_code": { + "type": "integer" } + } } + } } + } } + } } diff --git a/server/adaptors/integrations/__data__/repository/apache/schemas/logs_apache-1.0.0.mapping.json b/server/adaptors/integrations/__data__/repository/apache/schemas/logs_apache-1.0.0.mapping.json index d9800c7739..7c9309c2a4 100644 --- a/server/adaptors/integrations/__data__/repository/apache/schemas/logs_apache-1.0.0.mapping.json +++ b/server/adaptors/integrations/__data__/repository/apache/schemas/logs_apache-1.0.0.mapping.json @@ -1,7 +1,5 @@ { - "index_patterns": [ - "ss4o_logs-apache-*" - ], + "index_patterns": ["ss4o_logs-apache-*"], "data_stream": {}, "template": { "aliases": { diff --git a/server/adaptors/integrations/__data__/repository/aws_cloudfront/aws_cloudfront-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_cloudfront/aws_cloudfront-1.0.0.json index 1f44ebd417..5a1c526e55 100644 --- a/server/adaptors/integrations/__data__/repository/aws_cloudfront/aws_cloudfront-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/aws_cloudfront/aws_cloudfront-1.0.0.json @@ -5,7 +5,7 @@ "description": "AWS cloudfront Object Store", "license": "Apache-2.0", "type": "logs-aws_cloudfront", - "labels": ["log","aws","s3","cloud","cloudfront"], + "labels": ["log", "aws", "s3", "cloud", "cloudfront"], "author": "OpenSearch", "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_cloudfront/info", "statics": { diff --git a/server/adaptors/integrations/__data__/repository/aws_cloudfront/info/README.md b/server/adaptors/integrations/__data__/repository/aws_cloudfront/info/README.md index 9eae53903c..0459813357 100644 --- a/server/adaptors/integrations/__data__/repository/aws_cloudfront/info/README.md +++ b/server/adaptors/integrations/__data__/repository/aws_cloudfront/info/README.md @@ -13,6 +13,7 @@ An integration is a bundle of pre-canned assets which are brought together in a AWS CloudFront integration includes dashboards, visualizations, queries, and an index mapping. ### Dashboards + The Dashboard uses the index alias `logs-aws-cloudfront` for shortening the index name - be advised. ![AWS CloudFront Dashboard](../static/dashboard.png) diff --git a/server/adaptors/integrations/__data__/repository/aws_cloudfront/schemas/aws_cloudfront-1.0.0.mapping.json b/server/adaptors/integrations/__data__/repository/aws_cloudfront/schemas/aws_cloudfront-1.0.0.mapping.json index fa54bb36ed..da6558aa1e 100644 --- a/server/adaptors/integrations/__data__/repository/aws_cloudfront/schemas/aws_cloudfront-1.0.0.mapping.json +++ b/server/adaptors/integrations/__data__/repository/aws_cloudfront/schemas/aws_cloudfront-1.0.0.mapping.json @@ -6,7 +6,7 @@ "catalog": "observability", "type": "logs", "component": "aws_cloudfront", - "labels": ["aws","cloudfront"] + "labels": ["aws", "cloudfront"] }, "properties": { "aws": { diff --git a/server/adaptors/integrations/__data__/repository/aws_cloudfront/schemas/aws_s3-1.0.0.mapping.json b/server/adaptors/integrations/__data__/repository/aws_cloudfront/schemas/aws_s3-1.0.0.mapping.json index f948597fc2..8057cd98ab 100644 --- a/server/adaptors/integrations/__data__/repository/aws_cloudfront/schemas/aws_s3-1.0.0.mapping.json +++ b/server/adaptors/integrations/__data__/repository/aws_cloudfront/schemas/aws_s3-1.0.0.mapping.json @@ -6,7 +6,7 @@ "catalog": "observability", "type": "logs", "component": "s3", - "labels": ["aws","s3"] + "labels": ["aws", "s3"] }, "properties": { "aws": { diff --git a/server/adaptors/integrations/__data__/repository/aws_cloudfront/schemas/logs-aws_cloudfront-1.0.0.mapping.json b/server/adaptors/integrations/__data__/repository/aws_cloudfront/schemas/logs-aws_cloudfront-1.0.0.mapping.json index f04ddcb8fa..aee3e31d37 100644 --- a/server/adaptors/integrations/__data__/repository/aws_cloudfront/schemas/logs-aws_cloudfront-1.0.0.mapping.json +++ b/server/adaptors/integrations/__data__/repository/aws_cloudfront/schemas/logs-aws_cloudfront-1.0.0.mapping.json @@ -1,7 +1,5 @@ { - "index_patterns": [ - "ss4o_logs-aws_cloudfront-*" - ], + "index_patterns": ["ss4o_logs-aws_cloudfront-*"], "priority": 900, "data_stream": {}, "template": { @@ -14,7 +12,7 @@ "catalog": "observability", "type": "logs", "component": "log", - "labels": ["log","aws","s3","cloud","cloudfront"], + "labels": ["log", "aws", "s3", "cloud", "cloudfront"], "correlations": [ { "field": "spanId", @@ -223,10 +221,6 @@ } } }, - "composed_of": [ - "aws_s3", - "aws_cloudfront", - "cloud" - ], + "composed_of": ["aws_s3", "aws_cloudfront", "cloud"], "version": 1 } diff --git a/server/adaptors/integrations/__data__/repository/aws_cloudtrail/aws_cloudtrail-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_cloudtrail/aws_cloudtrail-1.0.0.json index 6bcadeb952..3964ae11c9 100644 --- a/server/adaptors/integrations/__data__/repository/aws_cloudtrail/aws_cloudtrail-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/aws_cloudtrail/aws_cloudtrail-1.0.0.json @@ -5,7 +5,7 @@ "description": "AWS CloudTrail log collector", "license": "Apache-2.0", "type": "logs-aws_cloudtrail", - "labels": ["log","aws","s3","cloud","cloudtrail"], + "labels": ["log", "aws", "s3", "cloud", "cloudtrail"], "author": "OpenSearch", "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_cloudtrail/info", "statics": { diff --git a/server/adaptors/integrations/__data__/repository/aws_cloudtrail/data/samples.json b/server/adaptors/integrations/__data__/repository/aws_cloudtrail/data/samples.json index 16189d2603..12c62732d6 100644 --- a/server/adaptors/integrations/__data__/repository/aws_cloudtrail/data/samples.json +++ b/server/adaptors/integrations/__data__/repository/aws_cloudtrail/data/samples.json @@ -1536,9 +1536,7 @@ "instanceIds": [], "documentName": "AWS-RunShellScript", "parameters": { - "commands": [ - "curl -s http://127.0.0.1:2022/api/v1/health" - ] + "commands": ["curl -s http://127.0.0.1:2022/api/v1/health"] }, "interactive": false }, @@ -1550,9 +1548,7 @@ "comment": "", "expiresAfter": "Dec 3, 2021 12:11:27 AM", "parameters": { - "commands": [ - "curl -s http://127.0.0.1:2022/api/v1/health" - ] + "commands": ["curl -s http://127.0.0.1:2022/api/v1/health"] }, "instanceIds": [], "targets": [], diff --git a/server/adaptors/integrations/__data__/repository/aws_cloudtrail/info/README.md b/server/adaptors/integrations/__data__/repository/aws_cloudtrail/info/README.md index e270ed87f3..ebe70b8a7f 100644 --- a/server/adaptors/integrations/__data__/repository/aws_cloudtrail/info/README.md +++ b/server/adaptors/integrations/__data__/repository/aws_cloudtrail/info/README.md @@ -24,6 +24,7 @@ An integration is a set of pre-configured assets which are bundled together in a AWS CloudTrail log integration includes dashboards, visualizations, queries, and an index mapping. ### Dashboards -The Dashboard uses the index alias `logs-cloudtrail` for shortening the index name - be advised. + +The Dashboard uses the index alias `logs-cloudtrail` for shortening the index name - be advised. ![Dashboard](../static/dashboard.png) diff --git a/server/adaptors/integrations/__data__/repository/aws_cloudtrail/schemas/aws_cloudtrail-1.0.0.mapping.json b/server/adaptors/integrations/__data__/repository/aws_cloudtrail/schemas/aws_cloudtrail-1.0.0.mapping.json index 0642fcf476..11e5eb3796 100644 --- a/server/adaptors/integrations/__data__/repository/aws_cloudtrail/schemas/aws_cloudtrail-1.0.0.mapping.json +++ b/server/adaptors/integrations/__data__/repository/aws_cloudtrail/schemas/aws_cloudtrail-1.0.0.mapping.json @@ -6,7 +6,7 @@ "catalog": "observability", "type": "logs", "component": "aws_cloudtrail", - "labels": ["aws","cloudtrail"] + "labels": ["aws", "cloudtrail"] }, "properties": { "aws": { diff --git a/server/adaptors/integrations/__data__/repository/aws_cloudtrail/schemas/aws_s3-1.0.0.mapping.json b/server/adaptors/integrations/__data__/repository/aws_cloudtrail/schemas/aws_s3-1.0.0.mapping.json index 06f19e760c..fc0ffc2258 100644 --- a/server/adaptors/integrations/__data__/repository/aws_cloudtrail/schemas/aws_s3-1.0.0.mapping.json +++ b/server/adaptors/integrations/__data__/repository/aws_cloudtrail/schemas/aws_s3-1.0.0.mapping.json @@ -6,7 +6,7 @@ "catalog": "observability", "type": "logs", "component": "s3", - "labels": ["aws","s3"] + "labels": ["aws", "s3"] }, "properties": { "aws": { diff --git a/server/adaptors/integrations/__data__/repository/aws_cloudtrail/schemas/logs-aws_cloudtrail-1.0.0.mapping.json b/server/adaptors/integrations/__data__/repository/aws_cloudtrail/schemas/logs-aws_cloudtrail-1.0.0.mapping.json index 744fb46e30..013792ee4e 100644 --- a/server/adaptors/integrations/__data__/repository/aws_cloudtrail/schemas/logs-aws_cloudtrail-1.0.0.mapping.json +++ b/server/adaptors/integrations/__data__/repository/aws_cloudtrail/schemas/logs-aws_cloudtrail-1.0.0.mapping.json @@ -1,7 +1,5 @@ { - "index_patterns": [ - "ss4o_logs-aws_cloudtrail-*" - ], + "index_patterns": ["ss4o_logs-aws_cloudtrail-*"], "priority": 900, "data_stream": {}, "template": { @@ -14,7 +12,7 @@ "catalog": "observability", "type": "logs", "component": "log", - "labels": ["log","aws","s3","cloud","cloudtrail"], + "labels": ["log", "aws", "s3", "cloud", "cloudtrail"], "correlations": [ { "field": "spanId", @@ -223,10 +221,6 @@ } } }, - "composed_of": [ - "cloud", - "aws_cloudtrail", - "aws_s3" - ], + "composed_of": ["cloud", "aws_cloudtrail", "aws_s3"], "version": 1 } diff --git a/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json index 3731287f09..3a8a58d291 100644 --- a/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json @@ -1,66 +1,58 @@ { - "name": "aws_elb", - "version": "1.0.0", - "displayName": "AWS ELB", - "description": "AWS Elastic Load Balancer collector", - "license": "Apache-2.0", - "type": "logs_elb", - "labels": [ - "log", - "aws", - "communication", - "http", - "cloud", - "elb", - "url" - ], - "author": "OpenSearch", - "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_elb/info", - "statics": { - "logo": { - "annotation": "ELB Logo", - "path": "logo.svg" - }, - "gallery": [ - { - "annotation": "ELB Dashboard", - "path": "dashboard1.png" - } - ] + "name": "aws_elb", + "version": "1.0.0", + "displayName": "AWS ELB", + "description": "AWS Elastic Load Balancer collector", + "license": "Apache-2.0", + "type": "logs_elb", + "labels": ["log", "aws", "communication", "http", "cloud", "elb", "url"], + "author": "OpenSearch", + "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_elb/info", + "statics": { + "logo": { + "annotation": "ELB Logo", + "path": "logo.svg" }, - "components": [ - { - "name": "communication", - "version": "1.0.0" - }, - { - "name": "http", - "version": "1.0.0" - }, - { - "name": "cloud", - "version": "1.0.0" - }, - { - "name": "aws_elb", - "version": "1.0.0" - }, - { - "name": "url", - "version": "1.0.0" - }, - { - "name": "logs_elb", - "version": "1.0.0" - } - ], - "assets": { - "savedObjects": { - "name": "aws_elb", - "version": "1.0.0" - } + "gallery": [ + { + "annotation": "ELB Dashboard", + "path": "dashboard1.png" + } + ] + }, + "components": [ + { + "name": "communication", + "version": "1.0.0" }, - "sampleData": { - "path": "sample.json" + { + "name": "http", + "version": "1.0.0" + }, + { + "name": "cloud", + "version": "1.0.0" + }, + { + "name": "aws_elb", + "version": "1.0.0" + }, + { + "name": "url", + "version": "1.0.0" + }, + { + "name": "logs_elb", + "version": "1.0.0" + } + ], + "assets": { + "savedObjects": { + "name": "aws_elb", + "version": "1.0.0" } + }, + "sampleData": { + "path": "sample.json" + } } diff --git a/server/adaptors/integrations/__data__/repository/aws_elb/info/INGESTION.md b/server/adaptors/integrations/__data__/repository/aws_elb/info/INGESTION.md index 7f354ba017..459b2806db 100644 --- a/server/adaptors/integrations/__data__/repository/aws_elb/info/INGESTION.md +++ b/server/adaptors/integrations/__data__/repository/aws_elb/info/INGESTION.md @@ -1,15 +1,16 @@ # Sample Ingestion Pipeline -This is a brief overview of a sample ingestion pipeline for the AWS ELB integration. + +This is a brief overview of a sample ingestion pipeline for the AWS ELB integration. ## List of Components + - An OpenSearch domain running through Docker - A FluentBit agent running through Docker - An ELB instance generating logs (not covered in this sample) ### OpenSearch and FluentBit Setup -1. Look at [docker-compose.yaml]([docker-compose.yaml](https://github.com/opensearch-project/data-prepper/blob/93d06db5cad280e2e4c53e12dfb47c7cbaa7b364/examples/log-ingestion/docker-compose.yaml)https://github.com/opensearch-project/data-prepper/blob/93d06db5cad280e2e4c53e12dfb47c7cbaa7b364/examples/log-ingestion/docker-compose.yaml) to create FluentBit and OpenSearch Docker images and run them in the `opensearch-net` Docker network. - +1. Look at [docker-compose.yaml](<[docker-compose.yaml](https://github.com/opensearch-project/data-prepper/blob/93d06db5cad280e2e4c53e12dfb47c7cbaa7b364/examples/log-ingestion/docker-compose.yaml)https://github.com/opensearch-project/data-prepper/blob/93d06db5cad280e2e4c53e12dfb47c7cbaa7b364/examples/log-ingestion/docker-compose.yaml>) to create FluentBit and OpenSearch Docker images and run them in the `opensearch-net` Docker network. 2. Create the FluentBit as follows: ``` @@ -39,6 +40,7 @@ This is a brief overview of a sample ingestion pipeline for the AWS ELB integrat Index ss4o_logs-aws_elb-prod-sample Suppress_Type_Name On ``` + You would set INPUT as however you are ingesting your AWS ELB logs, more info [here](https://docs.fluentbit.io/manual/pipeline/inputs). 3. Create your `parsers.conf` as follows: diff --git a/server/adaptors/integrations/__data__/repository/aws_elb/info/README.md b/server/adaptors/integrations/__data__/repository/aws_elb/info/README.md index c4ff2dae15..b3affc4114 100644 --- a/server/adaptors/integrations/__data__/repository/aws_elb/info/README.md +++ b/server/adaptors/integrations/__data__/repository/aws_elb/info/README.md @@ -1,15 +1,16 @@ # AWS ELB Access Logs Integrations ## What is AWS ELB Access Logs ? + ELB Access Logs is a feature that allows you to capture information about requests sent to your load balancer. Access logs can help with a number of tasks, such as: -* Optimizing performance by showing response and processing times +- Optimizing performance by showing response and processing times -* Security analysis by monitoring unusual request patterns or user agents +- Security analysis by monitoring unusual request patterns or user agents -* Understanding traffic patterns and peak loads +- Understanding traffic patterns and peak loads While disabled by default, you can [enable storing access logs](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/enable-access-logging.html) for your load balancer in an AWS S3 bucket. @@ -20,10 +21,11 @@ Flow log data is collected outside of the path of your network traffic, and ther See additional details [here](https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs.html). ## What is AWS ELB Access Logs Integration ? + An integration is a bundle of pre-canned assets which are bundled togather in a meaningful manner. AWS ELB access logs integration includes dashboards, visualisations, queries and index mapping ### Dashboard -dashboard1 +dashboard1 diff --git a/server/adaptors/integrations/__data__/repository/aws_elb/schemas/aws_elb-1.0.0.mapping.json b/server/adaptors/integrations/__data__/repository/aws_elb/schemas/aws_elb-1.0.0.mapping.json index fdc1188e62..646857956f 100644 --- a/server/adaptors/integrations/__data__/repository/aws_elb/schemas/aws_elb-1.0.0.mapping.json +++ b/server/adaptors/integrations/__data__/repository/aws_elb/schemas/aws_elb-1.0.0.mapping.json @@ -6,7 +6,7 @@ "catalog": "observability", "type": "logs", "component": "aws_elb", - "labels": ["aws","elb"] + "labels": ["aws", "elb"] }, "properties": { "aws": { diff --git a/server/adaptors/integrations/__data__/repository/aws_elb/schemas/communication-1.0.0.mapping.json b/server/adaptors/integrations/__data__/repository/aws_elb/schemas/communication-1.0.0.mapping.json index d1793ee02d..d9af5d7193 100644 --- a/server/adaptors/integrations/__data__/repository/aws_elb/schemas/communication-1.0.0.mapping.json +++ b/server/adaptors/integrations/__data__/repository/aws_elb/schemas/communication-1.0.0.mapping.json @@ -1,135 +1,135 @@ { - "template": { - "mappings": { - "_meta": { - "version": "1.0.0", - "catalog": "observability", - "type": "logs", - "component": "communication", - "labels": ["communication"] + "template": { + "mappings": { + "_meta": { + "version": "1.0.0", + "catalog": "observability", + "type": "logs", + "component": "communication", + "labels": ["communication"] + }, + "properties": { + "communication": { + "properties": { + "sock.family": { + "type": "keyword", + "ignore_above": 256 }, - "properties": { - "communication": { - "properties": { - "sock.family": { - "type": "keyword", - "ignore_above": 256 - }, - "source": { - "type": "object", - "properties": { - "address": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "domain": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "bytes": { - "type": "long" - }, - "ip": { - "type": "ip" - }, - "port": { - "type": "long" - }, - "mac": { - "type": "keyword", - "ignore_above": 1024 - }, - "packets": { - "type": "long" - }, - "geo": { - "type": "object", - "properties": { - "city_name": { - "type": "keyword" - }, - "country_iso_code": { - "type": "keyword" - }, - "country_name": { - "type": "keyword" - }, - "location": { - "type": "geo_point" - } - } - } - } - }, - "destination": { - "type": "object", - "properties": { - "address": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "domain": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "bytes": { - "type": "long" - }, - "ip": { - "type": "ip" - }, - "port": { - "type": "long" - }, - "mac": { - "type": "keyword", - "ignore_above": 1024 - }, - "packets": { - "type": "long" - }, - "geo": { - "type": "object", - "properties": { - "city_name": { - "type": "keyword" - }, - "country_iso_code": { - "type": "keyword" - }, - "country_name": { - "type": "keyword" - }, - "location": { - "type": "geo_point" - } - } - } - } - } + "source": { + "type": "object", + "properties": { + "address": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 1024 } + } + }, + "domain": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "bytes": { + "type": "long" + }, + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + }, + "mac": { + "type": "keyword", + "ignore_above": 1024 + }, + "packets": { + "type": "long" + }, + "geo": { + "type": "object", + "properties": { + "city_name": { + "type": "keyword" + }, + "country_iso_code": { + "type": "keyword" + }, + "country_name": { + "type": "keyword" + }, + "location": { + "type": "geo_point" + } + } + } + } + }, + "destination": { + "type": "object", + "properties": { + "address": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "domain": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "bytes": { + "type": "long" + }, + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + }, + "mac": { + "type": "keyword", + "ignore_above": 1024 + }, + "packets": { + "type": "long" + }, + "geo": { + "type": "object", + "properties": { + "city_name": { + "type": "keyword" + }, + "country_iso_code": { + "type": "keyword" + }, + "country_name": { + "type": "keyword" + }, + "location": { + "type": "geo_point" + } + } } + } } + } } + } } + } } diff --git a/server/adaptors/integrations/__data__/repository/aws_elb/schemas/http-1.0.0.mapping.json b/server/adaptors/integrations/__data__/repository/aws_elb/schemas/http-1.0.0.mapping.json index 1c03fe3d6d..5fec510cb8 100644 --- a/server/adaptors/integrations/__data__/repository/aws_elb/schemas/http-1.0.0.mapping.json +++ b/server/adaptors/integrations/__data__/repository/aws_elb/schemas/http-1.0.0.mapping.json @@ -1,166 +1,166 @@ { - "template": { - "mappings": { - "_meta": { - "version": "1.0.0", - "catalog": "observability", - "type": "logs", - "component": "http", - "labels": ["http"] + "template": { + "mappings": { + "_meta": { + "version": "1.0.0", + "catalog": "observability", + "type": "logs", + "component": "http", + "labels": ["http"] + }, + "dynamic_templates": [ + { + "request_header_map": { + "mapping": { + "type": "keyword" }, - "dynamic_templates": [ - { - "request_header_map": { - "mapping": { - "type": "keyword" - }, - "path_match": "request.header.*" + "path_match": "request.header.*" + } + }, + { + "response_header_map": { + "mapping": { + "type": "keyword" + }, + "path_match": "response.header.*" + } + } + ], + "properties": { + "http": { + "properties": { + "flavor": { + "type": "keyword", + "ignore_above": 256 + }, + "user_agent": { + "type": "object", + "properties": { + "original": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "version": { + "type": "keyword" + }, + "device": { + "type": "object", + "properties": { + "name": { + "type": "keyword" } + } }, - { - "response_header_map": { - "mapping": { - "type": "keyword" - }, - "path_match": "response.header.*" + "os": { + "type": "object", + "properties": { + "type": { + "type": "keyword" + }, + "platform": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "full": { + "type": "keyword" + }, + "family": { + "type": "keyword" + }, + "version": { + "type": "keyword" + }, + "kernel": { + "type": "keyword" + } + } + } + } + }, + "url": { + "type": "keyword", + "ignore_above": 2048 + }, + "schema": { + "type": "keyword", + "ignore_above": 1024 + }, + "target": { + "type": "keyword", + "ignore_above": 1024 + }, + "route": { + "type": "keyword", + "ignore_above": 1024 + }, + "client.ip": { + "type": "ip" + }, + "resent_count": { + "type": "integer" + }, + "request": { + "type": "object", + "properties": { + "id": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 } + } + }, + "body.content": { + "type": "text" + }, + "bytes": { + "type": "long" + }, + "method": { + "type": "keyword", + "ignore_above": 256 + }, + "referrer": { + "type": "keyword", + "ignore_above": 1024 + }, + "mime_type": { + "type": "keyword", + "ignore_above": 1024 } - ], - "properties": { - "http": { - "properties": { - "flavor": { - "type": "keyword", - "ignore_above": 256 - }, - "user_agent": { - "type": "object", - "properties": { - "original": { - "type": "keyword" - }, - "name": { - "type": "keyword" - }, - "version": { - "type": "keyword" - }, - "device": { - "type": "object", - "properties": { - "name": { - "type": "keyword" - } - } - }, - "os": { - "type": "object", - "properties": { - "type": { - "type": "keyword" - }, - "platform": { - "type": "keyword" - }, - "name": { - "type": "keyword" - }, - "full": { - "type": "keyword" - }, - "family": { - "type": "keyword" - }, - "version": { - "type": "keyword" - }, - "kernel": { - "type": "keyword" - } - } - } - } - }, - "url": { - "type": "keyword", - "ignore_above": 2048 - }, - "schema": { - "type": "keyword", - "ignore_above": 1024 - }, - "target": { - "type": "keyword", - "ignore_above": 1024 - }, - "route": { - "type": "keyword", - "ignore_above": 1024 - }, - "client.ip": { - "type": "ip" - }, - "resent_count": { - "type": "integer" - }, - "request": { - "type": "object", - "properties": { - "id": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "body.content": { - "type": "text" - }, - "bytes": { - "type": "long" - }, - "method": { - "type": "keyword", - "ignore_above": 256 - }, - "referrer": { - "type": "keyword", - "ignore_above": 1024 - }, - "mime_type": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "response": { - "type": "object", - "properties": { - "id": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "body.content": { - "type": "text" - }, - "bytes": { - "type": "long" - }, - "status_code": { - "type": "integer" - } - } - } + } + }, + "response": { + "type": "object", + "properties": { + "id": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 } + } + }, + "body.content": { + "type": "text" + }, + "bytes": { + "type": "long" + }, + "status_code": { + "type": "integer" } + } } + } } + } } + } } diff --git a/server/adaptors/integrations/__data__/repository/aws_elb/schemas/logs_elb-1.0.0.mapping.json b/server/adaptors/integrations/__data__/repository/aws_elb/schemas/logs_elb-1.0.0.mapping.json index 7e365775bb..773d2a5f5d 100644 --- a/server/adaptors/integrations/__data__/repository/aws_elb/schemas/logs_elb-1.0.0.mapping.json +++ b/server/adaptors/integrations/__data__/repository/aws_elb/schemas/logs_elb-1.0.0.mapping.json @@ -1,7 +1,5 @@ { - "index_patterns": [ - "ss4o_logs-aws_elb-*" - ], + "index_patterns": ["ss4o_logs-aws_elb-*"], "data_stream": {}, "template": { "aliases": { @@ -222,12 +220,6 @@ } } }, - "composed_of": [ - "communication", - "http", - "cloud", - "aws_elb", - "url" - ], + "composed_of": ["communication", "http", "cloud", "aws_elb", "url"], "version": 1 } diff --git a/server/adaptors/integrations/__data__/repository/aws_rds/assets/README.md b/server/adaptors/integrations/__data__/repository/aws_rds/assets/README.md index db219e610a..36fac9aea5 100644 --- a/server/adaptors/integrations/__data__/repository/aws_rds/assets/README.md +++ b/server/adaptors/integrations/__data__/repository/aws_rds/assets/README.md @@ -5,10 +5,11 @@ API: http://osd:5601/api/saved_objects/_import?overwrite=true - [Assets](aws_rds-1.0.0.ndjson) ## Asset List + The next table details the assets | Name | Type | Description | -|---------------------------------------------|:-------------:|:----------------------------------------------------------------------------------------------------------:| +| ------------------------------------------- | :-----------: | :--------------------------------------------------------------------------------------------------------: | | `ss4o_logs_rds-*-*` | index-pattern | The Index Pattern | | `AWS RDS Log Event Overview` | dashboard | The pre-canned dashboard for AWS RDS | | `[AWS RDS] Filters` | visualization | [Controls] Interactive controls for easy dashboard manipulation | @@ -24,13 +25,14 @@ The next table details the assets | `[AWS RDS] Top Slow Query` | visualization | [Table] Top 10 source showing the slowest queries | | `[AWS RDS] Lock` | visualization | [Table] A visualization showing the number of active locks in your RDS instance | | `[AWS RDS] Total Deadlock Queries` | visualization | [Table] Represents the total count of deadlock scenarios encountered in the database | -| `[AWS RDS] Deadlock History` | visualization | [Table] Provides a timeline showing occurrences of deadlock scenarios | -| `[AWS RDS] Error Data` | visualization | [Table] Represents data related to various errors occurred in your RDS instance | +| `[AWS RDS] Deadlock History` | visualization | [Table] Provides a timeline showing occurrences of deadlock scenarios | +| `[AWS RDS] Error Data` | visualization | [Table] Represents data related to various errors occurred in your RDS instance | | `[AWS RDS] Audit Data` | visualization | [Table] overview of audit logs, showing actions that have been tracked for review | | `[AWS RDS] Total Error Logs` | visualization | [Line] Displays the total count of error logs recorded within a specific time frame | -| `[AWS RDS] Error History` | visualization | [Line] Provides a timeline representation of the errors occurred over a certain period. | -| `[AWS RDS] Audoit History` | visualization | [Line] Provides a timeline representation of the audited events occurred over a certain period. | +| `[AWS RDS] Error History` | visualization | [Line] Provides a timeline representation of the errors occurred over a certain period. | +| `[AWS RDS] Audoit History` | visualization | [Line] Provides a timeline representation of the audited events occurred over a certain period. | | `[AWS RDS] General Search` | search | The pre-canned search for AWS RDS | ## Dashboard -![](../static/dashboard.png) \ No newline at end of file + +![](../static/dashboard.png) diff --git a/server/adaptors/integrations/__data__/repository/aws_rds/aws_rds-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_rds/aws_rds-1.0.0.json index d9b05fa504..c34f2697ba 100644 --- a/server/adaptors/integrations/__data__/repository/aws_rds/aws_rds-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/aws_rds/aws_rds-1.0.0.json @@ -5,7 +5,7 @@ "description": "AWS RDS", "license": "Apache-2.0", "type": "logs_rds", - "labels": ["log","aws","s3","cloud","rds"], + "labels": ["log", "aws", "s3", "cloud", "rds"], "author": "OpenSearch", "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_rds/info", "statics": { diff --git a/server/adaptors/integrations/__data__/repository/aws_rds/info/README.md b/server/adaptors/integrations/__data__/repository/aws_rds/info/README.md index 27df96332b..9c087f6c64 100644 --- a/server/adaptors/integrations/__data__/repository/aws_rds/info/README.md +++ b/server/adaptors/integrations/__data__/repository/aws_rds/info/README.md @@ -1,24 +1,27 @@ # AWS RDS Integrations ## What is AWS RDS? + AWS RDS (Relational Database Service) is a managed service that makes it easy to set up, operate, and scale a relational database in the cloud. RDS helps you perform tasks such as: -* Managing database instances -* Scaling compute resources and storage capacity -* Automating time-consuming administration tasks including hardware provisioning, database setup, patching, and backups +- Managing database instances +- Scaling compute resources and storage capacity +- Automating time-consuming administration tasks including hardware provisioning, database setup, patching, and backups RDS keeps your database up-to-date with the latest patches, and it also provides automatic backups and disaster recovery capabilities. You can make database instances available in multiple regions to enhance availability and reliability for your data. See additional details [here](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_LogAccess.html). ## What is AWS RDS Integration? + An integration is a collection of predefined assets which are combined in a meaningful manner. AWS RDS integration includes dashboards, visualizations, queries, and index mapping to help you manage and monitor your database services more effectively. ### Dashboards + The Dashboard uses the index alias `logs-aws-rds` for shortening the index name - be advised. ![](../static/dashboard_rds1.png) diff --git a/server/adaptors/integrations/__data__/repository/aws_rds/schemas/aws_rds-1.0.0.mapping.json b/server/adaptors/integrations/__data__/repository/aws_rds/schemas/aws_rds-1.0.0.mapping.json index 525d7ca739..220abe36ec 100644 --- a/server/adaptors/integrations/__data__/repository/aws_rds/schemas/aws_rds-1.0.0.mapping.json +++ b/server/adaptors/integrations/__data__/repository/aws_rds/schemas/aws_rds-1.0.0.mapping.json @@ -6,7 +6,7 @@ "catalog": "observability", "type": "logs", "component": "aws_rds", - "labels": ["aws","rds"] + "labels": ["aws", "rds"] }, "properties": { "aws": { diff --git a/server/adaptors/integrations/__data__/repository/aws_rds/schemas/aws_s3-1.0.0.mapping.json b/server/adaptors/integrations/__data__/repository/aws_rds/schemas/aws_s3-1.0.0.mapping.json index fbdfa5237a..ca32e104a0 100644 --- a/server/adaptors/integrations/__data__/repository/aws_rds/schemas/aws_s3-1.0.0.mapping.json +++ b/server/adaptors/integrations/__data__/repository/aws_rds/schemas/aws_s3-1.0.0.mapping.json @@ -6,7 +6,7 @@ "catalog": "observability", "type": "logs", "component": "aws_s3", - "labels": ["aws","s3"] + "labels": ["aws", "s3"] }, "properties": { "aws": { diff --git a/server/adaptors/integrations/__data__/repository/aws_rds/schemas/logs_rds-1.0.0.mapping.json b/server/adaptors/integrations/__data__/repository/aws_rds/schemas/logs_rds-1.0.0.mapping.json index 4f032e743d..8f2faf6979 100644 --- a/server/adaptors/integrations/__data__/repository/aws_rds/schemas/logs_rds-1.0.0.mapping.json +++ b/server/adaptors/integrations/__data__/repository/aws_rds/schemas/logs_rds-1.0.0.mapping.json @@ -1,7 +1,5 @@ { - "index_patterns": [ - "ss4o_logs-aws_rds-*" - ], + "index_patterns": ["ss4o_logs-aws_rds-*"], "priority": 900, "data_stream": {}, "template": { @@ -14,7 +12,7 @@ "catalog": "observability", "type": "logs", "component": "log", - "labels": ["log","aws","s3","cloud","rds"], + "labels": ["log", "aws", "s3", "cloud", "rds"], "correlations": [ { "field": "spanId", @@ -107,8 +105,7 @@ }, "time": { "type": "alias", - "path" : "@timestamp" - + "path": "@timestamp" }, "observedTimestamp": { "type": "date" @@ -228,11 +225,6 @@ } } }, - "composed_of": [ - "aws_rds", - "cloud", - "aws_s3" - ], + "composed_of": ["aws_rds", "cloud", "aws_s3"], "version": 1 - } diff --git a/server/adaptors/integrations/__data__/repository/aws_s3/aws_s3-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_s3/aws_s3-1.0.0.json index 468a6a2b13..60a1b011d6 100644 --- a/server/adaptors/integrations/__data__/repository/aws_s3/aws_s3-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/aws_s3/aws_s3-1.0.0.json @@ -1,45 +1,45 @@ { - "name": "aws_s3", - "version": "1.0.0", - "displayName": "AWS S3 ", - "description": "AWS S3 Object Store", - "license": "Apache-2.0", - "type": "logs_s3", - "labels": ["log","aws","s3","cloud"], - "author": "OpenSearch", - "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_s3/info", - "statics": { - "logo": { - "annotation": "S3 Logo", - "path": "logo.png" - }, - "gallery": [ - { - "annotation": "AWS S3 Dashboard", - "path": "dashboard.png" - } - ] + "name": "aws_s3", + "version": "1.0.0", + "displayName": "AWS S3 ", + "description": "AWS S3 Object Store", + "license": "Apache-2.0", + "type": "logs_s3", + "labels": ["log", "aws", "s3", "cloud"], + "author": "OpenSearch", + "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_s3/info", + "statics": { + "logo": { + "annotation": "S3 Logo", + "path": "logo.png" }, - "components": [ - { - "name": "aws_s3", - "version": "1.0.0" - }, - { - "name": "logs_s3", - "version": "1.0.0" - }, - { - "name": "cloud", - "version": "1.0.0" - } - ], - "assets": { - "savedObjects": { - "name": "aws_s3", - "version": "1.0.0" - } + "gallery": [ + { + "annotation": "AWS S3 Dashboard", + "path": "dashboard.png" + } + ] + }, + "components": [ + { + "name": "aws_s3", + "version": "1.0.0" }, + { + "name": "logs_s3", + "version": "1.0.0" + }, + { + "name": "cloud", + "version": "1.0.0" + } + ], + "assets": { + "savedObjects": { + "name": "aws_s3", + "version": "1.0.0" + } + }, "sampleData": { "path": "sample.json" } diff --git a/server/adaptors/integrations/__data__/repository/aws_s3/info/README.md b/server/adaptors/integrations/__data__/repository/aws_s3/info/README.md index 9e4c960ca3..b91fbf4c25 100644 --- a/server/adaptors/integrations/__data__/repository/aws_s3/info/README.md +++ b/server/adaptors/integrations/__data__/repository/aws_s3/info/README.md @@ -13,8 +13,7 @@ An integration is a bundle of pre-canned assets which are brought together in a AWS S3 integration includes dashboards, visualizations, queries, and an index mapping. ### Dashboards + The Dashboard uses the index alias `logs-aws-s3` for shortening the index name - be advised. ![AWS S3 Dashboard](../static/dashboard_s3.png) - - diff --git a/server/adaptors/integrations/__data__/repository/aws_s3/schemas/aws_s3-1.0.0.mapping.json b/server/adaptors/integrations/__data__/repository/aws_s3/schemas/aws_s3-1.0.0.mapping.json index f948597fc2..8057cd98ab 100644 --- a/server/adaptors/integrations/__data__/repository/aws_s3/schemas/aws_s3-1.0.0.mapping.json +++ b/server/adaptors/integrations/__data__/repository/aws_s3/schemas/aws_s3-1.0.0.mapping.json @@ -6,7 +6,7 @@ "catalog": "observability", "type": "logs", "component": "s3", - "labels": ["aws","s3"] + "labels": ["aws", "s3"] }, "properties": { "aws": { diff --git a/server/adaptors/integrations/__data__/repository/aws_s3/schemas/logs_s3-1.0.0.mapping.json b/server/adaptors/integrations/__data__/repository/aws_s3/schemas/logs_s3-1.0.0.mapping.json index fbe2182b72..3d1e48d26b 100644 --- a/server/adaptors/integrations/__data__/repository/aws_s3/schemas/logs_s3-1.0.0.mapping.json +++ b/server/adaptors/integrations/__data__/repository/aws_s3/schemas/logs_s3-1.0.0.mapping.json @@ -1,7 +1,5 @@ { - "index_patterns": [ - "ss4o_logs-aws_s3-*" - ], + "index_patterns": ["ss4o_logs-aws_s3-*"], "priority": 900, "data_stream": {}, "template": { @@ -14,7 +12,7 @@ "catalog": "observability", "type": "logs", "component": "log", - "labels": ["log","aws","s3","cloud"], + "labels": ["log", "aws", "s3", "cloud"], "correlations": [ { "field": "spanId", @@ -223,9 +221,6 @@ } } }, - "composed_of": [ - "aws_s3", - "cloud" - ], + "composed_of": ["aws_s3", "cloud"], "version": 1 } diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/README.md b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/README.md index dc9eb13a35..fb5ff8ce8c 100644 --- a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/README.md +++ b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/README.md @@ -5,28 +5,29 @@ API: http://osd:5601/api/saved_objects/_import?overwrite=true - [Assets](aws_vpc_flow-1.0.0.ndjson) ## Asset List + The next table details the assets -| Name | Type | Description | -|-------------------------------|:-------------:|:---------------------------------------------------------------------------:| -| `ss4o_logs_vpc-aws_vpc_flow-*-*` | index-pattern | The Index Pattern | -| `AWS VPC Flow Logs Overview` | dashboard | The pre-canned dashboard for AWS VPC flow logs | -| `[AWS VPC Flow Logs] Filters` | visualization | [Controls] Interactive controls for easy dashboard manipulation | -| `[AWS VPC Flow Logs] Total Requests` | visualization | [Metric] Total requests through the VPC | -| `[AWS VPC Flow Logs] Request History` | visualization | [Vertical Bar] Number of Requests counted against time | -| `[AWS VPC Flow Logs] Requests by VPC ID` | visualization | [Pie] Compare parts of Requests from each VPC ID | -| `[AWS VPC Flow Logs] Total Requests By Action` | visualization | [Metric] Number of Accept/Reject requests | -| `[AWS VPC Flow Logs] Bytes` | visualization | [Line] Trend of bytes transferred during the flow | -| `[AWS VPC Flow Logs] Packets` | visualization | [Line] Trend of Packets transferred during the flow | -| `[AWS VPC Flow Logs] Bytes Metric` | visualization | [Metric] Total ingress/egress bytes transferred during the flow | -| `[AWS VPC Flow Logs] Requests by Direction` | visualization | [Pie] Compare parts of ingress/egress requests | +| Name | Type | Description | +| -------------------------------------------------- | :-----------: | :-------------------------------------------------------------------------: | +| `ss4o_logs_vpc-aws_vpc_flow-*-*` | index-pattern | The Index Pattern | +| `AWS VPC Flow Logs Overview` | dashboard | The pre-canned dashboard for AWS VPC flow logs | +| `[AWS VPC Flow Logs] Filters` | visualization | [Controls] Interactive controls for easy dashboard manipulation | +| `[AWS VPC Flow Logs] Total Requests` | visualization | [Metric] Total requests through the VPC | +| `[AWS VPC Flow Logs] Request History` | visualization | [Vertical Bar] Number of Requests counted against time | +| `[AWS VPC Flow Logs] Requests by VPC ID` | visualization | [Pie] Compare parts of Requests from each VPC ID | +| `[AWS VPC Flow Logs] Total Requests By Action` | visualization | [Metric] Number of Accept/Reject requests | +| `[AWS VPC Flow Logs] Bytes` | visualization | [Line] Trend of bytes transferred during the flow | +| `[AWS VPC Flow Logs] Packets` | visualization | [Line] Trend of Packets transferred during the flow | +| `[AWS VPC Flow Logs] Bytes Metric` | visualization | [Metric] Total ingress/egress bytes transferred during the flow | +| `[AWS VPC Flow Logs] Requests by Direction` | visualization | [Pie] Compare parts of ingress/egress requests | | `[AWS VPC Flow Logs] Requests by Direction Metric` | visualization | [Metric] Number of ingress/egress requests | -| `[AWS VPC Flow Logs] Top Source Bytes` | visualization | [Table] Top 10 source with number of bytes transferred during the flow | -| `[AWS VPC Flow Logs] Top Destination Bytes` | visualization | [Table] Top 10 destination with number of bytes transferred during the flow | -| `[AWS VPC Flow Logs] Top Sources` | visualization | [Table] Top 10 source with number of requests send during the flow | -| `[AWS VPC Flow Logs] Top Destinations` | visualization | [Table] Top 10 destination with number of requests send during the flow | -| `[AWS VPC Flow Logs] Flow` | visualization | [Vega] Illustrates the flow from Source to Destination | -| `[AWS VPC Flow Logs] Heat Map` | visualization | [Heat Map] Heat Map of source and destination | -| `[AWS VPC Flow Logs] Top Source AWS Services` | visualization | [Pie] Compare parts of AWS service as flow source | +| `[AWS VPC Flow Logs] Top Source Bytes` | visualization | [Table] Top 10 source with number of bytes transferred during the flow | +| `[AWS VPC Flow Logs] Top Destination Bytes` | visualization | [Table] Top 10 destination with number of bytes transferred during the flow | +| `[AWS VPC Flow Logs] Top Sources` | visualization | [Table] Top 10 source with number of requests send during the flow | +| `[AWS VPC Flow Logs] Top Destinations` | visualization | [Table] Top 10 destination with number of requests send during the flow | +| `[AWS VPC Flow Logs] Flow` | visualization | [Vega] Illustrates the flow from Source to Destination | +| `[AWS VPC Flow Logs] Heat Map` | visualization | [Heat Map] Heat Map of source and destination | +| `[AWS VPC Flow Logs] Top Source AWS Services` | visualization | [Pie] Compare parts of AWS service as flow source | | `[AWS VPC Flow Logs] Top Destination AWS Services` | visualization | [Pie] Compare parts of AWS service as flow destination | -| `[AWS VPC Flow Logs] General Search` | search | The pre-canned search for AWS VPC flow logs | +| `[AWS VPC Flow Logs] General Search` | search | The pre-canned search for AWS VPC flow logs | diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json index c9367e1aa3..deee7d3626 100644 --- a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json @@ -5,7 +5,7 @@ "description": "AWS VPC Flow log collector", "license": "Apache-2.0", "type": "logs_vpc", - "labels": ["log","aws","s3","cloud","communication","vpc"], + "labels": ["log", "aws", "s3", "cloud", "communication", "vpc"], "author": "Haidong Wang", "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_vpc_flow/info", "statics": { diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/info/README.md b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/info/README.md index af6635934a..7997141e3d 100644 --- a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/info/README.md +++ b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/info/README.md @@ -1,26 +1,29 @@ # AWS VPC Flow Logs Integration ## What is AWS VPC Flow Logs ? + VPC Flow Logs is a feature that enables you to capture information about the IP traffic going to and from network interfaces in your VPC. Flow logs can help you with a number of tasks, such as: -* Diagnosing overly restrictive security group rules +- Diagnosing overly restrictive security group rules -* Monitoring the traffic that is reaching your instance +- Monitoring the traffic that is reaching your instance -* Determining the direction of the traffic to and from the network interfaces +- Determining the direction of the traffic to and from the network interfaces Flow log data is collected outside of the path of your network traffic, and therefore does not affect network throughput or latency. You can create or delete flow logs without any risk of impact to network performance. See additional details [here](https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs.html). ## What is AWS VPC FLow Logs Integration ? + An integration is a bundle of pre-canned assets which are bundled togather in a meaningful manner. AWS VPC flow logs integration includes dashboards, visualisations, queries and an index mapping. ### Dashboards + The Dashboard uses the index alias `logs-vpc` for shortening the index name - be advised. ![](../static/dashboard1.png) diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/schemas/aws_s3-1.0.0.mapping.json b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/schemas/aws_s3-1.0.0.mapping.json index b4f4fe597e..4217d6e16a 100644 --- a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/schemas/aws_s3-1.0.0.mapping.json +++ b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/schemas/aws_s3-1.0.0.mapping.json @@ -6,7 +6,7 @@ "catalog": "observability", "type": "logs", "component": "aws_s3", - "labels": ["aws","s3"] + "labels": ["aws", "s3"] }, "properties": { "aws": { diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/schemas/aws_vpc_flow-1.0.0.mapping.json b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/schemas/aws_vpc_flow-1.0.0.mapping.json index 2a3851cf5c..2369f953d7 100644 --- a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/schemas/aws_vpc_flow-1.0.0.mapping.json +++ b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/schemas/aws_vpc_flow-1.0.0.mapping.json @@ -6,7 +6,7 @@ "catalog": "observability", "type": "logs", "component": "aws_vpc_flow", - "labels": ["aws","vpc"] + "labels": ["aws", "vpc"] }, "properties": { "aws": { diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/schemas/communication-1.0.0.mapping.json b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/schemas/communication-1.0.0.mapping.json index 88b211ab9d..d9af5d7193 100644 --- a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/schemas/communication-1.0.0.mapping.json +++ b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/schemas/communication-1.0.0.mapping.json @@ -6,9 +6,7 @@ "catalog": "observability", "type": "logs", "component": "communication", - "labels": [ - "communication" - ] + "labels": ["communication"] }, "properties": { "communication": { diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/schemas/logs_vpc-1.0.0.mapping.json b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/schemas/logs_vpc-1.0.0.mapping.json index 5ece5066de..6b07534115 100644 --- a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/schemas/logs_vpc-1.0.0.mapping.json +++ b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/schemas/logs_vpc-1.0.0.mapping.json @@ -1,7 +1,5 @@ { - "index_patterns": [ - "ss4o_logs-aws_vpc-*" - ], + "index_patterns": ["ss4o_logs-aws_vpc-*"], "priority": 900, "data_stream": {}, "template": { @@ -14,7 +12,7 @@ "catalog": "observability", "type": "logs", "component": "log", - "labels": ["log","aws","s3","cloud","communication","vpc"], + "labels": ["log", "aws", "s3", "cloud", "communication", "vpc"], "correlations": [ { "field": "spanId", @@ -223,11 +221,6 @@ } } }, - "composed_of": [ - "aws_vpc_flow", - "aws_s3", - "cloud", - "communication" - ], + "composed_of": ["aws_vpc_flow", "aws_s3", "cloud", "communication"], "version": 1 } diff --git a/server/adaptors/integrations/__data__/repository/aws_waf/aws_waf-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_waf/aws_waf-1.0.0.json index 76f9d74fd3..1b1171eb9e 100644 --- a/server/adaptors/integrations/__data__/repository/aws_waf/aws_waf-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/aws_waf/aws_waf-1.0.0.json @@ -5,7 +5,7 @@ "description": "AWS waf log collector", "license": "Apache-2.0", "type": "logs_waf", - "labels": ["log","aws","s3","cloud","waf"], + "labels": ["log", "aws", "s3", "cloud", "waf"], "author": "OpenSearch", "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_waf/info", "statics": { diff --git a/server/adaptors/integrations/__data__/repository/aws_waf/info/README.md b/server/adaptors/integrations/__data__/repository/aws_waf/info/README.md index 3c51586abf..f4620fbece 100644 --- a/server/adaptors/integrations/__data__/repository/aws_waf/info/README.md +++ b/server/adaptors/integrations/__data__/repository/aws_waf/info/README.md @@ -22,6 +22,7 @@ An integration is a set of pre-configured assets bundled together to facilitate AWS WAF log integration includes dashboards, visualizations, queries, and an index mapping. ### Dashboards + The Dashboard uses the index alias `logs-waf` for shortening the index name - be advised. ![Dashboard](../static/dashboard.png) diff --git a/server/adaptors/integrations/__data__/repository/aws_waf/schemas/aws_s3-1.0.0.mapping.json b/server/adaptors/integrations/__data__/repository/aws_waf/schemas/aws_s3-1.0.0.mapping.json index fbdfa5237a..ca32e104a0 100644 --- a/server/adaptors/integrations/__data__/repository/aws_waf/schemas/aws_s3-1.0.0.mapping.json +++ b/server/adaptors/integrations/__data__/repository/aws_waf/schemas/aws_s3-1.0.0.mapping.json @@ -6,7 +6,7 @@ "catalog": "observability", "type": "logs", "component": "aws_s3", - "labels": ["aws","s3"] + "labels": ["aws", "s3"] }, "properties": { "aws": { diff --git a/server/adaptors/integrations/__data__/repository/aws_waf/schemas/aws_waf-1.0.0.mapping.json b/server/adaptors/integrations/__data__/repository/aws_waf/schemas/aws_waf-1.0.0.mapping.json index 92bc8cd220..5722911f10 100644 --- a/server/adaptors/integrations/__data__/repository/aws_waf/schemas/aws_waf-1.0.0.mapping.json +++ b/server/adaptors/integrations/__data__/repository/aws_waf/schemas/aws_waf-1.0.0.mapping.json @@ -6,7 +6,7 @@ "catalog": "observability", "type": "logs", "component": "aws_waf", - "labels": ["aws","waf"] + "labels": ["aws", "waf"] }, "properties": { "aws": { diff --git a/server/adaptors/integrations/__data__/repository/aws_waf/schemas/logs_waf-1.0.0.mapping.json b/server/adaptors/integrations/__data__/repository/aws_waf/schemas/logs_waf-1.0.0.mapping.json index b24d7f6be7..f79b80dfa0 100644 --- a/server/adaptors/integrations/__data__/repository/aws_waf/schemas/logs_waf-1.0.0.mapping.json +++ b/server/adaptors/integrations/__data__/repository/aws_waf/schemas/logs_waf-1.0.0.mapping.json @@ -1,7 +1,5 @@ { - "index_patterns": [ - "ss4o_logs-aws_waf-*" - ], + "index_patterns": ["ss4o_logs-aws_waf-*"], "priority": 900, "data_stream": {}, "template": { @@ -14,7 +12,7 @@ "catalog": "observability", "type": "logs", "component": "log", - "labels": ["log","aws","s3","cloud","waf"], + "labels": ["log", "aws", "s3", "cloud", "waf"], "correlations": [ { "field": "spanId", @@ -223,10 +221,6 @@ } } }, - "composed_of": [ - "cloud", - "aws_waf", - "aws_s3" - ], + "composed_of": ["cloud", "aws_waf", "aws_s3"], "version": 1 } diff --git a/server/adaptors/integrations/__data__/repository/k8s/assets/README.md b/server/adaptors/integrations/__data__/repository/k8s/assets/README.md index 8d447b7655..f970c5334e 100644 --- a/server/adaptors/integrations/__data__/repository/k8s/assets/README.md +++ b/server/adaptors/integrations/__data__/repository/k8s/assets/README.md @@ -1,62 +1,54 @@ # K8s Dashboard Explained The following queries are used for the k8s dashboard: - + - Deployment names Graph: - - Filter: `event.domain:kubernetes AND event.dataset:kubernetes.deployment` - - Query: `kubernetes.deployment.name` + - Filter: `event.domain:kubernetes AND event.dataset:kubernetes.deployment` + - Query: `kubernetes.deployment.name` - Available pods per deployment (done per deployment aggregation) + - Filter: `event.domain:kubernetes AND event.dataset:kubernetes.deployment` - Query: `kubernetes.deployment.name` - - Desired pod - - Filter: `event.domain:kubernetes AND event.dataset:kubernetes.deployment` - - Query: `kubernetes.deployment.replicas.desired` + - Filter: `event.domain:kubernetes AND event.dataset:kubernetes.deployment` + - Query: `kubernetes.deployment.replicas.desired` - Available pods + - Filter: `event.domain:kubernetes AND event.dataset:kubernetes.deployment` - Query: `kubernetes.deployment.replicas.available` - - - Unavailable pods - Filter: `event.domain:kubernetes AND event.dataset:kubernetes.deployment` - Query: `kubernetes.deployment.replicas.unavailable` - - - Unavailable pods per deployment ( done per deployment aggregation) + - Filter: `event.domain:kubernetes AND event.dataset:kubernetes.deployment` - Query: `kubernetes.deployment.replicas.unavailable` - - - CPU usage by node + - Filter: `event.domain:kubernetes AND (event.dataset:kubernetes.container OR event.dataset:kubernetes.node)` - Query: `kubernetes.node.name` , `kubernetes.container.cpu.usage.nanocores`, `kubernetes.node.cpu.capacity.cores` - - - Top memory intensive pods + - Filter: `event.domain:kubernetes AND event.dataset:kubernetes.container` - Query: `kubernetes.container._module.pod.name`, `kubernetes.container.memory.usage.bytes` - - - Top CPU intensive pods + - Filter: `event.domain:kubernetes AND event.dataset:kubernetes.container` - Query: `kubernetes.container._module.pod.name`, `kubernetes.container.cpu.usage.core.ns` +- Network in by node - -- Network in by node - Filter: `event.domain:kubernetes AND event.dataset:kubernetes.pod` - - Query: `kubernetes.pod.network.rx.bytes` - - + - Query: `kubernetes.pod.network.rx.bytes` - Network out by node - Filter: `event.domain:kubernetes AND event.dataset:kubernetes.pod` diff --git a/server/adaptors/integrations/__data__/repository/k8s/info/README.md b/server/adaptors/integrations/__data__/repository/k8s/info/README.md index ee7111f358..94e9aaaae6 100644 --- a/server/adaptors/integrations/__data__/repository/k8s/info/README.md +++ b/server/adaptors/integrations/__data__/repository/k8s/info/README.md @@ -13,57 +13,69 @@ An integration is a collection of pre-configured assets that are bundled togethe Kubernetes integration includes dashboards, visualizations, queries, and an index mapping. ### Dashboards -![Dashboard](../static/dashboard.png) +![Dashboard](../static/dashboard.png) With the Kubernetes integration, you can gain valuable insights into the health and performance of your containerized applications. The pre-configured dashboards and visualizations help you monitor key metrics, track resource utilization, and identify potential issues within your Kubernetes clusters. This integration empowers you to efficiently manage your containerized workloads, scale applications as needed, and ensure the reliability and availability of your Kubernetes environment. -### Collecting K8s +### Collecting K8s + The next OpenTelemetry [page](https://opentelemetry.io/docs/kubernetes/collector/components/) describes the K8s attributes and other components -#### Kubernetes Attributes Processor +#### Kubernetes Attributes Processor + The Kubernetes Attributes Processor automatically discovers Kubernetes pods, extracts their metadata, and adds the extracted metadata to spans, metrics, and logs as resource attributes. The following attributes are added by default: **Cluster** + - `k8s.cluster.name` - `k8s.cluster.uid` **Namespace** + - `k8s.namespace.name` **Pod** + - `k8s.pod.name` - `k8s.pod.uid` - `k8s.pod.start_time` - `k8s.deployment.name` -**Node** +**Node** + - `k8s.node.name` - `k8s.node.uid` -**Container** +**Container** + - `k8s.container.name` - `k8s.container.restart_count` -**ReplicaSet** +**ReplicaSet** + - `k8s.replicaset.name` - `k8s.replicaset.uid` -**Deployment** +**Deployment** + - `k8s.deployment.name` - `k8s.deployment.uid` -**StatefulSet** +**StatefulSet** + - `k8s.statefulset.name` - `k8s.statefulset.uid` -**DaemonSet** +**DaemonSet** + - `k8s.daemon.name` - `k8s.daemon.uid` -**DaemonSet** +**DaemonSet** + - `k8s.job.name` - `k8s.job.uid` @@ -71,8 +83,8 @@ The following attributes are added by default: ### Important Components for Kubernetes - - [Kubeletstats Receiver](https://opentelemetry.io/docs/kubernetes/collector/components/#kubeletstats-receiver): pulls pod metrics from the API server on a kubelet. - - [Filelog Receiver](https://opentelemetry.io/docs/kubernetes/collector/components/#filelog-receiver): collects Kubernetes logs and application logs written to stdout/stderr. - - [Kubernetes Cluster Receiver](https://opentelemetry.io/docs/kubernetes/collector/components/#kubernetes-cluster-receiver): collects cluster-level metrics and entity events. - - [Kubernetes Objects Receiver](https://opentelemetry.io/docs/kubernetes/collector/components/#kubernetes-objects-receiver): collects objects, such as events, from the Kubernetes API server. - - [Host Metrics Receiver](https://opentelemetry.io/docs/kubernetes/collector/components/#host-metrics-receiver): scrapes host metrics from Kubernetes nodes. \ No newline at end of file +- [Kubeletstats Receiver](https://opentelemetry.io/docs/kubernetes/collector/components/#kubeletstats-receiver): pulls pod metrics from the API server on a kubelet. +- [Filelog Receiver](https://opentelemetry.io/docs/kubernetes/collector/components/#filelog-receiver): collects Kubernetes logs and application logs written to stdout/stderr. +- [Kubernetes Cluster Receiver](https://opentelemetry.io/docs/kubernetes/collector/components/#kubernetes-cluster-receiver): collects cluster-level metrics and entity events. +- [Kubernetes Objects Receiver](https://opentelemetry.io/docs/kubernetes/collector/components/#kubernetes-objects-receiver): collects objects, such as events, from the Kubernetes API server. +- [Host Metrics Receiver](https://opentelemetry.io/docs/kubernetes/collector/components/#host-metrics-receiver): scrapes host metrics from Kubernetes nodes. diff --git a/server/adaptors/integrations/__data__/repository/k8s/ingestion/README.md b/server/adaptors/integrations/__data__/repository/k8s/ingestion/README.md index e7b302f0f5..198e8a9558 100644 --- a/server/adaptors/integrations/__data__/repository/k8s/ingestion/README.md +++ b/server/adaptors/integrations/__data__/repository/k8s/ingestion/README.md @@ -1,7 +1,9 @@ # Setting Fluent Bit Ingestion Deployment in Kubernetes + Fluent Bit is an open-source and multi-platform Log Processor and Forwarder. It allows you to unify the data collection and logging of your system. This particular YAML configuration deploys Fluent Bit in a Kubernetes cluster for monitoring purposes. ## Components + The YAML consists of four primary components: **ClusterRole:** For defining permissions. @@ -18,25 +20,23 @@ The YAML consists of four primary components: 4. **DaemonSet** Fluent Bit is deployed as a DaemonSet, which means a Fluent Bit container will run on every node in the cluster. This ensures that logs from all nodes are collected and processed. - This Fluent Bit deployment in Kubernetes is instrumental in gathering, processing, and forwarding logs from different parts of the cluster. It can be integrated with various log analytics tools and used for monitoring the behavior of the cluster, facilitating prompt insights and responses to system events. Make sure to tailor the configuration to match your specific requirements and infrastructure. +### References -### References -- [K8s Filter Plugin](https://docs.fluentbit.io/manual/pipeline/filters/kubernetes) +- [K8s Filter Plugin](https://docs.fluentbit.io/manual/pipeline/filters/kubernetes) **Kubernetes filter performs the following operations:** Analyzes the data and extracts the metadata such as `Pod name`, `namespace`, `container name`, and `container ID` . -Queries Kubernetes API server to get extra metadata for the given Pod including the `Pod ID`, `labels`, `annotations`. +Queries Kubernetes API server to get extra metadata for the given Pod including the `Pod ID`, `labels`, `annotations`. This metadata is then appended to each record (log message). -This data is cached locally in memory and is appended to each log record. +This data is cached locally in memory and is appended to each log record. The following parameters represent a minimum configuration for this filter used in the ConfigMap above: - `Name` — the name of the filter plugin. - `Kube_URL` — API Server end-point. E.g https://kubernetes.default.svc.cluster.local/ - `Match` — a tag to match filtering against. - diff --git a/server/adaptors/integrations/__data__/repository/k8s/k8s-1.0.0.json b/server/adaptors/integrations/__data__/repository/k8s/k8s-1.0.0.json index a157ac9e86..7454b25a9a 100644 --- a/server/adaptors/integrations/__data__/repository/k8s/k8s-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/k8s/k8s-1.0.0.json @@ -1,50 +1,50 @@ { - "name": "k8s", - "version": "1.0.0", - "displayName": "Kubernetes Dashboard", - "description": "Kubernetes web logs collector", - "license": "Apache-2.0", - "type": "logs-k8s", - "labels": ["log", "k8s", "cloud", "container"], - "author": "OpenSearch", - "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/k8s/info", - "statics": { - "logo": { - "annotation": "Kubernetes Logo", - "path": "logo.png" - }, - "gallery": [ - { - "annotation": "Kubernetes Dashboard", - "path": "dashboard.png" - } - ] + "name": "k8s", + "version": "1.0.0", + "displayName": "Kubernetes Dashboard", + "description": "Kubernetes web logs collector", + "license": "Apache-2.0", + "type": "logs-k8s", + "labels": ["log", "k8s", "cloud", "container"], + "author": "OpenSearch", + "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/k8s/info", + "statics": { + "logo": { + "annotation": "Kubernetes Logo", + "path": "logo.png" }, - "components": [ - { - "name": "k8s", - "version": "1.0.0" - }, - { - "name": "cloud", - "version": "1.0.0" - }, - { - "name": "container", - "version": "1.0.0" - }, - { - "name": "logs-k8s", - "version": "1.0.0" - } - ], - "assets": { - "savedObjects": { - "name": "k8s", - "version": "1.0.0" - } + "gallery": [ + { + "annotation": "Kubernetes Dashboard", + "path": "dashboard.png" + } + ] + }, + "components": [ + { + "name": "k8s", + "version": "1.0.0" }, - "sampleData": { - "path": "sample.json" + { + "name": "cloud", + "version": "1.0.0" + }, + { + "name": "container", + "version": "1.0.0" + }, + { + "name": "logs-k8s", + "version": "1.0.0" + } + ], + "assets": { + "savedObjects": { + "name": "k8s", + "version": "1.0.0" } + }, + "sampleData": { + "path": "sample.json" + } } diff --git a/server/adaptors/integrations/__data__/repository/k8s/schemas/container-1.0.0.mapping.json b/server/adaptors/integrations/__data__/repository/k8s/schemas/container-1.0.0.mapping.json index 10d61b102d..e8fd9ec763 100644 --- a/server/adaptors/integrations/__data__/repository/k8s/schemas/container-1.0.0.mapping.json +++ b/server/adaptors/integrations/__data__/repository/k8s/schemas/container-1.0.0.mapping.json @@ -5,7 +5,7 @@ "version": "1.0.0", "catalog": "observability", "type": "logs", - "component" : "container", + "component": "container", "labels": ["container"] }, "properties": { diff --git a/server/adaptors/integrations/__data__/repository/k8s/schemas/k8s-1.0.0.mapping.json b/server/adaptors/integrations/__data__/repository/k8s/schemas/k8s-1.0.0.mapping.json index 0e22ef6755..7d88aab2f2 100644 --- a/server/adaptors/integrations/__data__/repository/k8s/schemas/k8s-1.0.0.mapping.json +++ b/server/adaptors/integrations/__data__/repository/k8s/schemas/k8s-1.0.0.mapping.json @@ -593,9 +593,7 @@ "message": { "type": "keyword", "ignore_above": 256, - "copy_to": [ - "message" - ] + "copy_to": ["message"] }, "metadata": { "properties": { diff --git a/server/adaptors/integrations/__data__/repository/k8s/schemas/logs-k8s-1.0.0.mapping.json b/server/adaptors/integrations/__data__/repository/k8s/schemas/logs-k8s-1.0.0.mapping.json index 7553e299de..7461f10190 100644 --- a/server/adaptors/integrations/__data__/repository/k8s/schemas/logs-k8s-1.0.0.mapping.json +++ b/server/adaptors/integrations/__data__/repository/k8s/schemas/logs-k8s-1.0.0.mapping.json @@ -1,7 +1,5 @@ { - "index_patterns": [ - "ss4o_logs-k8s-*" - ], + "index_patterns": ["ss4o_logs-k8s-*"], "data_stream": {}, "template": { "aliases": { diff --git a/server/adaptors/integrations/__data__/repository/nginx/info/README.md b/server/adaptors/integrations/__data__/repository/nginx/info/README.md index 1171a7b320..6bc757b1f5 100644 --- a/server/adaptors/integrations/__data__/repository/nginx/info/README.md +++ b/server/adaptors/integrations/__data__/repository/nginx/info/README.md @@ -1,14 +1,17 @@ # Nginx Integration ## What is Nginx ? + NGINX is open source software for web serving, reverse proxying, caching, load balancing, media streaming, and more. See additional details [here](https://www.nginx.com/). ## What is Nginx Integration ? + An integration is a bundle of pre-canned assets which are bundled togather in a meaningful manner. Nginx integration includes dashboards, visualisations, queries and an index mapping. ### Dashboards + ![](../static/dashboard1.png) diff --git a/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json b/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json index 2b37953eea..daca06aa87 100644 --- a/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json @@ -1,47 +1,46 @@ { - "name": "nginx", - "version": "1.0.0", - "displayName": "NginX Dashboard", - "description": "Nginx HTTP server collector", - "license": "Apache-2.0", - "type": "logs", - "labels": ["log","http","communication"], - "author": "OpenSearch", - "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/nginx/info", - "statics": { - "logo": { - "annotation": "NginX Logo", - "path": "logo.svg" - }, - "gallery": [ - { - "annotation": "NginX Dashboard", - "path": "dashboard1.png" - } - ] + "name": "nginx", + "version": "1.0.0", + "displayName": "NginX Dashboard", + "description": "Nginx HTTP server collector", + "license": "Apache-2.0", + "type": "logs", + "labels": ["log", "http", "communication"], + "author": "OpenSearch", + "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/nginx/info", + "statics": { + "logo": { + "annotation": "NginX Logo", + "path": "logo.svg" }, - "components": [ - { - "name": "communication", - "version": "1.0.0" - }, - { - "name": "http", - "version": "1.0.0" - }, - { - "name": "logs", - "version": "1.0.0" - } - ], - "assets": { - "savedObjects": { - "name": "nginx", - "version": "1.0.0" - } + "gallery": [ + { + "annotation": "NginX Dashboard", + "path": "dashboard1.png" + } + ] + }, + "components": [ + { + "name": "communication", + "version": "1.0.0" }, - "sampleData": { - "path": "sample.json" + { + "name": "http", + "version": "1.0.0" + }, + { + "name": "logs", + "version": "1.0.0" + } + ], + "assets": { + "savedObjects": { + "name": "nginx", + "version": "1.0.0" } + }, + "sampleData": { + "path": "sample.json" + } } - diff --git a/server/adaptors/integrations/__data__/repository/nginx/schemas/communication-1.0.0.mapping.json b/server/adaptors/integrations/__data__/repository/nginx/schemas/communication-1.0.0.mapping.json index 97263ad264..d9af5d7193 100644 --- a/server/adaptors/integrations/__data__/repository/nginx/schemas/communication-1.0.0.mapping.json +++ b/server/adaptors/integrations/__data__/repository/nginx/schemas/communication-1.0.0.mapping.json @@ -1,135 +1,135 @@ { - "template": { - "mappings": { - "_meta": { - "version": "1.0.0", - "catalog": "observability", - "type": "logs", - "component": "communication", - "labels": ["communication"] + "template": { + "mappings": { + "_meta": { + "version": "1.0.0", + "catalog": "observability", + "type": "logs", + "component": "communication", + "labels": ["communication"] + }, + "properties": { + "communication": { + "properties": { + "sock.family": { + "type": "keyword", + "ignore_above": 256 }, - "properties": { - "communication": { - "properties": { - "sock.family": { - "type": "keyword", - "ignore_above": 256 - }, - "source": { - "type": "object", - "properties": { - "address": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "domain": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "bytes": { - "type": "long" - }, - "ip": { - "type": "ip" - }, - "port": { - "type": "long" - }, - "mac": { - "type": "keyword", - "ignore_above": 1024 - }, - "packets": { - "type": "long" - }, - "geo": { - "type": "object", - "properties": { - "city_name": { - "type": "keyword" - }, - "country_iso_code": { - "type": "keyword" - }, - "country_name": { - "type": "keyword" - }, - "location": { - "type": "geo_point" - } - } - } - } - }, - "destination": { - "type": "object", - "properties": { - "address": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "domain": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "bytes": { - "type": "long" - }, - "ip": { - "type": "ip" - }, - "port": { - "type": "long" - }, - "mac": { - "type": "keyword", - "ignore_above": 1024 - }, - "packets": { - "type": "long" - }, - "geo": { - "type": "object", - "properties": { - "city_name": { - "type": "keyword" - }, - "country_iso_code": { - "type": "keyword" - }, - "country_name": { - "type": "keyword" - }, - "location": { - "type": "geo_point" - } - } - } - } - } + "source": { + "type": "object", + "properties": { + "address": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 1024 } + } + }, + "domain": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "bytes": { + "type": "long" + }, + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + }, + "mac": { + "type": "keyword", + "ignore_above": 1024 + }, + "packets": { + "type": "long" + }, + "geo": { + "type": "object", + "properties": { + "city_name": { + "type": "keyword" + }, + "country_iso_code": { + "type": "keyword" + }, + "country_name": { + "type": "keyword" + }, + "location": { + "type": "geo_point" + } + } + } + } + }, + "destination": { + "type": "object", + "properties": { + "address": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "domain": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "bytes": { + "type": "long" + }, + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + }, + "mac": { + "type": "keyword", + "ignore_above": 1024 + }, + "packets": { + "type": "long" + }, + "geo": { + "type": "object", + "properties": { + "city_name": { + "type": "keyword" + }, + "country_iso_code": { + "type": "keyword" + }, + "country_name": { + "type": "keyword" + }, + "location": { + "type": "geo_point" + } + } } + } } + } } + } } + } } diff --git a/server/adaptors/integrations/__data__/repository/nginx/schemas/http-1.0.0.mapping.json b/server/adaptors/integrations/__data__/repository/nginx/schemas/http-1.0.0.mapping.json index 8a1e18f04a..5fec510cb8 100644 --- a/server/adaptors/integrations/__data__/repository/nginx/schemas/http-1.0.0.mapping.json +++ b/server/adaptors/integrations/__data__/repository/nginx/schemas/http-1.0.0.mapping.json @@ -1,166 +1,166 @@ { - "template": { - "mappings": { - "_meta": { - "version": "1.0.0", - "catalog": "observability", - "type": "logs", - "component": "http", - "labels": ["http"] + "template": { + "mappings": { + "_meta": { + "version": "1.0.0", + "catalog": "observability", + "type": "logs", + "component": "http", + "labels": ["http"] + }, + "dynamic_templates": [ + { + "request_header_map": { + "mapping": { + "type": "keyword" }, - "dynamic_templates": [ - { - "request_header_map": { - "mapping": { - "type": "keyword" - }, - "path_match": "request.header.*" + "path_match": "request.header.*" + } + }, + { + "response_header_map": { + "mapping": { + "type": "keyword" + }, + "path_match": "response.header.*" + } + } + ], + "properties": { + "http": { + "properties": { + "flavor": { + "type": "keyword", + "ignore_above": 256 + }, + "user_agent": { + "type": "object", + "properties": { + "original": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "version": { + "type": "keyword" + }, + "device": { + "type": "object", + "properties": { + "name": { + "type": "keyword" } + } }, - { - "response_header_map": { - "mapping": { - "type": "keyword" - }, - "path_match": "response.header.*" + "os": { + "type": "object", + "properties": { + "type": { + "type": "keyword" + }, + "platform": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "full": { + "type": "keyword" + }, + "family": { + "type": "keyword" + }, + "version": { + "type": "keyword" + }, + "kernel": { + "type": "keyword" + } + } + } + } + }, + "url": { + "type": "keyword", + "ignore_above": 2048 + }, + "schema": { + "type": "keyword", + "ignore_above": 1024 + }, + "target": { + "type": "keyword", + "ignore_above": 1024 + }, + "route": { + "type": "keyword", + "ignore_above": 1024 + }, + "client.ip": { + "type": "ip" + }, + "resent_count": { + "type": "integer" + }, + "request": { + "type": "object", + "properties": { + "id": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 } + } + }, + "body.content": { + "type": "text" + }, + "bytes": { + "type": "long" + }, + "method": { + "type": "keyword", + "ignore_above": 256 + }, + "referrer": { + "type": "keyword", + "ignore_above": 1024 + }, + "mime_type": { + "type": "keyword", + "ignore_above": 1024 } - ], - "properties": { - "http": { - "properties": { - "flavor": { - "type": "keyword", - "ignore_above": 256 - }, - "user_agent": { - "type": "object", - "properties": { - "original": { - "type": "keyword" - }, - "name": { - "type": "keyword" - }, - "version": { - "type": "keyword" - }, - "device": { - "type": "object", - "properties": { - "name": { - "type": "keyword" - } - } - }, - "os": { - "type": "object", - "properties": { - "type": { - "type": "keyword" - }, - "platform": { - "type": "keyword" - }, - "name": { - "type": "keyword" - }, - "full": { - "type": "keyword" - }, - "family": { - "type": "keyword" - }, - "version": { - "type": "keyword" - }, - "kernel": { - "type": "keyword" - } - } - } - } - }, - "url": { - "type": "keyword", - "ignore_above": 2048 - }, - "schema": { - "type": "keyword", - "ignore_above": 1024 - }, - "target": { - "type": "keyword", - "ignore_above": 1024 - }, - "route": { - "type": "keyword", - "ignore_above": 1024 - }, - "client.ip": { - "type": "ip" - }, - "resent_count": { - "type": "integer" - }, - "request": { - "type": "object", - "properties": { - "id": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "body.content": { - "type": "text" - }, - "bytes": { - "type": "long" - }, - "method": { - "type": "keyword", - "ignore_above": 256 - }, - "referrer": { - "type": "keyword", - "ignore_above": 1024 - }, - "mime_type": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "response": { - "type": "object", - "properties": { - "id": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "body.content": { - "type": "text" - }, - "bytes": { - "type": "long" - }, - "status_code": { - "type": "integer" - } - } - } + } + }, + "response": { + "type": "object", + "properties": { + "id": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 } + } + }, + "body.content": { + "type": "text" + }, + "bytes": { + "type": "long" + }, + "status_code": { + "type": "integer" } + } } + } } + } } + } } diff --git a/server/adaptors/integrations/__data__/repository/nginx/schemas/logs-1.0.0.mapping.json b/server/adaptors/integrations/__data__/repository/nginx/schemas/logs-1.0.0.mapping.json index ab2529668c..054a1be850 100644 --- a/server/adaptors/integrations/__data__/repository/nginx/schemas/logs-1.0.0.mapping.json +++ b/server/adaptors/integrations/__data__/repository/nginx/schemas/logs-1.0.0.mapping.json @@ -1,7 +1,5 @@ { - "index_patterns": [ - "ss4o_logs-*-*" - ], + "index_patterns": ["ss4o_logs-*-*"], "priority": 900, "data_stream": {}, "template": { @@ -11,7 +9,7 @@ "catalog": "observability", "type": "logs", "component": "log", - "labels": ["log","http","communication"], + "labels": ["log", "http", "communication"], "correlations": [ { "field": "spanId", @@ -220,9 +218,6 @@ } } }, - "composed_of": [ - "communication", - "http" - ], + "composed_of": ["communication", "http"], "version": 1 } diff --git a/server/adaptors/integrations/types.ts b/server/adaptors/integrations/types.ts index 7651145a3d..c12476909f 100644 --- a/server/adaptors/integrations/types.ts +++ b/server/adaptors/integrations/types.ts @@ -9,7 +9,7 @@ interface IntegrationTemplate { displayName?: string; license: string; type: string; - labels: string[]; + labels?: string[]; author?: string; description?: string; sourceUrl?: string; diff --git a/server/adaptors/integrations/validators.ts b/server/adaptors/integrations/validators.ts index e5d1008cb1..3cb24212d2 100644 --- a/server/adaptors/integrations/validators.ts +++ b/server/adaptors/integrations/validators.ts @@ -25,7 +25,7 @@ const templateSchema: JSONSchemaType = { displayName: { type: 'string', nullable: true }, license: { type: 'string' }, type: { type: 'string' }, - labels: { type: 'array', items: { type: 'string' }}, + labels: { type: 'array', items: { type: 'string' }, nullable: true }, author: { type: 'string', nullable: true }, description: { type: 'string', nullable: true }, sourceUrl: { type: 'string', nullable: true }, From 69334739e02ba587ff5d54b0626609a326329d24 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Tue, 15 Aug 2023 15:13:46 -0700 Subject: [PATCH 2/3] Fix one more missing label change (#934) * Update label display on frontend Signed-off-by: Simeon Widdis * Clear type errors in added_integration.tsx Signed-off-by: Simeon Widdis * Remove unused imports Signed-off-by: Simeon Widdis * Add missing labels Signed-off-by: Simeon Widdis * Autoformat the whole repository Signed-off-by: Simeon Widdis * Revert large sample data changes Signed-off-by: Simeon Widdis * Undo _import autoescape Signed-off-by: Simeon Widdis * Modify snapshots Signed-off-by: Simeon Widdis * Update label handling in table Signed-off-by: Simeon Widdis --------- Signed-off-by: Simeon Widdis --- .../added_integration.test.tsx.snap | 3 +- ...lable_integration_table_view.test.tsx.snap | 90 +------------------ .../integration_details.test.tsx.snap | 90 +------------------ .../available_integration_table.tsx | 4 +- 4 files changed, 5 insertions(+), 182 deletions(-) diff --git a/public/components/integrations/components/__tests__/__snapshots__/added_integration.test.tsx.snap b/public/components/integrations/components/__tests__/__snapshots__/added_integration.test.tsx.snap index 8c8ad50731..7ef7eb7859 100644 --- a/public/components/integrations/components/__tests__/__snapshots__/added_integration.test.tsx.snap +++ b/public/components/integrations/components/__tests__/__snapshots__/added_integration.test.tsx.snap @@ -227,11 +227,10 @@ exports[`Added Integration View Test Renders added integration view using dummy data-test-subj="deleteInstanceButton" iconType="trash" onClick={[Function]} - size="l" >