From f0f58b7071a2842a39eb14fef8599e342f36ce39 Mon Sep 17 00:00:00 2001 From: Ron cohen Date: Mon, 15 Jul 2019 15:18:49 +0200 Subject: [PATCH] POC servicemap --- package.json | 2 + src/legacy/core_plugins/apm_oss/index.js | 12 +- x-pack/legacy/plugins/apm/index.ts | 97 +++- .../apm/public/components/app/Home/index.tsx | 11 + .../app/Main/route_config/index.tsx | 16 + .../app/Main/route_config/route_names.tsx | 2 + .../app/ServiceDetails/ServiceDetailTabs.tsx | 14 +- .../app/ServiceMap/MapOfServices.tsx | 102 +++++ .../components/app/ServiceMap/index.tsx | 52 +++ .../context/UrlParamsContext/helpers.ts | 3 +- .../apm/public/services/rest/apm/services.ts | 23 + .../apm/server/lib/helpers/setup_request.ts | 1 + .../plugins/apm/server/lib/servicemap.ts | 432 ++++++++++++++++++ .../plugins/apm/server/lib/services/map.ts | 115 +++++ .../plugins/apm/server/routes/services.ts | 26 ++ yarn.lock | 21 + 16 files changed, 917 insertions(+), 12 deletions(-) create mode 100644 x-pack/legacy/plugins/apm/public/components/app/ServiceMap/MapOfServices.tsx create mode 100644 x-pack/legacy/plugins/apm/public/components/app/ServiceMap/index.tsx create mode 100644 x-pack/legacy/plugins/apm/server/lib/servicemap.ts create mode 100644 x-pack/legacy/plugins/apm/server/lib/services/map.ts diff --git a/package.json b/package.json index 2f8a07ab7b29a..94a121a338b14 100644 --- a/package.json +++ b/package.json @@ -147,6 +147,7 @@ "core-js": "2.6.9", "css-loader": "2.1.1", "custom-event-polyfill": "^0.3.0", + "cytoscape": "^3.8.1", "d3": "3.5.17", "d3-cloud": "1.2.5", "del": "^4.0.0", @@ -211,6 +212,7 @@ "react": "^16.8.0", "react-addons-shallow-compare": "15.6.2", "react-color": "^2.13.8", + "react-cytoscapejs": "^1.1.0", "react-dom": "^16.8.0", "react-grid-layout": "^0.16.2", "react-hooks-testing-library": "^0.5.0", diff --git a/src/legacy/core_plugins/apm_oss/index.js b/src/legacy/core_plugins/apm_oss/index.js index 6c0c6d0e5fe52..43d283bafcc1f 100644 --- a/src/legacy/core_plugins/apm_oss/index.js +++ b/src/legacy/core_plugins/apm_oss/index.js @@ -38,7 +38,12 @@ export default function apmOss(kibana) { spanIndices: Joi.string().default('apm-*'), metricsIndices: Joi.string().default('apm-*'), onboardingIndices: Joi.string().default('apm-*'), - apmAgentConfigurationIndex: Joi.string().default('.apm-agent-configuration') + apmAgentConfigurationIndex: Joi.string().default('.apm-agent-configuration'), + + serviceMapIndexPattern: Joi.string().default('apm-*'), + serviceMapDestinationIndex: Joi.string(), + serviceMapDestinationPipeline: Joi.string(), + }).default(); }, @@ -49,7 +54,10 @@ export default function apmOss(kibana) { 'transactionIndices', 'spanIndices', 'metricsIndices', - 'onboardingIndices' + 'onboardingIndices', + 'serviceMapIndexPattern', + 'serviceMapDestinationIndex', + 'serviceMapDestinationPipeline' ].map(type => server.config().get(`apm_oss.${type}`)))); } }); diff --git a/x-pack/legacy/plugins/apm/index.ts b/x-pack/legacy/plugins/apm/index.ts index aac946a36b458..42f62d23e152d 100644 --- a/x-pack/legacy/plugins/apm/index.ts +++ b/x-pack/legacy/plugins/apm/index.ts @@ -14,10 +14,18 @@ import { import { LegacyPluginInitializer } from '../../../../src/legacy/types'; import mappings from './mappings.json'; import { plugin } from './server/new-platform/index'; +import { TaskManager, RunContext } from '../legacy/plugins/task_manager'; +import { serviceMapRun } from './server/lib/servicemap'; export const apm: LegacyPluginInitializer = kibana => { return new kibana.Plugin({ - require: ['kibana', 'elasticsearch', 'xpack_main', 'apm_oss'], + require: [ + 'kibana', + 'elasticsearch', + 'xpack_main', + 'apm_oss', + 'task_manager' + ], id: 'apm', configPrefix: 'xpack.apm', publicDir: resolve(__dirname, 'public'), @@ -104,13 +112,88 @@ export const apm: LegacyPluginInitializer = kibana => { } }); - const initializerContext = {} as PluginInitializerContext; - const core = { - http: { - server + // fires off the job + // needed this during debugging + server.route({ + method: 'GET', + path: '/api/apm/servicemap', + options: { + tags: ['access:apm'] + }, + handler: req => { + return serviceMapRun(this.kbnServer, this.kbnServer.config); } - } as InternalCoreSetup; - plugin(initializerContext).setup(core); + }); + + const { taskManager } = server; + if (taskManager) { + // console.log('registering task'); + taskManager.registerTaskDefinitions({ + // serviceMap is the task type, and must be unique across the entire system + serviceMap: { + // Human friendly name, used to represent this task in logs, UI, etc + title: 'ServiceMapTask', + + // Optional, human-friendly, more detailed description + description: 'Extract connections in traces for service maps', + + // Optional, how long, in minutes, the system should wait before + // a running instance of this task is considered to be timed out. + // This defaults to 5 minutes. + timeout: '5m', + + // The serviceMap task occupies 2 workers, so if the system has 10 worker slots, + // 5 serviceMap tasks could run concurrently per Kibana instance. This value is + // overridden by the `override_num_workers` config value, if specified. + numWorkers: 1, + + // The createTaskRunner function / method returns an object that is responsible for + // performing the work of the task. context: { taskInstance, kbnServer }, is documented below. + createTaskRunner({ kbnServer, taskInstance }: RunContext) { + // Perform the work of the task. The return value should fit the TaskResult interface, documented + // below. Invalid return values will result in a logged warning. + return { + async run() { + const { state } = taskInstance; + + const { mostRecent } = await serviceMapRun( + kbnServer, + kbnServer.config, + state.lastRun + ); + + return { + state: { + count: (state.count || 0) + 1, + lastRun: mostRecent + } + }; + } + }; + } + } + }); + + this.kbnServer.afterPluginsInit(() => { + // console.log('ahout to schedule'); + const task = taskManager.schedule({ + id: 'servicemap-processor', + taskType: 'serviceMap', + interval: '1m', + scope: ['apm'] + }); + // .catch(e => console.log('Err scheduling', e)); + // console.log('scheduled', JSON.stringify(task)); + }); + + const initializerContext = {} as PluginInitializerContext; + const core = { + http: { + server + } + } as InternalCoreSetup; + plugin(initializerContext).setup(core); + } } }); }; diff --git a/x-pack/legacy/plugins/apm/public/components/app/Home/index.tsx b/x-pack/legacy/plugins/apm/public/components/app/Home/index.tsx index 665bcf7e1aacb..3d8e29a31aace 100644 --- a/x-pack/legacy/plugins/apm/public/components/app/Home/index.tsx +++ b/x-pack/legacy/plugins/apm/public/components/app/Home/index.tsx @@ -17,7 +17,10 @@ import { HistoryTabs, IHistoryTab } from '../../shared/HistoryTabs'; import { SetupInstructionsLink } from '../../shared/Links/SetupInstructionsLink'; import { ServiceOverview } from '../ServiceOverview'; import { TraceOverview } from '../TraceOverview'; +import { ServiceMap } from '../ServiceMap'; + import { APMLink } from '../../shared/Links/APMLink'; +import { useUrlParams } from '../../../hooks/useUrlParams'; const homeTabs: IHistoryTab[] = [ { @@ -35,6 +38,14 @@ const homeTabs: IHistoryTab[] = [ }), render: () => , name: 'traces' + }, + { + path: '/servicemap', + title: i18n.translate('xpack.apm.home.tracesTabLabel', { + defaultMessage: 'Service Map' + }), + render: () => , + name: 'servicemap' } ]; diff --git a/x-pack/legacy/plugins/apm/public/components/app/Main/route_config/index.tsx b/x-pack/legacy/plugins/apm/public/components/app/Main/route_config/index.tsx index 5a2c16c9fbfe0..935b5eca058a7 100644 --- a/x-pack/legacy/plugins/apm/public/components/app/Main/route_config/index.tsx +++ b/x-pack/legacy/plugins/apm/public/components/app/Main/route_config/index.tsx @@ -57,6 +57,15 @@ export const routes: BreadcrumbRoute[] = [ }), name: RouteName.TRACES }, + { + exact: true, + path: '/servicemap', + component: Home, + breadcrumb: i18n.translate('xpack.apm.breadcrumb.tracesTitle', { + defaultMessage: 'Service Map' + }), + name: RouteName.SERVICEMAP + }, { exact: true, path: '/settings', @@ -126,5 +135,12 @@ export const routes: BreadcrumbRoute[] = [ breadcrumb: ({ match }) => legacyDecodeURIComponent(match.params.transactionName) || '', name: RouteName.TRANSACTION_NAME + }, + { + exact: true, + path: '/:serviceName/servicemap', + component: ServiceDetails, + breadcrumb: null, + name: RouteName.SERVICE_SERVICEMAP } ]; diff --git a/x-pack/legacy/plugins/apm/public/components/app/Main/route_config/route_names.tsx b/x-pack/legacy/plugins/apm/public/components/app/Main/route_config/route_names.tsx index 10b633c40560f..954c2163d2571 100644 --- a/x-pack/legacy/plugins/apm/public/components/app/Main/route_config/route_names.tsx +++ b/x-pack/legacy/plugins/apm/public/components/app/Main/route_config/route_names.tsx @@ -9,7 +9,9 @@ export enum RouteName { SERVICES = 'services', TRACES = 'traces', SERVICE = 'service', + SERVICEMAP = 'servicemap', TRANSACTIONS = 'transactions', + SERVICE_SERVICEMAP = 'service_servicemap', ERRORS = 'errors', ERROR = 'error', METRICS = 'metrics', diff --git a/x-pack/legacy/plugins/apm/public/components/app/ServiceDetails/ServiceDetailTabs.tsx b/x-pack/legacy/plugins/apm/public/components/app/ServiceDetails/ServiceDetailTabs.tsx index ec7d3a827ce5e..091275f80350a 100644 --- a/x-pack/legacy/plugins/apm/public/components/app/ServiceDetails/ServiceDetailTabs.tsx +++ b/x-pack/legacy/plugins/apm/public/components/app/ServiceDetails/ServiceDetailTabs.tsx @@ -11,6 +11,7 @@ import { HistoryTabs } from '../../shared/HistoryTabs'; import { ErrorGroupOverview } from '../ErrorGroupOverview'; import { TransactionOverview } from '../TransactionOverview'; import { ServiceMetrics } from '../ServiceMetrics'; +import { ServiceMap } from '../ServiceMap'; interface Props { transactionTypes: string[]; @@ -61,9 +62,18 @@ export function ServiceDetailTabs({ render: () => , name: 'metrics' }; + + const serviceMapTab = { + title: i18n.translate('xpack.apm.serviceDetails.mapTabLabel', { + defaultMessage: 'Service Map' + }), + path: `/${serviceName}/servicemap`, + render: () => , + name: 'servicemap' + }; const tabs = isRumAgent - ? [transactionsTab, errorsTab] - : [transactionsTab, errorsTab, metricsTab]; + ? [transactionsTab, errorsTab, serviceMapTab] + : [transactionsTab, errorsTab, serviceMapTab, metricsTab]; return ; } diff --git a/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/MapOfServices.tsx b/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/MapOfServices.tsx new file mode 100644 index 0000000000000..4f4b4b837f782 --- /dev/null +++ b/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/MapOfServices.tsx @@ -0,0 +1,102 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import CytoscapeComponent from 'react-cytoscapejs'; +import { IUrlParams } from '../../../context/UrlParamsContext/types'; + +function elementId(serviceName: string, environment?: string) { + return serviceName + '/' + (environment ? '/' + environment : ''); +} + +function label(serviceName: string, environment?: string) { + return serviceName + (environment ? '/' + environment : ''); +} + +// interface Props { +// connections: any; +// } +export function MapOfServices({ connections, layout }: any) { + const services: { [s: string]: object } = {}; + const conns: Array<{ data: object }> = []; + let destNodeID; + connections.forEach(c => { + if (c['callee.name']) { + destNodeID = elementId(c['callee.name'], c['callee.environment']); + const node = { + data: { + id: destNodeID, + label: label(c['callee.name'], c['callee.environment']), + color: 'black' + } + }; + services[destNodeID] = node; + } else { + destNodeID = elementId(c['destination.address']); + services[destNodeID] = { + data: { + id: destNodeID, + label: label(c['destination.address']) + } + }; + } + const sourceNodeID = elementId(c['service.name'], c['service.environment']); + + services[sourceNodeID] = { + data: { + id: sourceNodeID, + label: label(c['service.name'], c['service.environment']), + color: 'black' + } + }; + + conns.push({ + data: { + source: sourceNodeID, + target: destNodeID, + label: c['connection.subtype'] + } + }); + }); + const elements = Object.values(services).concat(conns); + // const elements = []; + // { data: { id: 'one', label: 'Node 1' }}, + // { data: { id: 'two', label: 'Node 2' }}, + // { data: { source: 'one', target: 'two', label: 'Edge from Node1 to Node2' } } + // ]; + const stylesheet = [ + { + selector: 'node', + style: { + label: 'data(label)' // maps to data.label + } + }, + { + selector: 'node[color]', + style: { + 'background-color': 'data(color)' + } + }, + { + selector: 'edge', + style: { + width: 2, + label: 'data(label)', // maps to data.label + 'curve-style': 'bezier', + 'target-arrow-shape': 'triangle' + } + } + ]; + + return ( + + ); +} diff --git a/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/index.tsx b/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/index.tsx new file mode 100644 index 0000000000000..00dc80dc1b36a --- /dev/null +++ b/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/index.tsx @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { EuiPanel } from '@elastic/eui'; +import React from 'react'; +import { useFetcher } from '../../../hooks/useFetcher'; +import { loadServiceMap } from '../../../services/rest/apm/services'; +import { MapOfServices } from './MapOfServices'; +import { useUrlParams } from '../../../hooks/useUrlParams'; + +const initalData = { + items: [], + hasHistoricalData: true, + hasLegacyData: false +}; + +interface Props { + global?: boolean; + layout?: string; +} + +export function ServiceMap({ global = false, layout = 'circle' }: Props) { + // console.log(global); + const { + urlParams: { start, end, serviceName }, + uiFilters + } = useUrlParams(); + + const { data = initalData } = useFetcher(() => { + if (start && end) { + return loadServiceMap({ + start, + end, + // TODO: i did this because useUrlParams() does not return + // the right service name when used at the higher level + serviceName: global ? undefined : serviceName, + uiFilters + }); + } + }, [start, end, uiFilters]); + + return ( + + {data.map ? ( + + ) : null} + + ); +} diff --git a/x-pack/legacy/plugins/apm/public/context/UrlParamsContext/helpers.ts b/x-pack/legacy/plugins/apm/public/context/UrlParamsContext/helpers.ts index 12c58d8ad54cc..261d1a0ebec15 100644 --- a/x-pack/legacy/plugins/apm/public/context/UrlParamsContext/helpers.ts +++ b/x-pack/legacy/plugins/apm/public/context/UrlParamsContext/helpers.ts @@ -59,7 +59,6 @@ export function removeUndefinedProps(obj: T): Partial { export function getPathParams(pathname: string = '') { const paths = getPathAsArray(pathname); const pageName = paths.length > 1 ? paths[1] : paths[0]; - // TODO: use react router's real match params instead of guessing the path order switch (pageName) { case 'transactions': @@ -69,6 +68,8 @@ export function getPathParams(pathname: string = '') { transactionType: paths[2], transactionName: paths[3] }; + case 'servicemap': + return { serviceName: paths[0] }; case 'errors': return { processorEvent: 'error', diff --git a/x-pack/legacy/plugins/apm/public/services/rest/apm/services.ts b/x-pack/legacy/plugins/apm/public/services/rest/apm/services.ts index 538cdfa79fdb2..5f30ad84fc42d 100644 --- a/x-pack/legacy/plugins/apm/public/services/rest/apm/services.ts +++ b/x-pack/legacy/plugins/apm/public/services/rest/apm/services.ts @@ -9,6 +9,7 @@ import { ServiceListAPIResponse } from '../../../../server/lib/services/get_serv import { callApi } from '../callApi'; import { getUiFiltersES } from '../../ui_filters/get_ui_filters_es'; import { UIFilters } from '../../../../typings/ui-filters'; +import { ServiceMapAPIResponse } from '../../../../server/lib/services/map'; export async function loadServiceList({ start, @@ -49,3 +50,25 @@ export async function loadServiceDetails({ } }); } + +export async function loadServiceMap({ + start, + end, + uiFilters, + serviceName +}: { + start: string; + end: string; + uiFilters: UIFilters; + serviceName?: string; +}) { + return callApi({ + pathname: '/api/apm/services/map', + query: { + start, + end, + serviceName, + uiFiltersES: await getUiFiltersES(uiFilters) + } + }); +} diff --git a/x-pack/legacy/plugins/apm/server/lib/helpers/setup_request.ts b/x-pack/legacy/plugins/apm/server/lib/helpers/setup_request.ts index 20db3fc6bd526..8138a1cd4f01e 100644 --- a/x-pack/legacy/plugins/apm/server/lib/helpers/setup_request.ts +++ b/x-pack/legacy/plugins/apm/server/lib/helpers/setup_request.ts @@ -17,6 +17,7 @@ export interface APMRequestQuery { start: string; end: string; uiFiltersES?: string; + serviceName?: string; } export type Setup = ReturnType; diff --git a/x-pack/legacy/plugins/apm/server/lib/servicemap.ts b/x-pack/legacy/plugins/apm/server/lib/servicemap.ts new file mode 100644 index 0000000000000..2598f32240c20 --- /dev/null +++ b/x-pack/legacy/plugins/apm/server/lib/servicemap.ts @@ -0,0 +1,432 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +/* + +Necessary scripts: + +POST _scripts/map-service-conns +{ + "script": { + "lang": "painless", + "source": """ + def s = new HashMap(); + + if (!doc['span.id'].empty) { + s.id = doc['span.id'].value + } else { + s.id = doc['transaction.id'].value; + s.transaction = true; + } + if (!doc['parent.id'].empty) { + s.parent = doc['parent.id'].value; + } + if (!doc['service.environment'].empty) { + s.environment = doc['service.environment'].value; + } + + if (!doc['destination.address'].empty) { + s.destination = doc['destination.address'].value; + } + + if (!doc['_index'].empty) { + s._index = doc['_index'].value; + } + + if (!doc['span.type'].empty) { + s.span_type = doc['span.type'].value; + } + + if (!doc['span.subtype'].empty) { + s.span_subtype = doc['span.subtype'].value; + } + + s.timestamp = doc['@timestamp'].value; + s.service_name = doc['service.name'].value; + if(!state.spans.containsKey(s.parent)) { + state.spans.put(s.parent, new ArrayList()) + } + + if (s.parent != s.id) { + state.spans[s.parent].add(s) + } + """ + } +} + +POST _scripts/reduce-service-conns +{ + "script": { + "lang": "painless", + "source": """ + void extractChildren(def caller, def spans, def upstream, def conns, def count) { + // TODO: simplify this + if (spans.containsKey(caller.id)) { + for(s in spans[caller.id]) { + if (caller.span_type=='external') { + upstream.add(caller.service_name+"/"+caller.environment); + + def conn = new HashMap(); + conn.caller = caller; + conn.callee = s; + conn.upstream = new ArrayList(upstream); + conns.add(conn); + + extractChildren(s, spans, upstream, conns, count); + upstream.remove(upstream.size()-1); + } else { + extractChildren(s, spans, upstream, conns, count); + } + } + } else { + // no connection found + def conn = new HashMap(); + conn.caller = caller; + conn.upstream = new ArrayList(upstream); + conn.upstream.add(caller.service_name+"/"+caller.environment); + conns.add(conn); + } + } + def conns = new HashSet(); + def spans = new HashMap(); + + // merge results from shards + for(state in states) { + for(s in state.entrySet()) { + def v = s.getValue(); + def k = s.getKey(); + if(!spans.containsKey(k)) { + spans[k] = v; + } else { + for (p in v) { + spans[k].add(p); + } + } + } + } + + if (spans.containsKey(null) && spans[null].size() > 0) { + def node = spans[null][0]; + def upstream = new ArrayList(); + + extractChildren(node, spans, upstream, conns, 0); + + return new ArrayList(conns) + } + return []; + """ + } +} + +POST _scripts/combine-service-conns +{ + "script": { + "lang": "painless", + "source": "return state.spans" + } +} + + + +PUT /_ingest/pipeline/extract_destination +{ + "description": "sets destination on ext spans based on their name", + "processors": [ + { + "set": { + "if": "ctx.span != null && ctx.span.type == 'ext'", + "field": "span.type", + "value": "external" + } + }, + { + "script": """ + if(ctx['span'] != null) { + if (ctx['span']['type'] == 'external') { + def spanName = ctx['span']['name']; + if (spanName.indexOf('/') > -1) { + spanName = spanName.substring(0, spanName.indexOf('/')); + } + + if (spanName.indexOf(' ') > -1) { + spanName = spanName.substring(spanName.indexOf(' ')+1, spanName.length()); + } + ctx['destination.address']=spanName; + } + + if (ctx['span']['type'] == 'resource') { + def spanName = ctx['span']['name']; + + if (spanName.indexOf('://') > -1) { + spanName = spanName.substring(spanName.indexOf('://')+3, spanName.length()); + } + if (spanName.indexOf('/') > -1) { + spanName = spanName.substring(0, spanName.indexOf('/')); + } + + ctx['destination.address']=spanName; + } + + if (ctx['span']['type'] == 'db') { + def dest = ctx['span']['subtype']; + ctx['destination.address']=dest; + } + + if (ctx['span']['type'] == 'cache') { + def dest = ctx['span']['subtype']; + ctx['destination.address']=dest; + } + } + """ + } + ] +} + +*/ + +function interestingTransactions(since?: string, afterKey?: any) { + if (!since) { + since = 'now-1h'; + } + const query = { + size: 0, + query: { + bool: { + filter: [ + { exists: { field: 'destination.address' } }, + { exists: { field: 'trace.id' } }, + { exists: { field: 'span.duration.us' } }, + { range: { '@timestamp': { gt: since } } } + ] + } + }, + aggs: { + 'ext-conns': { + composite: { + sources: [ + { 'service.name': { terms: { field: 'service.name' } } }, + { 'span.type': { terms: { field: 'span.type' } } }, + { + 'span.subtype': { + terms: { field: 'span.subtype', missing_bucket: true } + } + }, + { + 'service.environment': { + terms: { field: 'service.environment', missing_bucket: true } + } + }, + { + 'destination.address': { terms: { field: 'destination.address' } } + } + ], + // TODO: needs to be balanced with the 20 below + size: 200 + }, + aggs: { + smpl: { + diversified_sampler: { + shard_size: 20, + script: { + lang: 'painless', + source: "(int)doc['span.duration.us'].value/100000" + } + }, + aggs: { + tracesample: { + top_hits: { + size: 20, + _source: ['trace.id', '@timestamp'] + } + } + } + } + } + } + } + }; + + if (afterKey) { + query.aggs['ext-conns'].composite.after = afterKey; + } + return query; +} + +function findConns(traceIds: string[]) { + const query = { + size: 0, + query: { + bool: { + should: [ + { exists: { field: 'span.id' } }, + { exists: { field: 'transaction.type' } } + ], + minimum_should_match: 2 + } + }, + aggs: { + trace_id: { + terms: { + field: 'trace.id', + order: { _key: 'asc' }, + size: traceIds.length + }, + aggs: { + connections: { + scripted_metric: { + init_script: 'state.spans = new HashMap();', + map_script: { id: 'map-service-conns' }, + combine_script: { id: 'combine-service-conns' }, + reduce_script: { id: 'reduce-service-conns' } + } + } + } + } + } + }; + + for (const tid of traceIds) { + query.query.bool.should.push({ term: { 'trace.id': tid } }); + } + return query; +} + +interface Service { + name: string; + environment?: string; +} + +interface ConnectionDoc { + '@timestamp': string; + observer: { version_major: number }; // TODO: make this dynamic + service: Service; + callee?: Service; + connection: Connection; + destination?: { address: string }; +} + +interface Connection { + upstream: { list: string[]; keyword: string }; + in_trace: string[]; + type: string; + subtype?: string; +} + +export async function serviceMapRun(kbnServer, config, lastRun?: string) { + const apmIdxPattern = config.get('apm_oss.indexPattern'); + const serviceConnsDestinationIndex = config.get( + 'apm_oss.serviceMapDestinationIndex' + ); + const serviceConnsDestinationPipeline = config.get( + 'apm_oss.serviceMapDestinationPipeline' + ); + + const callCluster = kbnServer.server.plugins.elasticsearch.getCluster('data') + .callWithInternalUser; + + let mostRecent = ''; + let afterKey = null; + // console.log('HELLO TASK', lastRun); + while (true) { + const q = interestingTransactions(lastRun, afterKey); + const txs = await callCluster('search', { + index: apmIdxPattern, + body: q + }); + + if (txs.aggregations['ext-conns'].buckets.length < 1) { + // console.log('No buckets'); + // console.log(JSON.stringify(q)); + return { mostRecent }; + } + afterKey = txs.aggregations['ext-conns'].after_key; + + const traces = new Set(); + + txs.aggregations['ext-conns'].buckets.forEach((bucket: any) => { + bucket.smpl.tracesample.hits.hits.forEach((hit: any) => { + traces.add(hit._source.trace.id); + mostRecent = + mostRecent > hit._source['@timestamp'] + ? mostRecent + : hit._source['@timestamp']; + }); + }); + + const traceIds = Array.from(traces.values()); + if (traceIds.length < 1) { + return { mostRecent: null }; + } + + const findConnsQ = findConns(traceIds); + + const connsResult = await callCluster('search', { + index: apmIdxPattern, + body: findConnsQ + }); + + const connDocs = []; + + connsResult.aggregations.trace_id.buckets.forEach((bucket: any) => { + const allServices = new Set( + bucket.connections.value.map( + (conn: any) => + conn.caller.service_name + + '/' + + (conn.caller.environment ? conn.caller.environment : 'null') + ) + ); + + bucket.connections.value.forEach((conn: any) => { + const index = serviceConnsDestinationIndex + ? serviceConnsDestinationIndex + : conn.caller._index; + const bulkOpts = { index: { _index: index } }; + + if (serviceConnsDestinationPipeline) { + bulkOpts.pipeline = serviceConnsDestinationPipeline; + } + connDocs.push(bulkOpts); + const doc: ConnectionDoc = { + '@timestamp': conn.caller.timestamp, + observer: { version_major: 7 }, // TODO: make this dynamic + service: { + name: conn.caller.service_name, + environment: conn.caller.environment + }, + callee: conn.callee + ? { + name: conn.callee.service_name, + environment: conn.callee.environment + } + : undefined, + connection: { + upstream: { + list: conn.upstream, + keyword: conn.upstream.join('->') + }, + in_trace: Array.from(allServices), + type: conn.caller.span_type, + subtype: conn.caller.span_substype + }, + destination: conn.caller.destination + ? { address: conn.caller.destination } + : undefined + }; + + connDocs.push(doc); + }); + }); + + // console.log(JSON.stringify(connDocs)); + const body = connDocs.map(JSON.stringify).join('\n'); + // console.log(body); + const bulkResult = await callCluster('bulk', { + body + }); + // TODO: check result + } +} diff --git a/x-pack/legacy/plugins/apm/server/lib/services/map.ts b/x-pack/legacy/plugins/apm/server/lib/services/map.ts new file mode 100644 index 0000000000000..f14e9c7d3dbd1 --- /dev/null +++ b/x-pack/legacy/plugins/apm/server/lib/services/map.ts @@ -0,0 +1,115 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { idx } from '@kbn/elastic-idx'; +import { PromiseReturnType } from '../../../typings/common'; +import { rangeFilter } from '../helpers/range_filter'; +import { Setup } from '../helpers/setup_request'; + +export interface IEnvOptions { + setup: Setup; + serviceName?: string; + environment?: string; +} + +export type ServiceMapAPIResponse = PromiseReturnType; +export async function getConnections({ + setup, + serviceName, + environment +}: IEnvOptions) { + const { start, end, client, config } = setup; + + const params = { + index: config.get('apm_oss.serviceMapIndexPattern'), + body: { + size: 0, + query: { + bool: { + filter: [ + { range: rangeFilter(start, end) }, + { exists: { field: 'connection.type' } } + ] + } + }, + aggs: { + conns: { + composite: { + sources: [ + { 'service.name': { terms: { field: 'service.name' } } }, + { + 'service.environment': { + terms: { field: 'service.environment', missing_bucket: true } + } + }, + { + 'destination.address': { + terms: { + field: 'destination.address' + } + } + }, + { 'connection.type': { terms: { field: 'connection.type' } } }, // will filter out regular spans etc. + { + 'connection.subtype': { + terms: { field: 'connection.subtype', missing_bucket: true } + } + } + ], + size: 1000 + }, + aggs: { + dests: { + terms: { + field: 'callee.name' + }, + aggs: { + envs: { + terms: { + field: 'callee.environment', + missing: '' + } + } + } + } + } + } + } + } + }; + + if (serviceName) { + serviceName = serviceName + '/' + (environment ? environment : 'null'); + params.body.query.bool.filter.push({ + term: { ['connection.upstream.list']: serviceName } + }); + } + + // console.log(JSON.stringify(params.body)); + + const { aggregations } = await client.search(params); + // console.log(JSON.stringify(aggregations)); + const buckets = idx(aggregations, _ => _.conns.buckets) || []; + const conns = buckets.flatMap((bucket: any) => { + const base = bucket.key; + if (bucket.dests.buckets.length === 0) { + // no connection found + return base; + } else { + return bucket.dests.buckets.flatMap((calleeResult: any) => + calleeResult.envs.buckets.map((env: any) => { + return { + 'callee.environment': env.key.length >= 0 ? env.key : null, + 'callee.name': calleeResult.key, + // 'upstream.list': base['connection.upstream.keyword'].split('->'), + ...base + }; + }) + ); + } + }); + + return { conns }; +} diff --git a/x-pack/legacy/plugins/apm/server/routes/services.ts b/x-pack/legacy/plugins/apm/server/routes/services.ts index 063bf0556daf5..3ec7a4497f029 100644 --- a/x-pack/legacy/plugins/apm/server/routes/services.ts +++ b/x-pack/legacy/plugins/apm/server/routes/services.ts @@ -5,6 +5,8 @@ */ import Boom from 'boom'; +import Joi from 'joi'; + import { InternalCoreSetup } from 'src/core/server'; import { AgentName } from '../../typings/es_schemas/ui/fields/Agent'; import { createApmTelementry, storeApmTelemetry } from '../lib/apm_telemetry'; @@ -12,6 +14,7 @@ import { withDefaultValidators } from '../lib/helpers/input_validation'; import { setupRequest } from '../lib/helpers/setup_request'; import { getService } from '../lib/services/get_service'; import { getServices } from '../lib/services/get_services'; +import { getConnections } from '../lib/services/map'; const ROOT = '/api/apm/services'; const defaultErrorHandler = (err: Error) => { @@ -46,6 +49,29 @@ export function initServicesApi(core: InternalCoreSetup) { } }); + server.route({ + method: 'GET', + path: `${ROOT}/map`, + options: { + validate: { + query: withDefaultValidators({ serviceName: Joi.string() }) + }, + tags: ['access:apm'] + }, + handler: async req => { + const setup = setupRequest(req); + + const { serviceName, environment } = req.query; + const mapProm = getConnections({ setup, serviceName, environment }).catch( + defaultErrorHandler + ); + + const map = await mapProm; + + return { map }; + } + }); + server.route({ method: 'GET', path: `${ROOT}/{serviceName}`, diff --git a/yarn.lock b/yarn.lock index a44da7ff970b5..78033bec213f3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9544,6 +9544,14 @@ cypress@^3.3.1: url "0.11.0" yauzl "2.10.0" +cytoscape@^3.2.19, cytoscape@^3.8.1: + version "3.8.1" + resolved "https://registry.yarnpkg.com/cytoscape/-/cytoscape-3.8.1.tgz#0b87f646054ea3e86f347e9c53a2fc64df1be31b" + integrity sha512-hfoGBlqj8LJPnU5WgmQKdGpZaMknk8dPyGEJlzLa1xPwhmBscPMrmPxK2ffyKJIozeHm4UoYw4gPc0rAif4kVA== + dependencies: + heap "^0.2.6" + lodash.debounce "^4.0.8" + d3-array@1, d3-array@^1.1.1, d3-array@^1.2.0, d3-array@^1.2.4: version "1.2.4" resolved "https://registry.yarnpkg.com/d3-array/-/d3-array-1.2.4.tgz#635ce4d5eea759f6f605863dbcfc30edc737f71f" @@ -14796,6 +14804,11 @@ header-case@^1.0.0: no-case "^2.2.0" upper-case "^1.1.3" +heap@^0.2.6: + version "0.2.6" + resolved "https://registry.yarnpkg.com/heap/-/heap-0.2.6.tgz#087e1f10b046932fc8594dd9e6d378afc9d1e5ac" + integrity sha1-CH4fELBGky/IWU3Z5tN4r8nR5aw= + heavy@6.x.x: version "6.1.0" resolved "https://registry.yarnpkg.com/heavy/-/heavy-6.1.0.tgz#1bbfa43dc61dd4b543ede3ff87db8306b7967274" @@ -22746,6 +22759,14 @@ react-color@^2.17.0: reactcss "^1.2.0" tinycolor2 "^1.4.1" +react-cytoscapejs@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/react-cytoscapejs/-/react-cytoscapejs-1.1.0.tgz#2175e444f12f4023901a960f71e761cb5b5d0b3a" + integrity sha512-CdeEVsOQmw3Cqz47Hx83RelZLT9y3ct6t+YTQ0nRdAt7n+bYD/bTskAmfAm1GvQhzCKylYi2VcY05B8nC4u2Gw== + dependencies: + cytoscape "^3.2.19" + prop-types "^15.6.2" + react-datepicker@v1.4.1: version "1.4.1" resolved "https://registry.yarnpkg.com/react-datepicker/-/react-datepicker-1.4.1.tgz#ee171b71d9853e56f9eece5fc3186402f4648683"