From 6f4ae989a44671cd20f74592539382989bf68368 Mon Sep 17 00:00:00 2001 From: Oliver Gupte Date: Tue, 26 Nov 2019 00:05:29 -0800 Subject: [PATCH 01/15] Adds data later for the service maps feature in APM: - creates required scripts in elasticsearch - adds ingest pipeline that sets destination.address from span names - schedules periodic task that create connection documents - implements endpoints that query/filters connections to render the service map --- src/legacy/core_plugins/apm_oss/index.js | 1 + x-pack/legacy/plugins/apm/index.ts | 23 +- .../components/app/ServiceMap/index.tsx | 42 ++- .../server/lib/service_map/get_service_map.ts | 164 ++++++++++ .../service_map/initialize_service_maps.ts | 109 +++++++ .../lib/service_map/run_service_map_task.ts | 248 +++++++++++++++ .../lib/service_map/setup_required_scripts.ts | 298 ++++++++++++++++++ .../plugins/apm/server/lib/services/map.ts | 88 ------ .../apm/server/routes/create_apm_api.ts | 4 +- .../plugins/apm/server/routes/services.ts | 47 ++- x-pack/plugins/apm/server/index.ts | 6 + 11 files changed, 926 insertions(+), 104 deletions(-) create mode 100644 x-pack/legacy/plugins/apm/server/lib/service_map/get_service_map.ts create mode 100644 x-pack/legacy/plugins/apm/server/lib/service_map/initialize_service_maps.ts create mode 100644 x-pack/legacy/plugins/apm/server/lib/service_map/run_service_map_task.ts create mode 100644 x-pack/legacy/plugins/apm/server/lib/service_map/setup_required_scripts.ts delete mode 100644 x-pack/legacy/plugins/apm/server/lib/services/map.ts diff --git a/src/legacy/core_plugins/apm_oss/index.js b/src/legacy/core_plugins/apm_oss/index.js index 5923c1e85ee12..31ab905d6f628 100644 --- a/src/legacy/core_plugins/apm_oss/index.js +++ b/src/legacy/core_plugins/apm_oss/index.js @@ -38,6 +38,7 @@ export default function apmOss(kibana) { spanIndices: Joi.string().default('apm-*'), metricsIndices: Joi.string().default('apm-*'), onboardingIndices: Joi.string().default('apm-*'), + apmAgentConfigurationIndex: Joi.string().default('.apm-agent-configuration'), }).default(); }, diff --git a/x-pack/legacy/plugins/apm/index.ts b/x-pack/legacy/plugins/apm/index.ts index cf2cbd2507215..8fd08fd79f531 100644 --- a/x-pack/legacy/plugins/apm/index.ts +++ b/x-pack/legacy/plugins/apm/index.ts @@ -11,10 +11,17 @@ import { APMPluginContract } from '../../../plugins/apm/server'; import { LegacyPluginInitializer } from '../../../../src/legacy/types'; import mappings from './mappings.json'; import { makeApmUsageCollector } from './server/lib/apm_telemetry'; +import { initializeServiceMaps } from './server/lib/service_map/initialize_service_maps'; export const apm: LegacyPluginInitializer = kibana => { return new kibana.Plugin({ - require: ['kibana', 'elasticsearch', 'xpack_main', 'apm_oss'], + require: [ + 'kibana', + 'elasticsearch', + 'xpack_main', + 'apm_oss', + 'task_manager' + ], id: 'apm', configPrefix: 'xpack.apm', publicDir: resolve(__dirname, 'public'), @@ -71,12 +78,18 @@ export const apm: LegacyPluginInitializer = kibana => { autocreateApmIndexPattern: Joi.boolean().default(true), // service map - serviceMapEnabled: Joi.boolean().default(false) + + serviceMapEnabled: Joi.boolean().default(false), + serviceMapIndexPattern: Joi.string().default('apm-*'), + serviceMapDestinationIndex: Joi.string(), + serviceMapDestinationPipeline: Joi.string() }).default(); }, // TODO: get proper types init(server: Server) { + const config = server.config(); + server.plugins.xpack_main.registerFeature({ id: 'apm', name: i18n.translate('xpack.apm.featureRegistry.apmFeatureName', { @@ -113,6 +126,12 @@ export const apm: LegacyPluginInitializer = kibana => { .apm as APMPluginContract; apmPlugin.registerLegacyAPI({ server }); + + if (config.get('xpack.apm.serviceMapEnabled')) { + initializeServiceMaps(server).catch(error => { + throw error; + }); + } } }); }; diff --git a/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/index.tsx b/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/index.tsx index 16a91116ae762..1b7680907fe3e 100644 --- a/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/index.tsx +++ b/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/index.tsx @@ -5,7 +5,7 @@ */ import theme from '@elastic/eui/dist/eui_theme_light.json'; -import React from 'react'; +import React, { useMemo } from 'react'; import { useFetcher } from '../../../hooks/useFetcher'; import { useLicense } from '../../../hooks/useLicense'; import { useUrlParams } from '../../../hooks/useUrlParams'; @@ -39,19 +39,51 @@ ${theme.euiColorLightShade}`, export function ServiceMap({ serviceName }: ServiceMapProps) { const { - urlParams: { start, end } + urlParams: { start, end, environment }, + uiFilters } = useUrlParams(); + const uiFiltersOmitEnv = useMemo( + () => ({ + ...uiFilters, + environment: undefined + }), + [uiFilters] + ); + const { data } = useFetcher( callApmApi => { + if (serviceName && start && end) { + return callApmApi({ + pathname: '/api/apm/service-map/{serviceName}', + params: { + path: { + serviceName + }, + query: { + start, + end, + environment, + uiFilters: JSON.stringify(uiFiltersOmitEnv) + } + } + }); + } if (start && end) { return callApmApi({ - pathname: '/api/apm/service-map', - params: { query: { start, end } } + pathname: '/api/apm/service-map/all', + params: { + query: { + start, + end, + environment, + uiFilters: JSON.stringify(uiFiltersOmitEnv) + } + } }); } }, - [start, end] + [start, end, uiFiltersOmitEnv, environment, serviceName] ); const elements = Array.isArray(data) ? data : []; diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_map.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_map.ts new file mode 100644 index 0000000000000..911e6b9d6c178 --- /dev/null +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_map.ts @@ -0,0 +1,164 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { idx } from '@kbn/elastic-idx'; +import { PromiseReturnType } from '../../../typings/common'; +import { rangeFilter } from '../helpers/range_filter'; +import { + Setup, + SetupTimeRange, + SetupUIFilters +} from '../helpers/setup_request'; +import { ENVIRONMENT_NOT_DEFINED } from '../../../common/environment_filter_values'; + +export interface IEnvOptions { + setup: Setup & SetupTimeRange & SetupUIFilters; + serviceName?: string; + environment?: string; +} + +export type ServiceMapAPIResponse = PromiseReturnType; +export async function getServiceMap({ + setup, + serviceName, + environment +}: IEnvOptions) { + const { start, end, client, config, uiFiltersES } = setup; + + const params = { + index: config['xpack.apm.serviceMapIndexPattern'], + body: { + size: 0, + query: { + bool: { + filter: [ + { range: rangeFilter(start, end) }, + { exists: { field: 'connection.type' } }, + ...uiFiltersES + ] + } + }, + aggs: { + conns: { + composite: { + sources: [ + { 'service.name': { terms: { field: 'service.name' } } }, + { + 'service.environment': { + terms: { field: 'service.environment', missing_bucket: true } + } + }, + { + 'destination.address': { + terms: { + field: 'destination.address' + } + } + }, + { 'connection.type': { terms: { field: 'connection.type' } } }, // will filter out regular spans etc. + { + 'connection.subtype': { + terms: { field: 'connection.subtype', missing_bucket: true } + } + } + ], + size: 1000 + }, + aggs: { + dests: { + terms: { + field: 'callee.name' + }, + aggs: { + envs: { + terms: { + field: 'callee.environment', + missing: '' + } + } + } + } + } + } + } + } + }; + + if (serviceName || environment) { + const upstreamServiceName = serviceName || '*'; + + let upstreamEnvironment = '*'; + if (environment) { + upstreamEnvironment = + environment === ENVIRONMENT_NOT_DEFINED ? 'null' : environment; + } + + params.body.query.bool.filter.push({ + wildcard: { + ['connection.upstream.list']: `${upstreamServiceName}/${upstreamEnvironment}` + } + }); + } + + // @ts-ignore + const { aggregations } = await client.search(params); + const buckets: Array<{ + key: { + 'service.name': string; + 'service.environment': string | null; + 'destination.address': string; + 'connection.type': string; + 'connection.subtype': string | null; + }; + dests: { + buckets: Array<{ + key: string; + envs: { + buckets: Array<{ key: string }>; + }; + }>; + }; + }> = idx(aggregations, _ => _.conns.buckets) || []; + + return buckets.reduce( + (acc, { key: connection, dests }) => { + const connectionServiceName = connection['service.name']; + let destinationNames = dests.buckets.map( + ({ key: destinationName }) => destinationName + ); + if (destinationNames.length === 0) { + destinationNames = [connection['destination.address']]; + } + + const serviceMapConnections = destinationNames.flatMap( + destinationName => [ + { + data: { + id: destinationName + } + }, + { + data: { + id: `${connectionServiceName}~${destinationName}`, + source: connectionServiceName, + target: destinationName + } + } + ] + ); + + return [...acc, ...serviceMapConnections]; + }, + buckets.length + ? [ + { + data: { + id: buckets[0].key['service.name'] + } + } + ] + : [] + ); +} diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/initialize_service_maps.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/initialize_service_maps.ts new file mode 100644 index 0000000000000..10fe5ef28d1da --- /dev/null +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/initialize_service_maps.ts @@ -0,0 +1,109 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Server } from 'hapi'; +// @ts-ignore +import { TaskManager, RunContext } from '../legacy/plugins/task_manager'; +import { runServiceMapTask } from './run_service_map_task'; +import { setupRequiredScripts } from './setup_required_scripts'; + +export async function initializeServiceMaps(server: Server) { + const config = server.config(); + + setupRequiredScripts(server).catch(error => { + server.log( + 'error', + 'Unable to set up required scripts for APM Service Maps:' + ); + server.log('error', error); + }); + + const taskManager = server.plugins.task_manager; + if (taskManager) { + // @ts-ignore + const kbnServer = this.kbnServer; + taskManager.registerTaskDefinitions({ + // serviceMap is the task type, and must be unique across the entire system + serviceMap: { + // Human friendly name, used to represent this task in logs, UI, etc + title: 'ServiceMapTask', + type: 'serviceMap', + + // Optional, human-friendly, more detailed description + description: 'Extract connections in traces for service maps', + + // Optional, how long, in minutes, the system should wait before + // a running instance of this task is considered to be timed out. + // This defaults to 5 minutes. + timeout: '5m', + + // The serviceMap task occupies 2 workers, so if the system has 10 worker slots, + // 5 serviceMap tasks could run concurrently per Kibana instance. This value is + // overridden by the `override_num_workers` config value, if specified. + // numWorkers: 1, + + // The createTaskRunner function / method returns an object that is responsible for + // performing the work of the task. context: { taskInstance, kbnServer }, is documented below. + createTaskRunner({ taskInstance }: RunContext) { + // Perform the work of the task. The return value should fit the TaskResult interface, documented + // below. Invalid return values will result in a logged warning. + return { + async run() { + const { state } = taskInstance; + + const { mostRecent } = await runServiceMapTask( + kbnServer, + config, + state.lastRun + ); + // console.log(`Task run count: ${(state.count || 0) + 1}`); + + return { + state: { + count: (state.count || 0) + 1, + lastRun: mostRecent + } + }; + } + }; + } + } + }); + + // @ts-ignore + this.kbnServer.afterPluginsInit(async () => { + const fetchedTasks = await taskManager.fetch({ + query: { + bool: { + must: [ + { + term: { + _id: 'task:servicemap-processor' + } + }, + { + term: { + 'task.taskType': 'serviceMap' + } + } + ] + } + } + }); + if (fetchedTasks.docs.length) { + await taskManager.remove('servicemap-processor'); + } + await taskManager.schedule({ + id: 'servicemap-processor', + taskType: 'serviceMap', + interval: '1m', + scope: ['apm'], + params: {}, + state: {} + }); + }); + } +} diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/run_service_map_task.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/run_service_map_task.ts new file mode 100644 index 0000000000000..9147cba1ad102 --- /dev/null +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/run_service_map_task.ts @@ -0,0 +1,248 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +function interestingTransactions(since?: string, afterKey?: any) { + if (!since) { + since = 'now-1h'; + } + const query = { + size: 0, + query: { + bool: { + filter: [ + { exists: { field: 'destination.address' } }, + { exists: { field: 'trace.id' } }, + { exists: { field: 'span.duration.us' } }, + { range: { '@timestamp': { gt: since } } } + ] + } + }, + aggs: { + 'ext-conns': { + composite: { + sources: [ + { 'service.name': { terms: { field: 'service.name' } } }, + { 'span.type': { terms: { field: 'span.type' } } }, + { + 'span.subtype': { + terms: { field: 'span.subtype', missing_bucket: true } + } + }, + { + 'service.environment': { + terms: { field: 'service.environment', missing_bucket: true } + } + }, + { + 'destination.address': { terms: { field: 'destination.address' } } + } + ], + // TODO: needs to be balanced with the 20 below + size: 200, + after: undefined + }, + aggs: { + smpl: { + diversified_sampler: { + shard_size: 20, + script: { + lang: 'painless', + source: "(int)doc['span.duration.us'].value/100000" + } + }, + aggs: { + tracesample: { + top_hits: { + size: 20, + _source: ['trace.id', '@timestamp'] + } + } + } + } + } + } + } + }; + + if (afterKey) { + query.aggs['ext-conns'].composite.after = afterKey; + } + return query; +} + +function findConns(traceIds: string[]) { + const query = { + size: 0, + query: { + bool: { + should: [ + { exists: { field: 'span.id' } }, + { exists: { field: 'transaction.type' } } + ] as any[], + minimum_should_match: 2 + } + }, + aggs: { + trace_id: { + terms: { + field: 'trace.id', + order: { _key: 'asc' }, + size: traceIds.length + }, + aggs: { + connections: { + scripted_metric: { + init_script: 'state.spans = new HashMap();', + map_script: { id: 'map-service-conns' }, + combine_script: { id: 'combine-service-conns' }, + reduce_script: { id: 'reduce-service-conns' } + } + } + } + } + } + }; + + for (const tid of traceIds) { + query.query.bool.should.push({ term: { 'trace.id': tid } }); + } + return query; +} + +interface Service { + name: string; + environment?: string; +} + +interface ConnectionDoc { + '@timestamp': string; + observer: { version_major: number }; // TODO: make this dynamic + service: Service; + callee?: Service; + connection: Connection; + destination?: { address: string }; +} + +interface Connection { + upstream: { list: string[]; keyword: string }; + in_trace: string[]; + type: string; + subtype?: string; +} + +export async function runServiceMapTask( + kbnServer: any, + config: any, + lastRun?: string +) { + const apmIdxPattern = config.get('apm_oss.indexPattern'); + const serviceConnsDestinationIndex = config.get( + 'xpack.apm.serviceMapDestinationIndex' + ); + const serviceConnsDestinationPipeline = config.get( + 'xpack.apm.serviceMapDestinationPipeline' + ); + + const callCluster = kbnServer.server.plugins.elasticsearch.getCluster('data') + .callWithInternalUser; + + let mostRecent = ''; + let afterKey = null; + while (true) { + const q = interestingTransactions(lastRun, afterKey); + const txs = await callCluster('search', { + index: apmIdxPattern, + body: q + }); + + if (txs.aggregations['ext-conns'].buckets.length < 1) { + return { mostRecent }; + } + afterKey = txs.aggregations['ext-conns'].after_key; + + const traces = new Set(); + + txs.aggregations['ext-conns'].buckets.forEach((bucket: any) => { + bucket.smpl.tracesample.hits.hits.forEach((hit: any) => { + traces.add(hit._source.trace.id); + mostRecent = + mostRecent > hit._source['@timestamp'] + ? mostRecent + : hit._source['@timestamp']; + }); + }); + + const traceIds = Array.from(traces.values()); + if (traceIds.length < 1) { + return { mostRecent: null }; + } + + const findConnsQ = findConns(traceIds); + + const connsResult = await callCluster('search', { + index: apmIdxPattern, + body: findConnsQ + }); + + const connDocs: Array<{ index: { _index: any } } | ConnectionDoc> = []; + + connsResult.aggregations.trace_id.buckets.forEach((bucket: any) => { + const allServices = new Set( + bucket.connections.value.map( + (conn: any) => + `${conn.caller.service_name}/${conn.caller.environment || 'null'}` + ) + ); + + bucket.connections.value.forEach((conn: any) => { + const index = serviceConnsDestinationIndex + ? serviceConnsDestinationIndex + : conn.caller._index; + const bulkOpts = { index: { _index: index }, pipeline: undefined }; + + if (serviceConnsDestinationPipeline) { + bulkOpts.pipeline = serviceConnsDestinationPipeline; + } + connDocs.push(bulkOpts); + const doc: ConnectionDoc = { + '@timestamp': conn.caller.timestamp, + observer: { version_major: 7 }, // TODO: make this dynamic + service: { + name: conn.caller.service_name, + environment: conn.caller.environment + }, + callee: conn.callee + ? { + name: conn.callee.service_name, + environment: conn.callee.environment + } + : undefined, + connection: { + upstream: { + list: conn.upstream, + keyword: conn.upstream.join('->') + }, + in_trace: Array.from(allServices), + type: conn.caller.span_type, + subtype: conn.caller.span_substype + }, + destination: conn.caller.destination + ? { address: conn.caller.destination } + : undefined + }; + + connDocs.push(doc); + }); + }); + + const body = connDocs + .map((connDoc: any) => JSON.stringify(connDoc)) + .join('\n'); + await callCluster('bulk', { + body + }); + } +} diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/setup_required_scripts.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/setup_required_scripts.ts new file mode 100644 index 0000000000000..9b539feb08d7b --- /dev/null +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/setup_required_scripts.ts @@ -0,0 +1,298 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Legacy } from 'kibana'; +import { CallCluster } from 'src/legacy/core_plugins/elasticsearch'; + +const MAP_SERVICE_CONNS_SCRIPT_ID = 'map-service-conns'; +const REDUCE_SERVICE_CONNS_SCRIPT_ID = 'reduce-service-conns'; +const COMBINE_SERVICE_CONNS_SCRIPT_ID = 'combine-service-conns'; +const EXTRACT_DESTINATION_INGEST_PIPELINE_ID = 'extract_destination'; +const APM_INGEST_PIPELINE_ID = 'apm'; + +async function putScriptMapServiceConns(callCluster: CallCluster) { + return await callCluster('putScript', { + id: MAP_SERVICE_CONNS_SCRIPT_ID, + body: { + script: { + lang: 'painless', + source: ` + def s = new HashMap(); + + if (!doc['span.id'].empty) { + s.id = doc['span.id'].value + } else { + s.id = doc['transaction.id'].value; + s.transaction = true; + } + if (!doc['parent.id'].empty) { + s.parent = doc['parent.id'].value; + } + if (!doc['service.environment'].empty) { + s.environment = doc['service.environment'].value; + } + + if (!doc['destination.address'].empty) { + s.destination = doc['destination.address'].value; + } + + if (!doc['_index'].empty) { + s._index = doc['_index'].value; + } + + if (!doc['span.type'].empty) { + s.span_type = doc['span.type'].value; + } + + if (!doc['span.subtype'].empty) { + s.span_subtype = doc['span.subtype'].value; + } + + s.timestamp = doc['@timestamp'].value; + s.service_name = doc['service.name'].value; + if(!state.spans.containsKey(s.parent)) { + state.spans.put(s.parent, new ArrayList()) + } + + if (s.parent != s.id) { + state.spans[s.parent].add(s) + } + ` + } + } + }); +} + +async function putScriptReduceServiceConns(callCluster: CallCluster) { + return await callCluster('putScript', { + id: REDUCE_SERVICE_CONNS_SCRIPT_ID, + body: { + script: { + lang: 'painless', + source: ` + void extractChildren(def caller, def spans, def upstream, def conns, def count) { + // TODO: simplify this + if (spans.containsKey(caller.id)) { + for(s in spans[caller.id]) { + if (caller.span_type=='external') { + upstream.add(caller.service_name+"/"+caller.environment); + + def conn = new HashMap(); + conn.caller = caller; + conn.callee = s; + conn.upstream = new ArrayList(upstream); + conns.add(conn); + + extractChildren(s, spans, upstream, conns, count); + upstream.remove(upstream.size()-1); + } else { + extractChildren(s, spans, upstream, conns, count); + } + } + } else { + // no connection found + def conn = new HashMap(); + conn.caller = caller; + conn.upstream = new ArrayList(upstream); + conn.upstream.add(caller.service_name+"/"+caller.environment); + conns.add(conn); + } + } + def conns = new HashSet(); + def spans = new HashMap(); + + // merge results from shards + for(state in states) { + for(s in state.entrySet()) { + def v = s.getValue(); + def k = s.getKey(); + if(!spans.containsKey(k)) { + spans[k] = v; + } else { + for (p in v) { + spans[k].add(p); + } + } + } + } + + if (spans.containsKey(null) && spans[null].size() > 0) { + def node = spans[null][0]; + def upstream = new ArrayList(); + + extractChildren(node, spans, upstream, conns, 0); + + return new ArrayList(conns) + } + return []; + ` + } + } + }); +} + +async function putScriptCombineServiceConns(callCluster: CallCluster) { + return await callCluster('putScript', { + id: COMBINE_SERVICE_CONNS_SCRIPT_ID, + body: { + script: { + lang: 'painless', + source: `return state.spans` + } + } + }); +} + +async function putIngestPipelineExtractDestination(callCluster: CallCluster) { + return await callCluster('ingest.putPipeline', { + id: EXTRACT_DESTINATION_INGEST_PIPELINE_ID, + body: { + description: 'sets destination on ext spans based on their name', + processors: [ + { + set: { + if: "ctx.span != null && ctx.span.type == 'ext'", + field: 'span.type', + value: 'external' + } + }, + { + script: ` + if(ctx['span'] != null) { + if (ctx['span']['type'] == 'external') { + def spanName = ctx['span']['name']; + if (spanName.indexOf('/') > -1) { + spanName = spanName.substring(0, spanName.indexOf('/')); + } + + if (spanName.indexOf(' ') > -1) { + spanName = spanName.substring(spanName.indexOf(' ')+1, spanName.length()); + } + ctx['destination.address']=spanName; + } + + if (ctx['span']['type'] == 'resource') { + def spanName = ctx['span']['name']; + + if (spanName.indexOf('://') > -1) { + spanName = spanName.substring(spanName.indexOf('://')+3, spanName.length()); + } + if (spanName.indexOf('/') > -1) { + spanName = spanName.substring(0, spanName.indexOf('/')); + } + + ctx['destination.address']=spanName; + } + + if (ctx['span']['type'] == 'db') { + def dest = ctx['span']['subtype']; + ctx['destination.address']=dest; + } + + if (ctx['span']['type'] == 'cache') { + def dest = ctx['span']['subtype']; + ctx['destination.address']=dest; + } + } + ` + } + ] + } + }); +} + +interface ApmIngestPipeline { + [APM_INGEST_PIPELINE_ID]: { + description: string; + processors: Array<{ + pipeline: { + name: string; + }; + }>; + }; +} + +async function getIngestPipelineApm( + callCluster: CallCluster +): Promise { + return await callCluster('ingest.getPipeline', { + id: APM_INGEST_PIPELINE_ID + }); +} + +async function putIngestPipelineApm( + callCluster: CallCluster, + processors: ApmIngestPipeline[typeof APM_INGEST_PIPELINE_ID]['processors'] +) { + return await callCluster('ingest.putPipeline', { + id: APM_INGEST_PIPELINE_ID, + body: { + description: 'Default enrichment for APM events', + processors + } + }); +} + +async function applyExtractDestinationToApm(callCluster: CallCluster) { + let apmIngestPipeline: ApmIngestPipeline; + try { + // get current apm ingest pipeline + apmIngestPipeline = await getIngestPipelineApm(callCluster); + } catch (error) { + if (error.statusCode !== 404) { + throw error; + } + // create apm ingest pipeline if it doesn't exist + return await putIngestPipelineApm(callCluster, [ + { + pipeline: { + name: EXTRACT_DESTINATION_INGEST_PIPELINE_ID + } + } + ]); + } + + const { + apm: { processors } + } = apmIngestPipeline; + + // check if 'extract destination' processor is already applied + if ( + processors.find( + ({ pipeline: { name } }) => + name === EXTRACT_DESTINATION_INGEST_PIPELINE_ID + ) + ) { + return apmIngestPipeline; + } + + // append 'extract destination' to existing processors + return await putIngestPipelineApm(callCluster, [ + ...processors, + { + pipeline: { + name: EXTRACT_DESTINATION_INGEST_PIPELINE_ID + } + } + ]); +} + +export async function setupRequiredScripts(server: Legacy.Server) { + const callCluster = server.plugins.elasticsearch.getCluster('data') + .callWithInternalUser; + + const putRequiredScriptsResults = await Promise.all([ + putScriptMapServiceConns(callCluster), + putScriptReduceServiceConns(callCluster), + putScriptCombineServiceConns(callCluster), + putIngestPipelineExtractDestination(callCluster) // TODO remove this when agents set destination.address (elastic/apm#115) + ]); + + return [ + ...putRequiredScriptsResults, + await applyExtractDestinationToApm(callCluster) // TODO remove this when agents set destination.address (elastic/apm#115) + ]; +} diff --git a/x-pack/legacy/plugins/apm/server/lib/services/map.ts b/x-pack/legacy/plugins/apm/server/lib/services/map.ts deleted file mode 100644 index 97bb925674e26..0000000000000 --- a/x-pack/legacy/plugins/apm/server/lib/services/map.ts +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -import cytoscape from 'cytoscape'; -import { PromiseReturnType } from '../../../typings/common'; - -// This response right now just returns experimental data. -export type ServiceMapResponse = PromiseReturnType; -export async function getServiceMap(): Promise { - return [ - { data: { id: 'client', agentName: 'js-base' } }, - { data: { id: 'opbeans-node', agentName: 'nodejs' } }, - { data: { id: 'opbeans-python', agentName: 'python' } }, - { data: { id: 'opbeans-java', agentName: 'java' } }, - { data: { id: 'opbeans-ruby', agentName: 'ruby' } }, - { data: { id: 'opbeans-go', agentName: 'go' } }, - { data: { id: 'opbeans-go-2', agentName: 'go' } }, - { data: { id: 'opbeans-dotnet', agentName: 'dotnet' } }, - { data: { id: 'database', agentName: 'database' } }, - { data: { id: 'external API', agentName: 'external' } }, - - { - data: { - id: 'opbeans-client~opbeans-node', - source: 'client', - target: 'opbeans-node' - } - }, - { - data: { - id: 'opbeans-client~opbeans-python', - source: 'client', - target: 'opbeans-python' - } - }, - { - data: { - id: 'opbeans-python~opbeans-go', - source: 'opbeans-python', - target: 'opbeans-go' - } - }, - { - data: { - id: 'opbeans-python~opbeans-go-2', - source: 'opbeans-python', - target: 'opbeans-go-2' - } - }, - { - data: { - id: 'opbeans-python~opbeans-dotnet', - source: 'opbeans-python', - target: 'opbeans-dotnet' - } - }, - { - data: { - id: 'opbeans-node~opbeans-java', - source: 'opbeans-node', - target: 'opbeans-java' - } - }, - { - data: { - id: 'opbeans-node~database', - source: 'opbeans-node', - target: 'database' - } - }, - { - data: { - id: 'opbeans-go-2~opbeans-ruby', - source: 'opbeans-go-2', - target: 'opbeans-ruby' - } - }, - { - data: { - id: 'opbeans-go-2~external API', - source: 'opbeans-go-2', - target: 'external API' - } - } - ]; -} diff --git a/x-pack/legacy/plugins/apm/server/routes/create_apm_api.ts b/x-pack/legacy/plugins/apm/server/routes/create_apm_api.ts index e98842151da84..def8a87820833 100644 --- a/x-pack/legacy/plugins/apm/server/routes/create_apm_api.ts +++ b/x-pack/legacy/plugins/apm/server/routes/create_apm_api.ts @@ -58,7 +58,7 @@ import { uiFiltersEnvironmentsRoute } from './ui_filters'; import { createApi } from './create_api'; -import { serviceMapRoute } from './services'; +import { serviceMapAllRoute, serviceMapRoute } from './services'; const createApmApi = () => { const api = createApi() @@ -76,6 +76,7 @@ const createApmApi = () => { .add(serviceTransactionTypesRoute) .add(servicesRoute) .add(serviceNodeMetadataRoute) + .add(serviceMapRoute) .add(serviceAnnotationsRoute) // Agent configuration @@ -118,7 +119,6 @@ const createApmApi = () => { .add(transactionsLocalFiltersRoute) .add(serviceNodesLocalFiltersRoute) .add(uiFiltersEnvironmentsRoute) - .add(serviceMapRoute) // Transaction .add(transactionByTraceIdRoute); diff --git a/x-pack/legacy/plugins/apm/server/routes/services.ts b/x-pack/legacy/plugins/apm/server/routes/services.ts index 78cb092b85db6..95aa1337920fc 100644 --- a/x-pack/legacy/plugins/apm/server/routes/services.ts +++ b/x-pack/legacy/plugins/apm/server/routes/services.ts @@ -18,7 +18,7 @@ import { getServiceTransactionTypes } from '../lib/services/get_service_transact import { getServiceNodeMetadata } from '../lib/services/get_service_node_metadata'; import { createRoute } from './create_route'; import { uiFiltersRt, rangeRt } from './default_api_types'; -import { getServiceMap } from '../lib/services/map'; +import { getServiceMap } from '../lib/service_map/get_service_map'; import { getServiceAnnotations } from '../lib/services/annotations'; export const servicesRoute = createRoute(() => ({ @@ -87,16 +87,49 @@ export const serviceNodeMetadataRoute = createRoute(() => ({ } })); +export const serviceMapAllRoute = createRoute(() => ({ + path: '/api/apm/service-map/all', + params: { + query: t.intersection([ + t.partial({ environment: t.string }), + uiFiltersRt, + rangeRt + ]) + }, + handler: async ({ context, request }) => { + if (!context.config['xpack.apm.serviceMapEnabled']) { + return new Boom('Not found', { statusCode: 404 }); + } + const setup = await setupRequest(context, request); + const { + query: { environment } + } = context.params; + return getServiceMap({ setup, environment }); + } +})); + export const serviceMapRoute = createRoute(() => ({ - path: '/api/apm/service-map', + path: '/api/apm/service-map/{serviceName}', params: { - query: rangeRt + path: t.type({ + serviceName: t.string + }), + query: t.intersection([ + t.partial({ environment: t.string }), + uiFiltersRt, + rangeRt + ]) }, - handler: async ({ context }) => { - if (context.config['xpack.apm.serviceMapEnabled']) { - return getServiceMap(); + handler: async ({ context, request }) => { + if (!context.config['xpack.apm.serviceMapEnabled']) { + return new Boom('Not found', { statusCode: 404 }); } - return new Boom('Not found', { statusCode: 404 }); + const setup = await setupRequest(context, request); + const { + path: { serviceName }, + query: { environment } + } = context.params; + return getServiceMap({ setup, serviceName, environment }); } })); diff --git a/x-pack/plugins/apm/server/index.ts b/x-pack/plugins/apm/server/index.ts index b0e10d245e0b9..9c111626dd5a3 100644 --- a/x-pack/plugins/apm/server/index.ts +++ b/x-pack/plugins/apm/server/index.ts @@ -16,6 +16,9 @@ export const config = { }, schema: schema.object({ serviceMapEnabled: schema.boolean({ defaultValue: false }), + serviceMapIndexPattern: schema.string({ defaultValue: 'apm-*' }), + serviceMapDestinationIndex: schema.maybe(schema.string()), + serviceMapDestinationPipeline: schema.maybe(schema.string()), autocreateApmIndexPattern: schema.boolean({ defaultValue: true }), ui: schema.object({ enabled: schema.boolean({ defaultValue: true }), @@ -41,6 +44,9 @@ export function mergeConfigs(apmOssConfig: APMOSSConfig, apmConfig: APMXPackConf 'xpack.apm.ui.maxTraceItems': apmConfig.ui.maxTraceItems, 'xpack.apm.ui.transactionGroupBucketSize': apmConfig.ui.transactionGroupBucketSize, 'xpack.apm.autocreateApmIndexPattern': apmConfig.autocreateApmIndexPattern, + 'xpack.apm.serviceMapIndexPattern': apmConfig.serviceMapIndexPattern, + 'xpack.apm.serviceMapDestinationIndex': apmConfig.serviceMapDestinationIndex, + 'xpack.apm.serviceMapDestinationPipeline': apmConfig.serviceMapDestinationPipeline, }; } From 1deb2343d207f6f6c1556035757eec104e9aca1a Mon Sep 17 00:00:00 2001 From: Oliver Gupte Date: Wed, 4 Dec 2019 01:19:38 -0800 Subject: [PATCH 02/15] - fix UI bug where service map data did not update correctly - replace this.kbnServer.afterPluginsInit and task object fetch with taskManager.ensureScheduled - simplify routes: move serviceName to query param and consolidate endpoints - clean up code in get_service_map.ts, constants, typescript fixes - add typescript support for composite aggregations --- .../elasticsearch_fieldnames.test.ts.snap | 36 ++++++ .../apm/common/elasticsearch_fieldnames.ts | 7 ++ .../apm/common/service_map_constants.ts | 8 ++ x-pack/legacy/plugins/apm/index.ts | 4 +- .../components/app/ServiceMap/Cytoscape.tsx | 1 + .../components/app/ServiceMap/index.tsx | 19 +-- .../server/lib/service_map/get_service_map.ts | 117 ++++++++++-------- .../service_map/initialize_service_maps.ts | 68 ++-------- .../lib/service_map/run_service_map_task.ts | 6 +- .../apm/server/routes/create_apm_api.ts | 2 +- .../plugins/apm/server/routes/services.ts | 31 +---- .../apm/typings/elasticsearch/aggregations.ts | 27 ++++ 12 files changed, 166 insertions(+), 160 deletions(-) create mode 100644 x-pack/legacy/plugins/apm/common/service_map_constants.ts diff --git a/x-pack/legacy/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap b/x-pack/legacy/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap index e345ca3552e5a..0c6429f55fc9a 100644 --- a/x-pack/legacy/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap +++ b/x-pack/legacy/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap @@ -1,9 +1,21 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[`Error CALLEE_ENVIRONMENT 1`] = `undefined`; + +exports[`Error CALLEE_NAME 1`] = `undefined`; + exports[`Error CLIENT_GEO_COUNTRY_ISO_CODE 1`] = `undefined`; +exports[`Error CONNECTION_SUBTYPE 1`] = `undefined`; + +exports[`Error CONNECTION_TYPE 1`] = `undefined`; + +exports[`Error CONNECTION_UPSTREAM_LIST 1`] = `undefined`; + exports[`Error CONTAINER_ID 1`] = `undefined`; +exports[`Error DESTINATION_ADDRESS 1`] = `undefined`; + exports[`Error ERROR_CULPRIT 1`] = `"handleOopsie"`; exports[`Error ERROR_EXC_HANDLED 1`] = `undefined`; @@ -108,10 +120,22 @@ exports[`Error USER_AGENT_NAME 1`] = `undefined`; exports[`Error USER_ID 1`] = `undefined`; +exports[`Span CALLEE_ENVIRONMENT 1`] = `undefined`; + +exports[`Span CALLEE_NAME 1`] = `undefined`; + exports[`Span CLIENT_GEO_COUNTRY_ISO_CODE 1`] = `undefined`; +exports[`Span CONNECTION_SUBTYPE 1`] = `undefined`; + +exports[`Span CONNECTION_TYPE 1`] = `undefined`; + +exports[`Span CONNECTION_UPSTREAM_LIST 1`] = `undefined`; + exports[`Span CONTAINER_ID 1`] = `undefined`; +exports[`Span DESTINATION_ADDRESS 1`] = `undefined`; + exports[`Span ERROR_CULPRIT 1`] = `undefined`; exports[`Span ERROR_EXC_HANDLED 1`] = `undefined`; @@ -216,10 +240,22 @@ exports[`Span USER_AGENT_NAME 1`] = `undefined`; exports[`Span USER_ID 1`] = `undefined`; +exports[`Transaction CALLEE_ENVIRONMENT 1`] = `undefined`; + +exports[`Transaction CALLEE_NAME 1`] = `undefined`; + exports[`Transaction CLIENT_GEO_COUNTRY_ISO_CODE 1`] = `undefined`; +exports[`Transaction CONNECTION_SUBTYPE 1`] = `undefined`; + +exports[`Transaction CONNECTION_TYPE 1`] = `undefined`; + +exports[`Transaction CONNECTION_UPSTREAM_LIST 1`] = `undefined`; + exports[`Transaction CONTAINER_ID 1`] = `"container1234567890abcdef"`; +exports[`Transaction DESTINATION_ADDRESS 1`] = `undefined`; + exports[`Transaction ERROR_CULPRIT 1`] = `undefined`; exports[`Transaction ERROR_EXC_HANDLED 1`] = `undefined`; diff --git a/x-pack/legacy/plugins/apm/common/elasticsearch_fieldnames.ts b/x-pack/legacy/plugins/apm/common/elasticsearch_fieldnames.ts index 0d7ff3114e73f..e5d2b00b1741c 100644 --- a/x-pack/legacy/plugins/apm/common/elasticsearch_fieldnames.ts +++ b/x-pack/legacy/plugins/apm/common/elasticsearch_fieldnames.ts @@ -37,6 +37,13 @@ export const SPAN_ACTION = 'span.action'; export const SPAN_NAME = 'span.name'; export const SPAN_ID = 'span.id'; +export const DESTINATION_ADDRESS = 'destination.address'; +export const CONNECTION_TYPE = 'connection.type'; +export const CONNECTION_SUBTYPE = 'connection.subtype'; +export const CALLEE_NAME = 'callee.name'; +export const CALLEE_ENVIRONMENT = 'callee.environment'; +export const CONNECTION_UPSTREAM_LIST = 'connection.upstream.list'; + // Parent ID for a transaction or span export const PARENT_ID = 'parent.id'; diff --git a/x-pack/legacy/plugins/apm/common/service_map_constants.ts b/x-pack/legacy/plugins/apm/common/service_map_constants.ts new file mode 100644 index 0000000000000..70ad80f08e4b6 --- /dev/null +++ b/x-pack/legacy/plugins/apm/common/service_map_constants.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export const SERVICE_MAP_TASK_TYPE = 'serviceMap'; +export const SERVICE_MAP_TASK_ID = 'servicemap-processor'; diff --git a/x-pack/legacy/plugins/apm/index.ts b/x-pack/legacy/plugins/apm/index.ts index 8fd08fd79f531..e2ac07901fce1 100644 --- a/x-pack/legacy/plugins/apm/index.ts +++ b/x-pack/legacy/plugins/apm/index.ts @@ -128,9 +128,7 @@ export const apm: LegacyPluginInitializer = kibana => { apmPlugin.registerLegacyAPI({ server }); if (config.get('xpack.apm.serviceMapEnabled')) { - initializeServiceMaps(server).catch(error => { - throw error; - }); + initializeServiceMaps(server); } } }); diff --git a/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/Cytoscape.tsx b/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/Cytoscape.tsx index 238158c5bf224..669e45cae9cea 100644 --- a/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/Cytoscape.tsx +++ b/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/Cytoscape.tsx @@ -62,6 +62,7 @@ export function Cytoscape({ // Trigger a custom "data" event when data changes useEffect(() => { if (cy) { + cy.remove('*'); cy.add(elements); cy.trigger('data'); } diff --git a/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/index.tsx b/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/index.tsx index 1b7680907fe3e..cb4f93527d5c5 100644 --- a/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/index.tsx +++ b/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/index.tsx @@ -53,30 +53,15 @@ export function ServiceMap({ serviceName }: ServiceMapProps) { const { data } = useFetcher( callApmApi => { - if (serviceName && start && end) { - return callApmApi({ - pathname: '/api/apm/service-map/{serviceName}', - params: { - path: { - serviceName - }, - query: { - start, - end, - environment, - uiFilters: JSON.stringify(uiFiltersOmitEnv) - } - } - }); - } if (start && end) { return callApmApi({ - pathname: '/api/apm/service-map/all', + pathname: '/api/apm/service-map', params: { query: { start, end, environment, + serviceName, uiFilters: JSON.stringify(uiFiltersOmitEnv) } } diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_map.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_map.ts index 911e6b9d6c178..8fa89b4b9aa33 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_map.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_map.ts @@ -3,7 +3,6 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { idx } from '@kbn/elastic-idx'; import { PromiseReturnType } from '../../../typings/common'; import { rangeFilter } from '../helpers/range_filter'; import { @@ -12,6 +11,16 @@ import { SetupUIFilters } from '../helpers/setup_request'; import { ENVIRONMENT_NOT_DEFINED } from '../../../common/environment_filter_values'; +import { + SERVICE_NAME, + SERVICE_ENVIRONMENT, + DESTINATION_ADDRESS, + CONNECTION_TYPE, + CONNECTION_SUBTYPE, + CALLEE_NAME, + CALLEE_ENVIRONMENT, + CONNECTION_UPSTREAM_LIST +} from '../../../common/elasticsearch_fieldnames'; export interface IEnvOptions { setup: Setup & SetupTimeRange & SetupUIFilters; @@ -19,6 +28,18 @@ export interface IEnvOptions { environment?: string; } +interface ServiceMapElement { + data: + | { + id: string; + } + | { + id: string; + source: string; + target: string; + }; +} + export type ServiceMapAPIResponse = PromiseReturnType; export async function getServiceMap({ setup, @@ -35,7 +56,7 @@ export async function getServiceMap({ bool: { filter: [ { range: rangeFilter(start, end) }, - { exists: { field: 'connection.type' } }, + { exists: { field: CONNECTION_TYPE } }, ...uiFiltersES ] } @@ -44,23 +65,23 @@ export async function getServiceMap({ conns: { composite: { sources: [ - { 'service.name': { terms: { field: 'service.name' } } }, + { [SERVICE_NAME]: { terms: { field: SERVICE_NAME } } }, { - 'service.environment': { - terms: { field: 'service.environment', missing_bucket: true } + [SERVICE_ENVIRONMENT]: { + terms: { field: SERVICE_ENVIRONMENT, missing_bucket: true } } }, { - 'destination.address': { + [DESTINATION_ADDRESS]: { terms: { - field: 'destination.address' + field: DESTINATION_ADDRESS } } }, - { 'connection.type': { terms: { field: 'connection.type' } } }, // will filter out regular spans etc. + { [CONNECTION_TYPE]: { terms: { field: CONNECTION_TYPE } } }, // will filter out regular spans etc. { - 'connection.subtype': { - terms: { field: 'connection.subtype', missing_bucket: true } + [CONNECTION_SUBTYPE]: { + terms: { field: CONNECTION_SUBTYPE, missing_bucket: true } } } ], @@ -69,12 +90,12 @@ export async function getServiceMap({ aggs: { dests: { terms: { - field: 'callee.name' + field: CALLEE_NAME }, aggs: { envs: { terms: { - field: 'callee.environment', + field: CALLEE_ENVIRONMENT, missing: '' } } @@ -88,49 +109,42 @@ export async function getServiceMap({ if (serviceName || environment) { const upstreamServiceName = serviceName || '*'; - - let upstreamEnvironment = '*'; - if (environment) { - upstreamEnvironment = - environment === ENVIRONMENT_NOT_DEFINED ? 'null' : environment; - } + const upstreamEnvironment = environment + ? environment === ENVIRONMENT_NOT_DEFINED + ? 'null' + : environment + : '*'; params.body.query.bool.filter.push({ wildcard: { - ['connection.upstream.list']: `${upstreamServiceName}/${upstreamEnvironment}` + [CONNECTION_UPSTREAM_LIST]: `${upstreamServiceName}/${upstreamEnvironment}` } }); } - // @ts-ignore const { aggregations } = await client.search(params); - const buckets: Array<{ - key: { - 'service.name': string; - 'service.environment': string | null; - 'destination.address': string; - 'connection.type': string; - 'connection.subtype': string | null; - }; - dests: { - buckets: Array<{ - key: string; - envs: { - buckets: Array<{ key: string }>; - }; - }>; - }; - }> = idx(aggregations, _ => _.conns.buckets) || []; + const buckets = aggregations?.conns.buckets ?? []; + + if (buckets.length === 0) { + return []; + } + + const initialServiceMapNode: ServiceMapElement = { + data: { + id: buckets[0].key[SERVICE_NAME] + } + }; return buckets.reduce( - (acc, { key: connection, dests }) => { - const connectionServiceName = connection['service.name']; - let destinationNames = dests.buckets.map( - ({ key: destinationName }) => destinationName - ); - if (destinationNames.length === 0) { - destinationNames = [connection['destination.address']]; - } + (acc: ServiceMapElement[], { key: connection, dests }) => { + const connectionServiceName = connection[SERVICE_NAME]; + + const destinationNames = + dests.buckets.length === 0 + ? [connection[DESTINATION_ADDRESS]] + : dests.buckets.map( + ({ key: destinationName }) => destinationName as string + ); const serviceMapConnections = destinationNames.flatMap( destinationName => [ @@ -149,16 +163,11 @@ export async function getServiceMap({ ] ); + if (acc.length === 0) { + return [initialServiceMapNode, ...serviceMapConnections]; + } return [...acc, ...serviceMapConnections]; }, - buckets.length - ? [ - { - data: { - id: buckets[0].key['service.name'] - } - } - ] - : [] + [] ); } diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/initialize_service_maps.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/initialize_service_maps.ts index 10fe5ef28d1da..fd62bca5e5a9c 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/initialize_service_maps.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/initialize_service_maps.ts @@ -9,6 +9,10 @@ import { Server } from 'hapi'; import { TaskManager, RunContext } from '../legacy/plugins/task_manager'; import { runServiceMapTask } from './run_service_map_task'; import { setupRequiredScripts } from './setup_required_scripts'; +import { + SERVICE_MAP_TASK_TYPE, + SERVICE_MAP_TASK_ID +} from '../../../common/service_map_constants'; export async function initializeServiceMaps(server: Server) { const config = server.config(); @@ -23,47 +27,25 @@ export async function initializeServiceMaps(server: Server) { const taskManager = server.plugins.task_manager; if (taskManager) { - // @ts-ignore - const kbnServer = this.kbnServer; taskManager.registerTaskDefinitions({ - // serviceMap is the task type, and must be unique across the entire system serviceMap: { - // Human friendly name, used to represent this task in logs, UI, etc title: 'ServiceMapTask', - type: 'serviceMap', - - // Optional, human-friendly, more detailed description + type: SERVICE_MAP_TASK_TYPE, description: 'Extract connections in traces for service maps', - - // Optional, how long, in minutes, the system should wait before - // a running instance of this task is considered to be timed out. - // This defaults to 5 minutes. timeout: '5m', - - // The serviceMap task occupies 2 workers, so if the system has 10 worker slots, - // 5 serviceMap tasks could run concurrently per Kibana instance. This value is - // overridden by the `override_num_workers` config value, if specified. - // numWorkers: 1, - - // The createTaskRunner function / method returns an object that is responsible for - // performing the work of the task. context: { taskInstance, kbnServer }, is documented below. createTaskRunner({ taskInstance }: RunContext) { - // Perform the work of the task. The return value should fit the TaskResult interface, documented - // below. Invalid return values will result in a logged warning. return { async run() { const { state } = taskInstance; const { mostRecent } = await runServiceMapTask( - kbnServer, + server, config, state.lastRun ); - // console.log(`Task run count: ${(state.count || 0) + 1}`); return { state: { - count: (state.count || 0) + 1, lastRun: mostRecent } }; @@ -73,37 +55,13 @@ export async function initializeServiceMaps(server: Server) { } }); - // @ts-ignore - this.kbnServer.afterPluginsInit(async () => { - const fetchedTasks = await taskManager.fetch({ - query: { - bool: { - must: [ - { - term: { - _id: 'task:servicemap-processor' - } - }, - { - term: { - 'task.taskType': 'serviceMap' - } - } - ] - } - } - }); - if (fetchedTasks.docs.length) { - await taskManager.remove('servicemap-processor'); - } - await taskManager.schedule({ - id: 'servicemap-processor', - taskType: 'serviceMap', - interval: '1m', - scope: ['apm'], - params: {}, - state: {} - }); + return await taskManager.ensureScheduled({ + id: SERVICE_MAP_TASK_ID, + taskType: SERVICE_MAP_TASK_TYPE, + interval: '10s', + scope: ['apm'], + params: {}, + state: {} }); } } diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/run_service_map_task.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/run_service_map_task.ts index 9147cba1ad102..97ed214008c6e 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/run_service_map_task.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/run_service_map_task.ts @@ -4,6 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ +import { Server } from 'hapi'; + function interestingTransactions(since?: string, afterKey?: any) { if (!since) { since = 'now-1h'; @@ -134,7 +136,7 @@ interface Connection { } export async function runServiceMapTask( - kbnServer: any, + server: Server, config: any, lastRun?: string ) { @@ -146,7 +148,7 @@ export async function runServiceMapTask( 'xpack.apm.serviceMapDestinationPipeline' ); - const callCluster = kbnServer.server.plugins.elasticsearch.getCluster('data') + const callCluster = server.plugins.elasticsearch.getCluster('data') .callWithInternalUser; let mostRecent = ''; diff --git a/x-pack/legacy/plugins/apm/server/routes/create_apm_api.ts b/x-pack/legacy/plugins/apm/server/routes/create_apm_api.ts index def8a87820833..a9da61c5c1f43 100644 --- a/x-pack/legacy/plugins/apm/server/routes/create_apm_api.ts +++ b/x-pack/legacy/plugins/apm/server/routes/create_apm_api.ts @@ -58,7 +58,7 @@ import { uiFiltersEnvironmentsRoute } from './ui_filters'; import { createApi } from './create_api'; -import { serviceMapAllRoute, serviceMapRoute } from './services'; +import { serviceMapRoute } from './services'; const createApmApi = () => { const api = createApi() diff --git a/x-pack/legacy/plugins/apm/server/routes/services.ts b/x-pack/legacy/plugins/apm/server/routes/services.ts index 95aa1337920fc..af3dd025b70b9 100644 --- a/x-pack/legacy/plugins/apm/server/routes/services.ts +++ b/x-pack/legacy/plugins/apm/server/routes/services.ts @@ -87,35 +87,11 @@ export const serviceNodeMetadataRoute = createRoute(() => ({ } })); -export const serviceMapAllRoute = createRoute(() => ({ - path: '/api/apm/service-map/all', - params: { - query: t.intersection([ - t.partial({ environment: t.string }), - uiFiltersRt, - rangeRt - ]) - }, - handler: async ({ context, request }) => { - if (!context.config['xpack.apm.serviceMapEnabled']) { - return new Boom('Not found', { statusCode: 404 }); - } - const setup = await setupRequest(context, request); - const { - query: { environment } - } = context.params; - return getServiceMap({ setup, environment }); - } -})); - export const serviceMapRoute = createRoute(() => ({ - path: '/api/apm/service-map/{serviceName}', + path: '/api/apm/service-map', params: { - path: t.type({ - serviceName: t.string - }), query: t.intersection([ - t.partial({ environment: t.string }), + t.partial({ environment: t.string, serviceName: t.string }), uiFiltersRt, rangeRt ]) @@ -126,8 +102,7 @@ export const serviceMapRoute = createRoute(() => ({ } const setup = await setupRequest(context, request); const { - path: { serviceName }, - query: { environment } + query: { serviceName, environment } } = context.params; return getServiceMap({ setup, serviceName, environment }); } diff --git a/x-pack/legacy/plugins/apm/typings/elasticsearch/aggregations.ts b/x-pack/legacy/plugins/apm/typings/elasticsearch/aggregations.ts index 74a9436d7a4bc..cabff77fd2a7b 100644 --- a/x-pack/legacy/plugins/apm/typings/elasticsearch/aggregations.ts +++ b/x-pack/legacy/plugins/apm/typings/elasticsearch/aggregations.ts @@ -97,6 +97,20 @@ export interface AggregationOptionsByType { buckets_path: BucketsPath; script?: Script; }; + + composite: { + sources: Array<{ + [name: string]: + | { + terms: { + field: string; + missing_bucket?: boolean; + }; + } + | undefined; + }>; + size?: number; + }; } type AggregationType = keyof AggregationOptionsByType; @@ -229,6 +243,19 @@ interface AggregationResponsePart< value: number | null; } | undefined; + composite: { + buckets: Array< + { + doc_count: number; + key: { + [name: string]: string; + }; + } & BucketSubAggregationResponse< + TAggregationOptionsMap['aggs'], + TDocument + > + >; + }; } // Type for debugging purposes. If you see an error in AggregationResponseMap From 658f048d0f871baed82db11a75ae8d2940aa2b0e Mon Sep 17 00:00:00 2001 From: Oliver Gupte Date: Fri, 6 Dec 2019 03:52:19 -0800 Subject: [PATCH 03/15] - rewrite the task run logic to be more readable and less procedural - broke up the task into more files for better organization and testing --- .../apm/server/lib/helpers/es_client.ts | 16 + .../get_next_transaction_samples.ts | 107 ++++++ .../service_map/get_service_connections.ts | 60 ++++ .../server/lib/service_map/get_service_map.ts | 7 +- .../service_map/initialize_service_maps.ts | 10 +- .../map_service_connection_to_bulk_index.ts | 56 +++ .../lib/service_map/run_service_map_task.ts | 334 ++++++------------ .../lib/service_map/setup_required_scripts.ts | 4 +- .../apm/typings/elasticsearch/aggregations.ts | 22 +- 9 files changed, 371 insertions(+), 245 deletions(-) create mode 100644 x-pack/legacy/plugins/apm/server/lib/service_map/get_next_transaction_samples.ts create mode 100644 x-pack/legacy/plugins/apm/server/lib/service_map/get_service_connections.ts create mode 100644 x-pack/legacy/plugins/apm/server/lib/service_map/map_service_connection_to_bulk_index.ts diff --git a/x-pack/legacy/plugins/apm/server/lib/helpers/es_client.ts b/x-pack/legacy/plugins/apm/server/lib/helpers/es_client.ts index aeeb39733b5db..406ca064ae1f9 100644 --- a/x-pack/legacy/plugins/apm/server/lib/helpers/es_client.ts +++ b/x-pack/legacy/plugins/apm/server/lib/helpers/es_client.ts @@ -14,6 +14,7 @@ import { import { merge } from 'lodash'; import { cloneDeep, isString } from 'lodash'; import { KibanaRequest } from 'src/core/server'; +import { CallCluster } from 'src/legacy/core_plugins/elasticsearch'; import { OBSERVER_VERSION_MAJOR } from '../../../common/elasticsearch_fieldnames'; import { ESSearchResponse, @@ -165,3 +166,18 @@ export function getESClient( } }; } + +export type SearchClient = ReturnType; +export function getSearchClient(callCluster: CallCluster) { + async function search< + TDocument = unknown, + TSearchRequest extends ESSearchRequest = {} + >( + params: TSearchRequest + ): Promise> { + return (callCluster('search', params) as unknown) as Promise< + ESSearchResponse + >; + } + return search; +} diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/get_next_transaction_samples.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/get_next_transaction_samples.ts new file mode 100644 index 0000000000000..2e13c2826efbc --- /dev/null +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/get_next_transaction_samples.ts @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { uniq } from 'lodash'; +import { SearchClient } from '../helpers/es_client'; +import { Transaction } from '../../../typings/es_schemas/ui/Transaction'; + +export async function getNextTransactionSamples({ + apmIdxPattern, + startTimeInterval, + afterKey, + searchClient +}: { + apmIdxPattern: string; + startTimeInterval: string; + afterKey?: object; + searchClient: SearchClient; +}) { + const params = { + index: apmIdxPattern, + body: { + size: 0, + query: { + bool: { + filter: [ + { exists: { field: 'destination.address' } }, + { exists: { field: 'trace.id' } }, + { exists: { field: 'span.duration.us' } }, + { range: { '@timestamp': { gt: startTimeInterval } } } + ] + } + }, + aggs: { + 'ext-conns': { + composite: { + sources: [ + { 'service.name': { terms: { field: 'service.name' } } }, + { 'span.type': { terms: { field: 'span.type' } } }, + { + 'span.subtype': { + terms: { field: 'span.subtype', missing_bucket: true } + } + }, + { + 'service.environment': { + terms: { field: 'service.environment', missing_bucket: true } + } + }, + { + 'destination.address': { + terms: { field: 'destination.address' } + } + } + ], + // TODO: needs to be balanced with the 20 below + size: 200, + after: afterKey + }, + aggs: { + smpl: { + diversified_sampler: { + composite: [] as never[], // TODO remove this + shard_size: 20, + script: { + lang: 'painless', + source: "(int)doc['span.duration.us'].value/100000" + } + }, + aggs: { + tracesample: { + top_hits: { + size: 20, + _source: ['trace.id', '@timestamp'] + } + } + } + } + } + } + } + } + }; + + const transactionsResponse = await searchClient(params); + const extConns = transactionsResponse.aggregations?.['ext-conns']; + const buckets = extConns?.buckets ?? []; + const sampleTraces = buckets.flatMap(bucket => { + return bucket.smpl.tracesample.hits.hits.map(hit => { + const transactionDoc = hit._source as Transaction; + const traceId = transactionDoc.trace.id; + const timestamp = parseInt(transactionDoc['@timestamp'], 10); + return { traceId, timestamp }; + }); + }); + const latestTransactionTime = Math.max( + ...sampleTraces.map(({ timestamp }) => timestamp) + ); + const traceIds = uniq(sampleTraces.map(({ traceId }) => traceId)); + return { + after_key: extConns?.after_key, + latestTransactionTime, + traceIds + }; +} diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_connections.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_connections.ts new file mode 100644 index 0000000000000..f3dfa2f7c54bb --- /dev/null +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_connections.ts @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchClient } from '../helpers/es_client'; + +export async function getServiceConnections({ + apmIdxPattern, + traceIds, + searchClient +}: { + apmIdxPattern: string; + traceIds: string[]; + searchClient: SearchClient; +}) { + const traceIdFilters = traceIds.map(traceId => ({ + term: { 'trace.id': traceId } + })); + const params = { + index: apmIdxPattern, + body: { + size: 0, + query: { + bool: { + should: [ + { exists: { field: 'span.id' } }, + { exists: { field: 'transaction.type' } }, + ...traceIdFilters + ], + minimum_should_match: 2 + } + }, + aggs: { + trace_id: { + terms: { + field: 'trace.id', + order: { _key: 'asc' as const }, + size: traceIds.length + }, + aggs: { + connections: { + scripted_metric: { + init_script: 'state.spans = new HashMap();', + map_script: { id: 'map-service-conns' }, + combine_script: { id: 'combine-service-conns' }, + reduce_script: { id: 'reduce-service-conns' } + } + } + } + } + } + } + }; + const serviceConnectionsResponse = await searchClient(params); + const traceConnectionsBuckets = + serviceConnectionsResponse.aggregations?.trace_id.buckets ?? []; + return traceConnectionsBuckets; +} diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_map.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_map.ts index 8fa89b4b9aa33..33a79710c5f55 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_map.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_map.ts @@ -10,7 +10,6 @@ import { SetupTimeRange, SetupUIFilters } from '../helpers/setup_request'; -import { ENVIRONMENT_NOT_DEFINED } from '../../../common/environment_filter_values'; import { SERVICE_NAME, SERVICE_ENVIRONMENT, @@ -109,11 +108,7 @@ export async function getServiceMap({ if (serviceName || environment) { const upstreamServiceName = serviceName || '*'; - const upstreamEnvironment = environment - ? environment === ENVIRONMENT_NOT_DEFINED - ? 'null' - : environment - : '*'; + const upstreamEnvironment = environment || '*'; params.body.query.bool.filter.push({ wildcard: { diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/initialize_service_maps.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/initialize_service_maps.ts index fd62bca5e5a9c..70e6b2d782406 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/initialize_service_maps.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/initialize_service_maps.ts @@ -38,17 +38,13 @@ export async function initializeServiceMaps(server: Server) { async run() { const { state } = taskInstance; - const { mostRecent } = await runServiceMapTask( + const { latestTransactionTime } = await runServiceMapTask( server, config, - state.lastRun + state.latestTransactionTime ); - return { - state: { - lastRun: mostRecent - } - }; + return { state: { latestTransactionTime } }; } }; } diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/map_service_connection_to_bulk_index.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/map_service_connection_to_bulk_index.ts new file mode 100644 index 0000000000000..bfe54647d25e0 --- /dev/null +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/map_service_connection_to_bulk_index.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ServiceConnection } from './run_service_map_task'; + +export function mapServiceConnectionToBulkIndex({ + serviceConnsDestinationIndex, + serviceConnsDestinationPipeline, + servicesInTrace +}: { + serviceConnsDestinationIndex: string; + serviceConnsDestinationPipeline: string; + servicesInTrace: string[]; +}) { + return (serviceConnection: ServiceConnection) => { + const indexAction = { + index: { + _index: + serviceConnsDestinationIndex || + (serviceConnection.caller._index as string) + }, + pipeline: serviceConnsDestinationPipeline || undefined // TODO is this even necessary? + }; + + const source = { + '@timestamp': serviceConnection.caller.timestamp, + observer: { version_major: 7 }, // TODO get stack version from NP api + service: { + name: serviceConnection.caller.service_name, + environment: serviceConnection.caller.environment + }, + callee: serviceConnection.callee + ? { + name: serviceConnection.callee.service_name, + environment: serviceConnection.callee.environment + } + : undefined, + connection: { + upstream: { + list: serviceConnection.upstream, + keyword: serviceConnection.upstream.join('->') // TODO is this even used/necessary? + }, + in_trace: servicesInTrace, + type: serviceConnection.caller.span_type, + subtype: serviceConnection.caller.span_subtype + }, + destination: serviceConnection.caller.destination + ? { address: serviceConnection.caller.destination } + : undefined + }; + return [indexAction, source]; + }; +} diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/run_service_map_task.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/run_service_map_task.ts index 97ed214008c6e..faa1fdf58af04 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/run_service_map_task.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/run_service_map_task.ts @@ -5,246 +5,124 @@ */ import { Server } from 'hapi'; +import { uniq } from 'lodash'; +import { BulkIndexDocumentsParams } from 'elasticsearch'; +import { getSearchClient, SearchClient } from '../helpers/es_client'; +import { Span } from '../../../typings/es_schemas/ui/Span'; +import { getNextTransactionSamples } from './get_next_transaction_samples'; +import { getServiceConnections } from './get_service_connections'; +import { mapServiceConnectionToBulkIndex } from './map_service_connection_to_bulk_index'; +import { ENVIRONMENT_NOT_DEFINED } from '../../../common/environment_filter_values'; + +interface MappedSpan { + transaction?: boolean; + id: string; // span or transaction id + parent?: string; // parent.id + environment?: string; // service.environment + destination?: string; // destination.address + _index?: string; // _index // TODO should this be required? + span_type?: Span['span']['type']; + span_subtype?: Span['span']['subtype']; + timestamp: Span['@timestamp']; + service_name: Span['service']['name']; +} -function interestingTransactions(since?: string, afterKey?: any) { - if (!since) { - since = 'now-1h'; - } - const query = { - size: 0, - query: { - bool: { - filter: [ - { exists: { field: 'destination.address' } }, - { exists: { field: 'trace.id' } }, - { exists: { field: 'span.duration.us' } }, - { range: { '@timestamp': { gt: since } } } - ] - } - }, - aggs: { - 'ext-conns': { - composite: { - sources: [ - { 'service.name': { terms: { field: 'service.name' } } }, - { 'span.type': { terms: { field: 'span.type' } } }, - { - 'span.subtype': { - terms: { field: 'span.subtype', missing_bucket: true } - } - }, - { - 'service.environment': { - terms: { field: 'service.environment', missing_bucket: true } - } - }, - { - 'destination.address': { terms: { field: 'destination.address' } } - } - ], - // TODO: needs to be balanced with the 20 below - size: 200, - after: undefined - }, - aggs: { - smpl: { - diversified_sampler: { - shard_size: 20, - script: { - lang: 'painless', - source: "(int)doc['span.duration.us'].value/100000" - } - }, - aggs: { - tracesample: { - top_hits: { - size: 20, - _source: ['trace.id', '@timestamp'] - } - } - } - } - } - } - } - }; - - if (afterKey) { - query.aggs['ext-conns'].composite.after = afterKey; - } - return query; +export interface ServiceConnection { + caller: MappedSpan; + callee?: MappedSpan; + upstream: string[]; // [`${service_name}/${environment}`] } -function findConns(traceIds: string[]) { - const query = { - size: 0, - query: { - bool: { - should: [ - { exists: { field: 'span.id' } }, - { exists: { field: 'transaction.type' } } - ] as any[], - minimum_should_match: 2 - } - }, - aggs: { - trace_id: { - terms: { - field: 'trace.id', - order: { _key: 'asc' }, - size: traceIds.length - }, - aggs: { - connections: { - scripted_metric: { - init_script: 'state.spans = new HashMap();', - map_script: { id: 'map-service-conns' }, - combine_script: { id: 'combine-service-conns' }, - reduce_script: { id: 'reduce-service-conns' } - } - } - } - } - } - }; +async function indexLatestConnections( + config: ReturnType, + searchClient: SearchClient, + bulkClient: (params: BulkIndexDocumentsParams) => Promise, + startTimeInterval = 'now-1h', + latestTransactionTime = 0, + afterKey?: object +): Promise<{ latestTransactionTime: number }> { + const apmIdxPattern = config.get('apm_oss.indexPattern'); + const serviceConnsDestinationIndex = config.get( + 'xpack.apm.serviceMapDestinationIndex' + ); + const serviceConnsDestinationPipeline = config.get( + 'xpack.apm.serviceMapDestinationPipeline' + ); - for (const tid of traceIds) { - query.query.bool.should.push({ term: { 'trace.id': tid } }); + const { + after_key: nextAfterKey, + latestTransactionTime: latestSampleTransactionTime, + traceIds + } = await getNextTransactionSamples({ + apmIdxPattern, + startTimeInterval, + afterKey, + searchClient + }); + + if (traceIds.length === 0) { + return { latestTransactionTime }; } - return query; -} -interface Service { - name: string; - environment?: string; -} - -interface ConnectionDoc { - '@timestamp': string; - observer: { version_major: number }; // TODO: make this dynamic - service: Service; - callee?: Service; - connection: Connection; - destination?: { address: string }; -} + const nextLatestTransactionTime = Math.max( + latestTransactionTime, + latestSampleTransactionTime + ); -interface Connection { - upstream: { list: string[]; keyword: string }; - in_trace: string[]; - type: string; - subtype?: string; + const traceConnectionsBuckets = await getServiceConnections({ + apmIdxPattern, + traceIds, + searchClient + }); + + const bulkIndexConnectionDocs = traceConnectionsBuckets.flatMap(bucket => { + const serviceConnections = bucket.connections.value as ServiceConnection[]; + const servicesInTrace = uniq( + serviceConnections.map( + serviceConnection => + `${serviceConnection.caller.service_name}/${serviceConnection.caller + .environment || ENVIRONMENT_NOT_DEFINED}` + ) + ); + return serviceConnections.flatMap( + mapServiceConnectionToBulkIndex({ + serviceConnsDestinationIndex, + serviceConnsDestinationPipeline, + servicesInTrace + }) + ); + }); + + await bulkClient({ + body: bulkIndexConnectionDocs + .map(bulkObject => JSON.stringify(bulkObject)) + .join('\n') + }); + return await indexLatestConnections( + config, + searchClient, + bulkClient, + startTimeInterval, + nextLatestTransactionTime, + nextAfterKey + ); } export async function runServiceMapTask( server: Server, - config: any, - lastRun?: string + config: ReturnType, + startTimeInterval?: string ) { - const apmIdxPattern = config.get('apm_oss.indexPattern'); - const serviceConnsDestinationIndex = config.get( - 'xpack.apm.serviceMapDestinationIndex' - ); - const serviceConnsDestinationPipeline = config.get( - 'xpack.apm.serviceMapDestinationPipeline' - ); - const callCluster = server.plugins.elasticsearch.getCluster('data') .callWithInternalUser; - - let mostRecent = ''; - let afterKey = null; - while (true) { - const q = interestingTransactions(lastRun, afterKey); - const txs = await callCluster('search', { - index: apmIdxPattern, - body: q - }); - - if (txs.aggregations['ext-conns'].buckets.length < 1) { - return { mostRecent }; - } - afterKey = txs.aggregations['ext-conns'].after_key; - - const traces = new Set(); - - txs.aggregations['ext-conns'].buckets.forEach((bucket: any) => { - bucket.smpl.tracesample.hits.hits.forEach((hit: any) => { - traces.add(hit._source.trace.id); - mostRecent = - mostRecent > hit._source['@timestamp'] - ? mostRecent - : hit._source['@timestamp']; - }); - }); - - const traceIds = Array.from(traces.values()); - if (traceIds.length < 1) { - return { mostRecent: null }; - } - - const findConnsQ = findConns(traceIds); - - const connsResult = await callCluster('search', { - index: apmIdxPattern, - body: findConnsQ - }); - - const connDocs: Array<{ index: { _index: any } } | ConnectionDoc> = []; - - connsResult.aggregations.trace_id.buckets.forEach((bucket: any) => { - const allServices = new Set( - bucket.connections.value.map( - (conn: any) => - `${conn.caller.service_name}/${conn.caller.environment || 'null'}` - ) - ); - - bucket.connections.value.forEach((conn: any) => { - const index = serviceConnsDestinationIndex - ? serviceConnsDestinationIndex - : conn.caller._index; - const bulkOpts = { index: { _index: index }, pipeline: undefined }; - - if (serviceConnsDestinationPipeline) { - bulkOpts.pipeline = serviceConnsDestinationPipeline; - } - connDocs.push(bulkOpts); - const doc: ConnectionDoc = { - '@timestamp': conn.caller.timestamp, - observer: { version_major: 7 }, // TODO: make this dynamic - service: { - name: conn.caller.service_name, - environment: conn.caller.environment - }, - callee: conn.callee - ? { - name: conn.callee.service_name, - environment: conn.callee.environment - } - : undefined, - connection: { - upstream: { - list: conn.upstream, - keyword: conn.upstream.join('->') - }, - in_trace: Array.from(allServices), - type: conn.caller.span_type, - subtype: conn.caller.span_substype - }, - destination: conn.caller.destination - ? { address: conn.caller.destination } - : undefined - }; - - connDocs.push(doc); - }); - }); - - const body = connDocs - .map((connDoc: any) => JSON.stringify(connDoc)) - .join('\n'); - await callCluster('bulk', { - body - }); - } + const searchClient = getSearchClient(callCluster); + const bulkClient = (params: BulkIndexDocumentsParams) => + callCluster('bulk', params); + + return await indexLatestConnections( + config, + searchClient, + bulkClient, + startTimeInterval + ); } diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/setup_required_scripts.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/setup_required_scripts.ts index 9b539feb08d7b..f6f9b5bc4511c 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/setup_required_scripts.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/setup_required_scripts.ts @@ -39,7 +39,7 @@ async function putScriptMapServiceConns(callCluster: CallCluster) { s.destination = doc['destination.address'].value; } - if (!doc['_index'].empty) { + if (!doc['_index'].empty) { // TODO is this ever empty? s._index = doc['_index'].value; } @@ -281,7 +281,7 @@ async function applyExtractDestinationToApm(callCluster: CallCluster) { } export async function setupRequiredScripts(server: Legacy.Server) { - const callCluster = server.plugins.elasticsearch.getCluster('data') + const callCluster = server.plugins.elasticsearch.getCluster('admin') .callWithInternalUser; const putRequiredScriptsResults = await Promise.all([ diff --git a/x-pack/legacy/plugins/apm/typings/elasticsearch/aggregations.ts b/x-pack/legacy/plugins/apm/typings/elasticsearch/aggregations.ts index cabff77fd2a7b..609ec49614043 100644 --- a/x-pack/legacy/plugins/apm/typings/elasticsearch/aggregations.ts +++ b/x-pack/legacy/plugins/apm/typings/elasticsearch/aggregations.ts @@ -29,7 +29,7 @@ type MetricsAggregationOptions = missing?: number; } | { - script?: Script; + script?: Script; // TODO should this be required? }; interface MetricsAggregationResponsePart { @@ -97,7 +97,6 @@ export interface AggregationOptionsByType { buckets_path: BucketsPath; script?: Script; }; - composite: { sources: Array<{ [name: string]: @@ -110,6 +109,18 @@ export interface AggregationOptionsByType { | undefined; }>; size?: number; + after?: object; + }; + diversified_sampler: { + shard_size?: number; + max_docs_per_value?: number; + } & ({ script: Script } | { field: string }); // TODO use MetricsAggregationOptions if possible + scripted_metric: { + params?: Record; + init_script?: Script; + map_script: Script; + combine_script: Script; + reduce_script: Script; }; } @@ -255,6 +266,13 @@ interface AggregationResponsePart< TDocument > >; + after_key?: object; + }; + diversified_sampler: { + doc_count: number; + } & AggregationResponseMap; + scripted_metric: { + value: unknown; }; } From dc930e856b3651ebe5a1e09e90979d20d47f0e5b Mon Sep 17 00:00:00 2001 From: Oliver Gupte Date: Fri, 6 Dec 2019 04:38:10 -0800 Subject: [PATCH 04/15] fix bug in query and timestamp --- .../apm/server/lib/service_map/get_next_transaction_samples.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/get_next_transaction_samples.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/get_next_transaction_samples.ts index 2e13c2826efbc..f67244541ff54 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/get_next_transaction_samples.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/get_next_transaction_samples.ts @@ -62,7 +62,6 @@ export async function getNextTransactionSamples({ aggs: { smpl: { diversified_sampler: { - composite: [] as never[], // TODO remove this shard_size: 20, script: { lang: 'painless', @@ -91,7 +90,7 @@ export async function getNextTransactionSamples({ return bucket.smpl.tracesample.hits.hits.map(hit => { const transactionDoc = hit._source as Transaction; const traceId = transactionDoc.trace.id; - const timestamp = parseInt(transactionDoc['@timestamp'], 10); + const timestamp = Date.parse(transactionDoc['@timestamp']); return { traceId, timestamp }; }); }); From 0939da863d806d596a02d14128d81d10fc9c6955 Mon Sep 17 00:00:00 2001 From: Oliver Gupte Date: Mon, 9 Dec 2019 16:15:57 -0800 Subject: [PATCH 05/15] Include scripted_metric scripts with the query rather than persisting them in ES. --- .../get_next_transaction_samples.ts | 2 +- .../service_map/get_service_connections.ts | 11 +- .../service_map/initialize_service_maps.ts | 25 +- .../lib/service_map/run_service_map_task.ts | 12 +- .../lib/service_map/setup_required_scripts.ts | 231 ++++++++---------- 5 files changed, 126 insertions(+), 155 deletions(-) diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/get_next_transaction_samples.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/get_next_transaction_samples.ts index f67244541ff54..26759c2f51e93 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/get_next_transaction_samples.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/get_next_transaction_samples.ts @@ -15,7 +15,7 @@ export async function getNextTransactionSamples({ searchClient }: { apmIdxPattern: string; - startTimeInterval: string; + startTimeInterval: string | number; afterKey?: object; searchClient: SearchClient; }) { diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_connections.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_connections.ts index f3dfa2f7c54bb..b0b116ffda914 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_connections.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_connections.ts @@ -5,6 +5,11 @@ */ import { SearchClient } from '../helpers/es_client'; +import { + mapServiceConnsScript, + combineServiceConnsScript, + reduceServiceConnsScript +} from './setup_required_scripts'; export async function getServiceConnections({ apmIdxPattern, @@ -43,9 +48,9 @@ export async function getServiceConnections({ connections: { scripted_metric: { init_script: 'state.spans = new HashMap();', - map_script: { id: 'map-service-conns' }, - combine_script: { id: 'combine-service-conns' }, - reduce_script: { id: 'reduce-service-conns' } + map_script: mapServiceConnsScript, + combine_script: combineServiceConnsScript, + reduce_script: reduceServiceConnsScript } } } diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/initialize_service_maps.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/initialize_service_maps.ts index 70e6b2d782406..92d8bcf767757 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/initialize_service_maps.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/initialize_service_maps.ts @@ -8,7 +8,7 @@ import { Server } from 'hapi'; // @ts-ignore import { TaskManager, RunContext } from '../legacy/plugins/task_manager'; import { runServiceMapTask } from './run_service_map_task'; -import { setupRequiredScripts } from './setup_required_scripts'; +import { setupIngestPipeline } from './setup_required_scripts'; import { SERVICE_MAP_TASK_TYPE, SERVICE_MAP_TASK_ID @@ -17,13 +17,20 @@ import { export async function initializeServiceMaps(server: Server) { const config = server.config(); - setupRequiredScripts(server).catch(error => { - server.log( - 'error', - 'Unable to set up required scripts for APM Service Maps:' - ); - server.log('error', error); - }); + // TODO remove setupIngestPipeline when agents set destination.address (elastic/apm#115) + setupIngestPipeline(server) + .then(() => { + server.log( + ['info', 'plugins', 'apm'], + `Created ingest pipeline to extract destination.address from span names.` + ); + }) + .catch(error => { + server.log( + ['error', 'plugins', 'apm'], + `Unable to setup the ingest pipeline to extract destination.address from span names.\n${error.stack}` + ); + }); const taskManager = server.plugins.task_manager; if (taskManager) { @@ -54,7 +61,7 @@ export async function initializeServiceMaps(server: Server) { return await taskManager.ensureScheduled({ id: SERVICE_MAP_TASK_ID, taskType: SERVICE_MAP_TASK_TYPE, - interval: '10s', + interval: '30s', scope: ['apm'], params: {}, state: {} diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/run_service_map_task.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/run_service_map_task.ts index faa1fdf58af04..b80732b6aafe0 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/run_service_map_task.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/run_service_map_task.ts @@ -37,7 +37,7 @@ async function indexLatestConnections( config: ReturnType, searchClient: SearchClient, bulkClient: (params: BulkIndexDocumentsParams) => Promise, - startTimeInterval = 'now-1h', + startTimeInterval?: string | number, latestTransactionTime = 0, afterKey?: object ): Promise<{ latestTransactionTime: number }> { @@ -48,33 +48,28 @@ async function indexLatestConnections( const serviceConnsDestinationPipeline = config.get( 'xpack.apm.serviceMapDestinationPipeline' ); - const { after_key: nextAfterKey, latestTransactionTime: latestSampleTransactionTime, traceIds } = await getNextTransactionSamples({ apmIdxPattern, - startTimeInterval, + startTimeInterval: startTimeInterval || 'now-1h', afterKey, searchClient }); - if (traceIds.length === 0) { return { latestTransactionTime }; } - const nextLatestTransactionTime = Math.max( latestTransactionTime, latestSampleTransactionTime ); - const traceConnectionsBuckets = await getServiceConnections({ apmIdxPattern, traceIds, searchClient }); - const bulkIndexConnectionDocs = traceConnectionsBuckets.flatMap(bucket => { const serviceConnections = bucket.connections.value as ServiceConnection[]; const servicesInTrace = uniq( @@ -92,7 +87,6 @@ async function indexLatestConnections( }) ); }); - await bulkClient({ body: bulkIndexConnectionDocs .map(bulkObject => JSON.stringify(bulkObject)) @@ -111,7 +105,7 @@ async function indexLatestConnections( export async function runServiceMapTask( server: Server, config: ReturnType, - startTimeInterval?: string + startTimeInterval?: string | number ) { const callCluster = server.plugins.elasticsearch.getCluster('data') .callWithInternalUser; diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/setup_required_scripts.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/setup_required_scripts.ts index f6f9b5bc4511c..f495ffe286e59 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/setup_required_scripts.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/setup_required_scripts.ts @@ -7,144 +7,118 @@ import { Legacy } from 'kibana'; import { CallCluster } from 'src/legacy/core_plugins/elasticsearch'; -const MAP_SERVICE_CONNS_SCRIPT_ID = 'map-service-conns'; -const REDUCE_SERVICE_CONNS_SCRIPT_ID = 'reduce-service-conns'; -const COMBINE_SERVICE_CONNS_SCRIPT_ID = 'combine-service-conns'; -const EXTRACT_DESTINATION_INGEST_PIPELINE_ID = 'extract_destination'; +const EXTRACT_DESTINATION_INGEST_PIPELINE_ID = 'apm_extract_destination'; const APM_INGEST_PIPELINE_ID = 'apm'; -async function putScriptMapServiceConns(callCluster: CallCluster) { - return await callCluster('putScript', { - id: MAP_SERVICE_CONNS_SCRIPT_ID, - body: { - script: { - lang: 'painless', - source: ` - def s = new HashMap(); +export const mapServiceConnsScript = { + lang: 'painless', + source: ` + def s = new HashMap(); - if (!doc['span.id'].empty) { - s.id = doc['span.id'].value - } else { - s.id = doc['transaction.id'].value; - s.transaction = true; - } - if (!doc['parent.id'].empty) { - s.parent = doc['parent.id'].value; - } - if (!doc['service.environment'].empty) { - s.environment = doc['service.environment'].value; - } + if (!doc['span.id'].empty) { + s.id = doc['span.id'].value + } else { + s.id = doc['transaction.id'].value; + s.transaction = true; + } + if (!doc['parent.id'].empty) { + s.parent = doc['parent.id'].value; + } + if (!doc['service.environment'].empty) { + s.environment = doc['service.environment'].value; + } - if (!doc['destination.address'].empty) { - s.destination = doc['destination.address'].value; - } + if (!doc['destination.address'].empty) { + s.destination = doc['destination.address'].value; + } - if (!doc['_index'].empty) { // TODO is this ever empty? - s._index = doc['_index'].value; - } + if (!doc['_index'].empty) { // TODO is this ever empty? + s._index = doc['_index'].value; + } - if (!doc['span.type'].empty) { - s.span_type = doc['span.type'].value; - } + if (!doc['span.type'].empty) { + s.span_type = doc['span.type'].value; + } - if (!doc['span.subtype'].empty) { - s.span_subtype = doc['span.subtype'].value; - } + if (!doc['span.subtype'].empty) { + s.span_subtype = doc['span.subtype'].value; + } - s.timestamp = doc['@timestamp'].value; - s.service_name = doc['service.name'].value; - if(!state.spans.containsKey(s.parent)) { - state.spans.put(s.parent, new ArrayList()) - } + s.timestamp = doc['@timestamp'].value; + s.service_name = doc['service.name'].value; + if(!state.spans.containsKey(s.parent)) { + state.spans.put(s.parent, new ArrayList()) + } - if (s.parent != s.id) { - state.spans[s.parent].add(s) + if (s.parent != s.id) { + state.spans[s.parent].add(s) + } + ` +}; + +export const reduceServiceConnsScript = { + lang: 'painless', + source: ` + void extractChildren(def caller, def spans, def upstream, def conns, def count) { + // TODO: simplify this + if (spans.containsKey(caller.id)) { + for (s in spans[caller.id]) { + if (caller.span_type == 'external') { + upstream.add(caller.service_name + "/" + caller.environment); + def conn = new HashMap(); + conn.caller = caller; + conn.callee = s; + conn.upstream = new ArrayList(upstream); + conns.add(conn); + extractChildren(s, spans, upstream, conns, count); + upstream.remove(upstream.size() - 1); + } else { + extractChildren(s, spans, upstream, conns, count); } - ` + } + } else { + // no connection found + def conn = new HashMap(); + conn.caller = caller; + conn.upstream = new ArrayList(upstream); + conn.upstream.add(caller.service_name + "/" + caller.environment); + conns.add(conn); } } - }); -} - -async function putScriptReduceServiceConns(callCluster: CallCluster) { - return await callCluster('putScript', { - id: REDUCE_SERVICE_CONNS_SCRIPT_ID, - body: { - script: { - lang: 'painless', - source: ` - void extractChildren(def caller, def spans, def upstream, def conns, def count) { - // TODO: simplify this - if (spans.containsKey(caller.id)) { - for(s in spans[caller.id]) { - if (caller.span_type=='external') { - upstream.add(caller.service_name+"/"+caller.environment); - def conn = new HashMap(); - conn.caller = caller; - conn.callee = s; - conn.upstream = new ArrayList(upstream); - conns.add(conn); - - extractChildren(s, spans, upstream, conns, count); - upstream.remove(upstream.size()-1); - } else { - extractChildren(s, spans, upstream, conns, count); - } - } - } else { - // no connection found - def conn = new HashMap(); - conn.caller = caller; - conn.upstream = new ArrayList(upstream); - conn.upstream.add(caller.service_name+"/"+caller.environment); - conns.add(conn); - } - } - def conns = new HashSet(); - def spans = new HashMap(); - - // merge results from shards - for(state in states) { - for(s in state.entrySet()) { - def v = s.getValue(); - def k = s.getKey(); - if(!spans.containsKey(k)) { - spans[k] = v; - } else { - for (p in v) { - spans[k].add(p); - } - } - } - } - - if (spans.containsKey(null) && spans[null].size() > 0) { - def node = spans[null][0]; - def upstream = new ArrayList(); - - extractChildren(node, spans, upstream, conns, 0); - - return new ArrayList(conns) + def conns = new HashSet(); + def spans = new HashMap(); + + // merge results from shards + for (state in states) { + for (s in state.entrySet()) { + def v = s.getValue(); + def k = s.getKey(); + if (!spans.containsKey(k)) { + spans[k] = v; + } else { + for (p in v) { + spans[k].add(p); } - return []; - ` + } } } - }); -} -async function putScriptCombineServiceConns(callCluster: CallCluster) { - return await callCluster('putScript', { - id: COMBINE_SERVICE_CONNS_SCRIPT_ID, - body: { - script: { - lang: 'painless', - source: `return state.spans` - } + if (spans.containsKey(null) && spans[null].size() > 0) { + def node = spans[null][0]; + def upstream = new ArrayList(); + extractChildren(node, spans, upstream, conns, 0); + return new ArrayList(conns) } - }); -} + + return []; + ` +}; + +export const combineServiceConnsScript = { + lang: 'painless', + source: `return state.spans` +}; async function putIngestPipelineExtractDestination(callCluster: CallCluster) { return await callCluster('ingest.putPipeline', { @@ -280,19 +254,10 @@ async function applyExtractDestinationToApm(callCluster: CallCluster) { ]); } -export async function setupRequiredScripts(server: Legacy.Server) { +// TODO remove this when agents set destination.address (elastic/apm#115) +export async function setupIngestPipeline(server: Legacy.Server) { const callCluster = server.plugins.elasticsearch.getCluster('admin') .callWithInternalUser; - - const putRequiredScriptsResults = await Promise.all([ - putScriptMapServiceConns(callCluster), - putScriptReduceServiceConns(callCluster), - putScriptCombineServiceConns(callCluster), - putIngestPipelineExtractDestination(callCluster) // TODO remove this when agents set destination.address (elastic/apm#115) - ]); - - return [ - ...putRequiredScriptsResults, - await applyExtractDestinationToApm(callCluster) // TODO remove this when agents set destination.address (elastic/apm#115) - ]; + await putIngestPipelineExtractDestination(callCluster); + return await applyExtractDestinationToApm(callCluster); } From e4860c51f92b2169b03d28fd7f2e412c7399e050 Mon Sep 17 00:00:00 2001 From: Oliver Gupte Date: Mon, 9 Dec 2019 16:37:37 -0800 Subject: [PATCH 06/15] replace field strings with shared constants --- .../elasticsearch_fieldnames.test.ts.snap | 6 +++ .../apm/common/elasticsearch_fieldnames.ts | 1 + .../get_next_transaction_samples.ts | 49 ++++++++++++------- .../service_map/get_service_connections.ts | 13 +++-- .../map_service_connection_to_bulk_index.ts | 3 +- 5 files changed, 48 insertions(+), 24 deletions(-) diff --git a/x-pack/legacy/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap b/x-pack/legacy/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap index 0c6429f55fc9a..4a148fb86beb0 100644 --- a/x-pack/legacy/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap +++ b/x-pack/legacy/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap @@ -96,6 +96,8 @@ exports[`Error SPAN_SUBTYPE 1`] = `undefined`; exports[`Error SPAN_TYPE 1`] = `undefined`; +exports[`Error TIMESTAMP 1`] = `"Mon Dec 09 2019 16:37:20 GMT-0800 (Pacific Standard Time)"`; + exports[`Error TRACE_ID 1`] = `"trace id"`; exports[`Error TRANSACTION_BREAKDOWN_COUNT 1`] = `undefined`; @@ -216,6 +218,8 @@ exports[`Span SPAN_SUBTYPE 1`] = `"my subtype"`; exports[`Span SPAN_TYPE 1`] = `"span type"`; +exports[`Span TIMESTAMP 1`] = `"Mon Dec 09 2019 16:37:20 GMT-0800 (Pacific Standard Time)"`; + exports[`Span TRACE_ID 1`] = `"trace id"`; exports[`Span TRANSACTION_BREAKDOWN_COUNT 1`] = `undefined`; @@ -336,6 +340,8 @@ exports[`Transaction SPAN_SUBTYPE 1`] = `undefined`; exports[`Transaction SPAN_TYPE 1`] = `undefined`; +exports[`Transaction TIMESTAMP 1`] = `"Mon Dec 09 2019 16:37:20 GMT-0800 (Pacific Standard Time)"`; + exports[`Transaction TRACE_ID 1`] = `"trace id"`; exports[`Transaction TRANSACTION_BREAKDOWN_COUNT 1`] = `undefined`; diff --git a/x-pack/legacy/plugins/apm/common/elasticsearch_fieldnames.ts b/x-pack/legacy/plugins/apm/common/elasticsearch_fieldnames.ts index e5d2b00b1741c..7460f5baf4c39 100644 --- a/x-pack/legacy/plugins/apm/common/elasticsearch_fieldnames.ts +++ b/x-pack/legacy/plugins/apm/common/elasticsearch_fieldnames.ts @@ -29,6 +29,7 @@ export const TRANSACTION_PAGE_URL = 'transaction.page.url'; export const TRACE_ID = 'trace.id'; +export const TIMESTAMP = '@timestamp'; export const SPAN_DURATION = 'span.duration.us'; export const SPAN_TYPE = 'span.type'; export const SPAN_SUBTYPE = 'span.subtype'; diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/get_next_transaction_samples.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/get_next_transaction_samples.ts index 26759c2f51e93..f9a7849ceb643 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/get_next_transaction_samples.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/get_next_transaction_samples.ts @@ -7,6 +7,16 @@ import { uniq } from 'lodash'; import { SearchClient } from '../helpers/es_client'; import { Transaction } from '../../../typings/es_schemas/ui/Transaction'; +import { + SERVICE_NAME, + SERVICE_ENVIRONMENT, + DESTINATION_ADDRESS, + TRACE_ID, + SPAN_DURATION, + SPAN_TYPE, + SPAN_SUBTYPE, + TIMESTAMP +} from '../../../common/elasticsearch_fieldnames'; export async function getNextTransactionSamples({ apmIdxPattern, @@ -26,32 +36,32 @@ export async function getNextTransactionSamples({ query: { bool: { filter: [ - { exists: { field: 'destination.address' } }, - { exists: { field: 'trace.id' } }, - { exists: { field: 'span.duration.us' } }, - { range: { '@timestamp': { gt: startTimeInterval } } } + { exists: { field: DESTINATION_ADDRESS } }, + { exists: { field: TRACE_ID } }, + { exists: { field: SPAN_DURATION } }, + { range: { [TIMESTAMP]: { gt: startTimeInterval } } } ] } }, aggs: { - 'ext-conns': { + externalConnections: { composite: { sources: [ - { 'service.name': { terms: { field: 'service.name' } } }, - { 'span.type': { terms: { field: 'span.type' } } }, + { [SERVICE_NAME]: { terms: { field: SERVICE_NAME } } }, + { [SPAN_TYPE]: { terms: { field: SPAN_TYPE } } }, { - 'span.subtype': { - terms: { field: 'span.subtype', missing_bucket: true } + [SPAN_SUBTYPE]: { + terms: { field: SPAN_SUBTYPE, missing_bucket: true } } }, { - 'service.environment': { - terms: { field: 'service.environment', missing_bucket: true } + [SERVICE_ENVIRONMENT]: { + terms: { field: SERVICE_ENVIRONMENT, missing_bucket: true } } }, { - 'destination.address': { - terms: { field: 'destination.address' } + [DESTINATION_ADDRESS]: { + terms: { field: DESTINATION_ADDRESS } } } ], @@ -65,14 +75,14 @@ export async function getNextTransactionSamples({ shard_size: 20, script: { lang: 'painless', - source: "(int)doc['span.duration.us'].value/100000" + source: `(int)doc['${SPAN_DURATION}'].value/100000` } }, aggs: { tracesample: { top_hits: { size: 20, - _source: ['trace.id', '@timestamp'] + _source: [TRACE_ID, TIMESTAMP] } } } @@ -84,13 +94,14 @@ export async function getNextTransactionSamples({ }; const transactionsResponse = await searchClient(params); - const extConns = transactionsResponse.aggregations?.['ext-conns']; - const buckets = extConns?.buckets ?? []; + const externalConnections = + transactionsResponse.aggregations?.externalConnections; + const buckets = externalConnections?.buckets ?? []; const sampleTraces = buckets.flatMap(bucket => { return bucket.smpl.tracesample.hits.hits.map(hit => { const transactionDoc = hit._source as Transaction; const traceId = transactionDoc.trace.id; - const timestamp = Date.parse(transactionDoc['@timestamp']); + const timestamp = Date.parse(transactionDoc[TIMESTAMP]); return { traceId, timestamp }; }); }); @@ -99,7 +110,7 @@ export async function getNextTransactionSamples({ ); const traceIds = uniq(sampleTraces.map(({ traceId }) => traceId)); return { - after_key: extConns?.after_key, + after_key: externalConnections?.after_key, latestTransactionTime, traceIds }; diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_connections.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_connections.ts index b0b116ffda914..e6592635542f3 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_connections.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_connections.ts @@ -10,6 +10,11 @@ import { combineServiceConnsScript, reduceServiceConnsScript } from './setup_required_scripts'; +import { + SPAN_ID, + TRACE_ID, + TRANSACTION_TYPE +} from '../../../common/elasticsearch_fieldnames'; export async function getServiceConnections({ apmIdxPattern, @@ -21,7 +26,7 @@ export async function getServiceConnections({ searchClient: SearchClient; }) { const traceIdFilters = traceIds.map(traceId => ({ - term: { 'trace.id': traceId } + term: { [TRACE_ID]: traceId } })); const params = { index: apmIdxPattern, @@ -30,8 +35,8 @@ export async function getServiceConnections({ query: { bool: { should: [ - { exists: { field: 'span.id' } }, - { exists: { field: 'transaction.type' } }, + { exists: { field: SPAN_ID } }, + { exists: { field: TRANSACTION_TYPE } }, ...traceIdFilters ], minimum_should_match: 2 @@ -40,7 +45,7 @@ export async function getServiceConnections({ aggs: { trace_id: { terms: { - field: 'trace.id', + field: TRACE_ID, order: { _key: 'asc' as const }, size: traceIds.length }, diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/map_service_connection_to_bulk_index.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/map_service_connection_to_bulk_index.ts index bfe54647d25e0..15c98a29f61cd 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/map_service_connection_to_bulk_index.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/map_service_connection_to_bulk_index.ts @@ -5,6 +5,7 @@ */ import { ServiceConnection } from './run_service_map_task'; +import { TIMESTAMP } from '../../../common/elasticsearch_fieldnames'; export function mapServiceConnectionToBulkIndex({ serviceConnsDestinationIndex, @@ -26,7 +27,7 @@ export function mapServiceConnectionToBulkIndex({ }; const source = { - '@timestamp': serviceConnection.caller.timestamp, + [TIMESTAMP]: serviceConnection.caller.timestamp, observer: { version_major: 7 }, // TODO get stack version from NP api service: { name: serviceConnection.caller.service_name, From 577297dee7a02f5f8d6048ce919ef6d070ac124e Mon Sep 17 00:00:00 2001 From: Oliver Gupte Date: Tue, 10 Dec 2019 00:35:49 -0800 Subject: [PATCH 07/15] Addressed feedback and fixed timestamp snapshot --- .../elasticsearch_fieldnames.test.ts.snap | 6 +- .../common/elasticsearch_fieldnames.test.ts | 6 +- .../get_next_transaction_samples.ts | 7 +- .../service_map/get_service_connections.ts | 10 +-- .../server/lib/service_map/get_service_map.ts | 83 +++++++++---------- ...> map_trace_to_bulk_service_connection.ts} | 31 ++++--- .../lib/service_map/run_service_map_task.ts | 29 ++++--- 7 files changed, 85 insertions(+), 87 deletions(-) rename x-pack/legacy/plugins/apm/server/lib/service_map/{map_service_connection_to_bulk_index.ts => map_trace_to_bulk_service_connection.ts} (53%) diff --git a/x-pack/legacy/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap b/x-pack/legacy/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap index 4a148fb86beb0..72ee0abb5b394 100644 --- a/x-pack/legacy/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap +++ b/x-pack/legacy/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap @@ -96,7 +96,7 @@ exports[`Error SPAN_SUBTYPE 1`] = `undefined`; exports[`Error SPAN_TYPE 1`] = `undefined`; -exports[`Error TIMESTAMP 1`] = `"Mon Dec 09 2019 16:37:20 GMT-0800 (Pacific Standard Time)"`; +exports[`Error TIMESTAMP 1`] = `"Wed Dec 31 1969 16:00:01 GMT-0800 (Pacific Standard Time)"`; exports[`Error TRACE_ID 1`] = `"trace id"`; @@ -218,7 +218,7 @@ exports[`Span SPAN_SUBTYPE 1`] = `"my subtype"`; exports[`Span SPAN_TYPE 1`] = `"span type"`; -exports[`Span TIMESTAMP 1`] = `"Mon Dec 09 2019 16:37:20 GMT-0800 (Pacific Standard Time)"`; +exports[`Span TIMESTAMP 1`] = `"Wed Dec 31 1969 16:00:01 GMT-0800 (Pacific Standard Time)"`; exports[`Span TRACE_ID 1`] = `"trace id"`; @@ -340,7 +340,7 @@ exports[`Transaction SPAN_SUBTYPE 1`] = `undefined`; exports[`Transaction SPAN_TYPE 1`] = `undefined`; -exports[`Transaction TIMESTAMP 1`] = `"Mon Dec 09 2019 16:37:20 GMT-0800 (Pacific Standard Time)"`; +exports[`Transaction TIMESTAMP 1`] = `"Wed Dec 31 1969 16:00:01 GMT-0800 (Pacific Standard Time)"`; exports[`Transaction TRACE_ID 1`] = `"trace id"`; diff --git a/x-pack/legacy/plugins/apm/common/elasticsearch_fieldnames.test.ts b/x-pack/legacy/plugins/apm/common/elasticsearch_fieldnames.test.ts index 82a679ccdd32e..46b60542727f5 100644 --- a/x-pack/legacy/plugins/apm/common/elasticsearch_fieldnames.test.ts +++ b/x-pack/legacy/plugins/apm/common/elasticsearch_fieldnames.test.ts @@ -13,7 +13,7 @@ import * as fieldnames from './elasticsearch_fieldnames'; describe('Transaction', () => { const transaction: AllowUnknownProperties = { - '@timestamp': new Date().toString(), + '@timestamp': new Date(1337).toString(), '@metadata': 'whatever', observer: 'whatever', agent: { @@ -61,7 +61,7 @@ describe('Transaction', () => { describe('Span', () => { const span: AllowUnknownProperties = { - '@timestamp': new Date().toString(), + '@timestamp': new Date(1337).toString(), '@metadata': 'whatever', observer: 'whatever', agent: { @@ -125,7 +125,7 @@ describe('Error', () => { id: 'error id', grouping_key: 'grouping key' }, - '@timestamp': new Date().toString(), + '@timestamp': new Date(1337).toString(), host: { hostname: 'my hostname' }, diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/get_next_transaction_samples.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/get_next_transaction_samples.ts index f9a7849ceb643..4950f37c399b1 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/get_next_transaction_samples.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/get_next_transaction_samples.ts @@ -19,18 +19,18 @@ import { } from '../../../common/elasticsearch_fieldnames'; export async function getNextTransactionSamples({ - apmIdxPattern, + targetApmIndices, startTimeInterval, afterKey, searchClient }: { - apmIdxPattern: string; + targetApmIndices: string[]; startTimeInterval: string | number; afterKey?: object; searchClient: SearchClient; }) { const params = { - index: apmIdxPattern, + index: targetApmIndices, body: { size: 0, query: { @@ -71,6 +71,7 @@ export async function getNextTransactionSamples({ }, aggs: { smpl: { + // get sample within a 0.1 second range diversified_sampler: { shard_size: 20, script: { diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_connections.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_connections.ts index e6592635542f3..0c0042001aee3 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_connections.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_connections.ts @@ -17,11 +17,11 @@ import { } from '../../../common/elasticsearch_fieldnames'; export async function getServiceConnections({ - apmIdxPattern, + targetApmIndices, traceIds, searchClient }: { - apmIdxPattern: string; + targetApmIndices: string[]; traceIds: string[]; searchClient: SearchClient; }) { @@ -29,7 +29,7 @@ export async function getServiceConnections({ term: { [TRACE_ID]: traceId } })); const params = { - index: apmIdxPattern, + index: targetApmIndices, body: { size: 0, query: { @@ -43,7 +43,7 @@ export async function getServiceConnections({ } }, aggs: { - trace_id: { + traces: { terms: { field: TRACE_ID, order: { _key: 'asc' as const }, @@ -65,6 +65,6 @@ export async function getServiceConnections({ }; const serviceConnectionsResponse = await searchClient(params); const traceConnectionsBuckets = - serviceConnectionsResponse.aggregations?.trace_id.buckets ?? []; + serviceConnectionsResponse.aggregations?.traces.buckets ?? []; return traceConnectionsBuckets; } diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_map.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_map.ts index 33a79710c5f55..cc5a19fe0f20c 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_map.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_map.ts @@ -27,16 +27,18 @@ export interface IEnvOptions { environment?: string; } -interface ServiceMapElement { - data: - | { - id: string; - } - | { - id: string; - source: string; - target: string; - }; +interface CyNode { + id: string; +} + +interface CyEdge { + id: string; + source: string; + target: string; +} + +interface CyElement { + data: CyNode | CyEdge; } export type ServiceMapAPIResponse = PromiseReturnType; @@ -124,45 +126,40 @@ export async function getServiceMap({ return []; } - const initialServiceMapNode: ServiceMapElement = { + const initialServiceMapNode: CyElement = { data: { id: buckets[0].key[SERVICE_NAME] } }; - return buckets.reduce( - (acc: ServiceMapElement[], { key: connection, dests }) => { - const connectionServiceName = connection[SERVICE_NAME]; + return buckets.reduce((acc: CyElement[], { key: connection, dests }) => { + const connectionServiceName = connection[SERVICE_NAME]; - const destinationNames = - dests.buckets.length === 0 - ? [connection[DESTINATION_ADDRESS]] - : dests.buckets.map( - ({ key: destinationName }) => destinationName as string - ); + const destinationNames = + dests.buckets.length === 0 + ? [connection[DESTINATION_ADDRESS]] + : dests.buckets.map( + ({ key: destinationName }) => destinationName as string + ); - const serviceMapConnections = destinationNames.flatMap( - destinationName => [ - { - data: { - id: destinationName - } - }, - { - data: { - id: `${connectionServiceName}~${destinationName}`, - source: connectionServiceName, - target: destinationName - } - } - ] - ); - - if (acc.length === 0) { - return [initialServiceMapNode, ...serviceMapConnections]; + const serviceMapConnections = destinationNames.flatMap(destinationName => [ + { + data: { + id: destinationName + } + }, + { + data: { + id: `${connectionServiceName}~${destinationName}`, + source: connectionServiceName, + target: destinationName + } } - return [...acc, ...serviceMapConnections]; - }, - [] - ); + ]); + + if (acc.length === 0) { + return [initialServiceMapNode, ...serviceMapConnections]; + } + return [...acc, ...serviceMapConnections]; + }, []); } diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/map_service_connection_to_bulk_index.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/map_trace_to_bulk_service_connection.ts similarity index 53% rename from x-pack/legacy/plugins/apm/server/lib/service_map/map_service_connection_to_bulk_index.ts rename to x-pack/legacy/plugins/apm/server/lib/service_map/map_trace_to_bulk_service_connection.ts index 15c98a29f61cd..2017407dd458b 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/map_service_connection_to_bulk_index.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/map_trace_to_bulk_service_connection.ts @@ -4,10 +4,10 @@ * you may not use this file except in compliance with the Elastic License. */ -import { ServiceConnection } from './run_service_map_task'; +import { TraceConnection } from './run_service_map_task'; import { TIMESTAMP } from '../../../common/elasticsearch_fieldnames'; -export function mapServiceConnectionToBulkIndex({ +export function mapTraceToBulkServiceConnection({ serviceConnsDestinationIndex, serviceConnsDestinationPipeline, servicesInTrace @@ -16,41 +16,38 @@ export function mapServiceConnectionToBulkIndex({ serviceConnsDestinationPipeline: string; servicesInTrace: string[]; }) { - return (serviceConnection: ServiceConnection) => { + return (traceConnection: TraceConnection) => { const indexAction = { index: { _index: serviceConnsDestinationIndex || - (serviceConnection.caller._index as string) + (traceConnection.caller._index as string) }, pipeline: serviceConnsDestinationPipeline || undefined // TODO is this even necessary? }; const source = { - [TIMESTAMP]: serviceConnection.caller.timestamp, + [TIMESTAMP]: traceConnection.caller.timestamp, observer: { version_major: 7 }, // TODO get stack version from NP api service: { - name: serviceConnection.caller.service_name, - environment: serviceConnection.caller.environment + name: traceConnection.caller.service_name, + environment: traceConnection.caller.environment }, - callee: serviceConnection.callee + callee: traceConnection.callee ? { - name: serviceConnection.callee.service_name, - environment: serviceConnection.callee.environment + name: traceConnection.callee.service_name, + environment: traceConnection.callee.environment } : undefined, connection: { upstream: { - list: serviceConnection.upstream, - keyword: serviceConnection.upstream.join('->') // TODO is this even used/necessary? + list: traceConnection.upstream }, in_trace: servicesInTrace, - type: serviceConnection.caller.span_type, - subtype: serviceConnection.caller.span_subtype + type: traceConnection.caller.span_type, + subtype: traceConnection.caller.span_subtype }, - destination: serviceConnection.caller.destination - ? { address: serviceConnection.caller.destination } - : undefined + destination: { address: traceConnection.caller.destination } }; return [indexAction, source]; }; diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/run_service_map_task.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/run_service_map_task.ts index b80732b6aafe0..b3e3c4ba0d0fd 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/run_service_map_task.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/run_service_map_task.ts @@ -11,10 +11,10 @@ import { getSearchClient, SearchClient } from '../helpers/es_client'; import { Span } from '../../../typings/es_schemas/ui/Span'; import { getNextTransactionSamples } from './get_next_transaction_samples'; import { getServiceConnections } from './get_service_connections'; -import { mapServiceConnectionToBulkIndex } from './map_service_connection_to_bulk_index'; +import { mapTraceToBulkServiceConnection } from './map_trace_to_bulk_service_connection'; import { ENVIRONMENT_NOT_DEFINED } from '../../../common/environment_filter_values'; -interface MappedSpan { +interface MappedDocument { transaction?: boolean; id: string; // span or transaction id parent?: string; // parent.id @@ -23,13 +23,13 @@ interface MappedSpan { _index?: string; // _index // TODO should this be required? span_type?: Span['span']['type']; span_subtype?: Span['span']['subtype']; - timestamp: Span['@timestamp']; + timestamp: string; service_name: Span['service']['name']; } -export interface ServiceConnection { - caller: MappedSpan; - callee?: MappedSpan; +export interface TraceConnection { + caller: MappedDocument; + callee?: MappedDocument; upstream: string[]; // [`${service_name}/${environment}`] } @@ -41,7 +41,10 @@ async function indexLatestConnections( latestTransactionTime = 0, afterKey?: object ): Promise<{ latestTransactionTime: number }> { - const apmIdxPattern = config.get('apm_oss.indexPattern'); + const targetApmIndices = [ + config.get('apm_oss.transactionIndices'), + config.get('apm_oss.spanIndices') + ]; const serviceConnsDestinationIndex = config.get( 'xpack.apm.serviceMapDestinationIndex' ); @@ -53,7 +56,7 @@ async function indexLatestConnections( latestTransactionTime: latestSampleTransactionTime, traceIds } = await getNextTransactionSamples({ - apmIdxPattern, + targetApmIndices, startTimeInterval: startTimeInterval || 'now-1h', afterKey, searchClient @@ -66,21 +69,21 @@ async function indexLatestConnections( latestSampleTransactionTime ); const traceConnectionsBuckets = await getServiceConnections({ - apmIdxPattern, + targetApmIndices, traceIds, searchClient }); const bulkIndexConnectionDocs = traceConnectionsBuckets.flatMap(bucket => { - const serviceConnections = bucket.connections.value as ServiceConnection[]; + const traceConnections = bucket.connections.value as TraceConnection[]; const servicesInTrace = uniq( - serviceConnections.map( + traceConnections.map( serviceConnection => `${serviceConnection.caller.service_name}/${serviceConnection.caller .environment || ENVIRONMENT_NOT_DEFINED}` ) ); - return serviceConnections.flatMap( - mapServiceConnectionToBulkIndex({ + return traceConnections.flatMap( + mapTraceToBulkServiceConnection({ serviceConnsDestinationIndex, serviceConnsDestinationPipeline, servicesInTrace From 0a3e0aedbc6d520ba6470343dc0d3b9a09623b91 Mon Sep 17 00:00:00 2001 From: Oliver Gupte Date: Tue, 10 Dec 2019 03:11:44 -0800 Subject: [PATCH 08/15] creates apm-service-connections index and adds it to getApmIndices --- .../apm/common/service_map_constants.ts | 4 +- x-pack/legacy/plugins/apm/index.ts | 5 +- .../plugins/apm/public/utils/testHelpers.tsx | 4 +- .../__tests__/get_buckets.test.ts | 3 +- .../create_service_connections_index.ts | 111 ++++++++++++++++++ .../service_map/get_service_connections.ts | 2 +- .../server/lib/service_map/get_service_map.ts | 4 +- .../service_map/initialize_service_maps.ts | 16 +-- .../map_trace_to_bulk_service_connection.ts | 32 ++--- .../lib/service_map/run_service_map_task.ts | 32 +++-- ...ts.ts => service-connection-es-scripts.ts} | 4 - .../settings/apm_indices/get_apm_indices.ts | 4 +- .../lib/transaction_groups/fetcher.test.ts | 3 +- .../lib/transactions/breakdown/index.test.ts | 3 +- .../charts/get_anomaly_data/index.test.ts | 3 +- .../get_timeseries_data/fetcher.test.ts | 3 +- x-pack/plugins/apm/server/index.ts | 6 - 17 files changed, 166 insertions(+), 73 deletions(-) create mode 100644 x-pack/legacy/plugins/apm/server/lib/service_map/create_service_connections_index.ts rename x-pack/legacy/plugins/apm/server/lib/service_map/{setup_required_scripts.ts => service-connection-es-scripts.ts} (98%) diff --git a/x-pack/legacy/plugins/apm/common/service_map_constants.ts b/x-pack/legacy/plugins/apm/common/service_map_constants.ts index 70ad80f08e4b6..13d3e9901a51a 100644 --- a/x-pack/legacy/plugins/apm/common/service_map_constants.ts +++ b/x-pack/legacy/plugins/apm/common/service_map_constants.ts @@ -4,5 +4,5 @@ * you may not use this file except in compliance with the Elastic License. */ -export const SERVICE_MAP_TASK_TYPE = 'serviceMap'; -export const SERVICE_MAP_TASK_ID = 'servicemap-processor'; +export const SERVICE_MAP_TASK_TYPE = 'apmServiceMap'; +export const SERVICE_MAP_TASK_ID = 'apm-servicemap-processor'; diff --git a/x-pack/legacy/plugins/apm/index.ts b/x-pack/legacy/plugins/apm/index.ts index e2ac07901fce1..1a78a61254bf0 100644 --- a/x-pack/legacy/plugins/apm/index.ts +++ b/x-pack/legacy/plugins/apm/index.ts @@ -79,10 +79,7 @@ export const apm: LegacyPluginInitializer = kibana => { // service map - serviceMapEnabled: Joi.boolean().default(false), - serviceMapIndexPattern: Joi.string().default('apm-*'), - serviceMapDestinationIndex: Joi.string(), - serviceMapDestinationPipeline: Joi.string() + serviceMapEnabled: Joi.boolean().default(false) }).default(); }, diff --git a/x-pack/legacy/plugins/apm/public/utils/testHelpers.tsx b/x-pack/legacy/plugins/apm/public/utils/testHelpers.tsx index 862c982d6b5ac..e104aced169d8 100644 --- a/x-pack/legacy/plugins/apm/public/utils/testHelpers.tsx +++ b/x-pack/legacy/plugins/apm/public/utils/testHelpers.tsx @@ -118,6 +118,7 @@ interface MockSetup { 'apm_oss.transactionIndices': string; 'apm_oss.metricsIndices': string; apmAgentConfigurationIndex: string; + apmServiceConnectionsIndex: string; }; } @@ -175,7 +176,8 @@ export async function inspectSearchParams( 'apm_oss.spanIndices': 'myIndex', 'apm_oss.transactionIndices': 'myIndex', 'apm_oss.metricsIndices': 'myIndex', - apmAgentConfigurationIndex: 'myIndex' + apmAgentConfigurationIndex: 'myIndex', + apmServiceConnectionsIndex: 'myIndex' }, dynamicIndexPattern: null as any }; diff --git a/x-pack/legacy/plugins/apm/server/lib/errors/distribution/__tests__/get_buckets.test.ts b/x-pack/legacy/plugins/apm/server/lib/errors/distribution/__tests__/get_buckets.test.ts index cf8798d445f8a..507d1a5af840e 100644 --- a/x-pack/legacy/plugins/apm/server/lib/errors/distribution/__tests__/get_buckets.test.ts +++ b/x-pack/legacy/plugins/apm/server/lib/errors/distribution/__tests__/get_buckets.test.ts @@ -53,7 +53,8 @@ describe('timeseriesFetcher', () => { 'apm_oss.spanIndices': 'apm-*', 'apm_oss.transactionIndices': 'apm-*', 'apm_oss.metricsIndices': 'apm-*', - apmAgentConfigurationIndex: '.apm-agent-configuration' + apmAgentConfigurationIndex: '.apm-agent-configuration', + apmServiceConnectionsIndex: 'apm-service-connections' }, dynamicIndexPattern: null as any } diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/create_service_connections_index.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/create_service_connections_index.ts new file mode 100644 index 0000000000000..490647421c63f --- /dev/null +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/create_service_connections_index.ts @@ -0,0 +1,111 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Server } from 'hapi'; +import { APMPluginContract } from '../../../../../../plugins/apm/server/plugin'; +import { getInternalSavedObjectsClient } from '../helpers/saved_objects_client'; +import { CallCluster } from '../../../../../../../src/legacy/core_plugins/elasticsearch'; +import { TIMESTAMP } from '../../../common/elasticsearch_fieldnames'; + +export async function createServiceConnectionsIndex(server: Server) { + const callCluster = server.plugins.elasticsearch.getCluster('data') + .callWithInternalUser; + const apmPlugin = server.newPlatform.setup.plugins.apm as APMPluginContract; + + try { + const apmIndices = await apmPlugin.getApmIndices( + getInternalSavedObjectsClient(server) + ); + const index = apmIndices.apmServiceConnectionsIndex; + const indexExists = await callCluster('indices.exists', { index }); + if (!indexExists) { + const result = await createNewIndex(index, callCluster); + + if (!result.acknowledged) { + const resultError = + result && result.error && JSON.stringify(result.error); + throw new Error( + `Unable to create APM Service Connections index '${index}': ${resultError}` + ); + } + } + } catch (e) { + // eslint-disable-next-line no-console + console.error('Could not create APM Service Connections:', e.message); + } +} + +function createNewIndex(index: string, callWithInternalUser: CallCluster) { + return callWithInternalUser('indices.create', { + index, + body: { + settings: { 'index.auto_expand_replicas': '0-1' }, + mappings: { properties: mappingProperties } + } + }); +} + +const mappingProperties = { + [TIMESTAMP]: { + type: 'date' + }, + service: { + properties: { + name: { + type: 'keyword', + ignore_above: 1024 + }, + environment: { + type: 'keyword', + ignore_above: 1024 + } + } + }, + callee: { + properties: { + name: { + type: 'keyword', + ignore_above: 1024 + }, + environment: { + type: 'keyword', + ignore_above: 1024 + } + } + }, + connection: { + properties: { + upstream: { + properties: { + list: { + type: 'keyword', + ignore_above: 1024 + } + } + }, + in_trace: { + type: 'keyword', + ignore_above: 1024 + }, + type: { + type: 'keyword', + ignore_above: 1024 + }, + subtype: { + type: 'keyword', + ignore_above: 1024 + } + } + }, + destination: { + properties: { + address: { + type: 'keyword', + ignore_above: 1024 + } + } + } +}; diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_connections.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_connections.ts index 0c0042001aee3..124900bed9b53 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_connections.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_connections.ts @@ -9,7 +9,7 @@ import { mapServiceConnsScript, combineServiceConnsScript, reduceServiceConnsScript -} from './setup_required_scripts'; +} from './service-connection-es-scripts'; import { SPAN_ID, TRACE_ID, diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_map.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_map.ts index cc5a19fe0f20c..77593e7435add 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_map.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_map.ts @@ -47,10 +47,10 @@ export async function getServiceMap({ serviceName, environment }: IEnvOptions) { - const { start, end, client, config, uiFiltersES } = setup; + const { start, end, client, uiFiltersES, indices } = setup; const params = { - index: config['xpack.apm.serviceMapIndexPattern'], + index: indices.apmServiceConnectionsIndex, body: { size: 0, query: { diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/initialize_service_maps.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/initialize_service_maps.ts index 92d8bcf767757..4e0dde778974e 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/initialize_service_maps.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/initialize_service_maps.ts @@ -8,15 +8,14 @@ import { Server } from 'hapi'; // @ts-ignore import { TaskManager, RunContext } from '../legacy/plugins/task_manager'; import { runServiceMapTask } from './run_service_map_task'; -import { setupIngestPipeline } from './setup_required_scripts'; +import { setupIngestPipeline } from './service-connection-es-scripts'; import { SERVICE_MAP_TASK_TYPE, SERVICE_MAP_TASK_ID } from '../../../common/service_map_constants'; +import { createServiceConnectionsIndex } from './create_service_connections_index'; export async function initializeServiceMaps(server: Server) { - const config = server.config(); - // TODO remove setupIngestPipeline when agents set destination.address (elastic/apm#115) setupIngestPipeline(server) .then(() => { @@ -32,13 +31,15 @@ export async function initializeServiceMaps(server: Server) { ); }); + await createServiceConnectionsIndex(server); + const taskManager = server.plugins.task_manager; if (taskManager) { taskManager.registerTaskDefinitions({ - serviceMap: { - title: 'ServiceMapTask', + [SERVICE_MAP_TASK_TYPE]: { + title: 'ApmServiceMapTask', type: SERVICE_MAP_TASK_TYPE, - description: 'Extract connections in traces for service maps', + description: 'Extract connections in traces for APM service maps', timeout: '5m', createTaskRunner({ taskInstance }: RunContext) { return { @@ -47,7 +48,6 @@ export async function initializeServiceMaps(server: Server) { const { latestTransactionTime } = await runServiceMapTask( server, - config, state.latestTransactionTime ); @@ -61,7 +61,7 @@ export async function initializeServiceMaps(server: Server) { return await taskManager.ensureScheduled({ id: SERVICE_MAP_TASK_ID, taskType: SERVICE_MAP_TASK_TYPE, - interval: '30s', + interval: '1m', scope: ['apm'], params: {}, state: {} diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/map_trace_to_bulk_service_connection.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/map_trace_to_bulk_service_connection.ts index 2017407dd458b..94bd437b9d05d 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/map_trace_to_bulk_service_connection.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/map_trace_to_bulk_service_connection.ts @@ -6,39 +6,29 @@ import { TraceConnection } from './run_service_map_task'; import { TIMESTAMP } from '../../../common/elasticsearch_fieldnames'; +import { ApmIndicesConfig } from '../settings/apm_indices/get_apm_indices'; -export function mapTraceToBulkServiceConnection({ - serviceConnsDestinationIndex, - serviceConnsDestinationPipeline, - servicesInTrace -}: { - serviceConnsDestinationIndex: string; - serviceConnsDestinationPipeline: string; - servicesInTrace: string[]; -}) { +export function mapTraceToBulkServiceConnection( + apmIndices: ApmIndicesConfig, + servicesInTrace: string[] +) { return (traceConnection: TraceConnection) => { const indexAction = { index: { - _index: - serviceConnsDestinationIndex || - (traceConnection.caller._index as string) - }, - pipeline: serviceConnsDestinationPipeline || undefined // TODO is this even necessary? + _index: apmIndices.apmServiceConnectionsIndex + } }; const source = { [TIMESTAMP]: traceConnection.caller.timestamp, - observer: { version_major: 7 }, // TODO get stack version from NP api service: { name: traceConnection.caller.service_name, environment: traceConnection.caller.environment }, - callee: traceConnection.callee - ? { - name: traceConnection.callee.service_name, - environment: traceConnection.callee.environment - } - : undefined, + callee: { + name: traceConnection.callee?.service_name, + environment: traceConnection.callee?.environment + }, connection: { upstream: { list: traceConnection.upstream diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/run_service_map_task.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/run_service_map_task.ts index b3e3c4ba0d0fd..3c2013085b274 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/run_service_map_task.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/run_service_map_task.ts @@ -7,12 +7,15 @@ import { Server } from 'hapi'; import { uniq } from 'lodash'; import { BulkIndexDocumentsParams } from 'elasticsearch'; +import { APMPluginContract } from '../../../../../../plugins/apm/server/plugin'; import { getSearchClient, SearchClient } from '../helpers/es_client'; import { Span } from '../../../typings/es_schemas/ui/Span'; import { getNextTransactionSamples } from './get_next_transaction_samples'; import { getServiceConnections } from './get_service_connections'; import { mapTraceToBulkServiceConnection } from './map_trace_to_bulk_service_connection'; import { ENVIRONMENT_NOT_DEFINED } from '../../../common/environment_filter_values'; +import { getInternalSavedObjectsClient } from '../helpers/saved_objects_client'; +import { ApmIndicesConfig } from '../settings/apm_indices/get_apm_indices'; interface MappedDocument { transaction?: boolean; @@ -20,7 +23,6 @@ interface MappedDocument { parent?: string; // parent.id environment?: string; // service.environment destination?: string; // destination.address - _index?: string; // _index // TODO should this be required? span_type?: Span['span']['type']; span_subtype?: Span['span']['subtype']; timestamp: string; @@ -34,7 +36,7 @@ export interface TraceConnection { } async function indexLatestConnections( - config: ReturnType, + apmIndices: ApmIndicesConfig, searchClient: SearchClient, bulkClient: (params: BulkIndexDocumentsParams) => Promise, startTimeInterval?: string | number, @@ -42,15 +44,9 @@ async function indexLatestConnections( afterKey?: object ): Promise<{ latestTransactionTime: number }> { const targetApmIndices = [ - config.get('apm_oss.transactionIndices'), - config.get('apm_oss.spanIndices') + apmIndices['apm_oss.transactionIndices'], + apmIndices['apm_oss.spanIndices'] ]; - const serviceConnsDestinationIndex = config.get( - 'xpack.apm.serviceMapDestinationIndex' - ); - const serviceConnsDestinationPipeline = config.get( - 'xpack.apm.serviceMapDestinationPipeline' - ); const { after_key: nextAfterKey, latestTransactionTime: latestSampleTransactionTime, @@ -83,11 +79,7 @@ async function indexLatestConnections( ) ); return traceConnections.flatMap( - mapTraceToBulkServiceConnection({ - serviceConnsDestinationIndex, - serviceConnsDestinationPipeline, - servicesInTrace - }) + mapTraceToBulkServiceConnection(apmIndices, servicesInTrace) ); }); await bulkClient({ @@ -96,7 +88,7 @@ async function indexLatestConnections( .join('\n') }); return await indexLatestConnections( - config, + apmIndices, searchClient, bulkClient, startTimeInterval, @@ -107,7 +99,6 @@ async function indexLatestConnections( export async function runServiceMapTask( server: Server, - config: ReturnType, startTimeInterval?: string | number ) { const callCluster = server.plugins.elasticsearch.getCluster('data') @@ -116,8 +107,13 @@ export async function runServiceMapTask( const bulkClient = (params: BulkIndexDocumentsParams) => callCluster('bulk', params); + const apmPlugin = server.newPlatform.setup.plugins.apm as APMPluginContract; + const apmIndices = await apmPlugin.getApmIndices( + getInternalSavedObjectsClient(server) + ); + return await indexLatestConnections( - config, + apmIndices, searchClient, bulkClient, startTimeInterval diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/setup_required_scripts.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/service-connection-es-scripts.ts similarity index 98% rename from x-pack/legacy/plugins/apm/server/lib/service_map/setup_required_scripts.ts rename to x-pack/legacy/plugins/apm/server/lib/service_map/service-connection-es-scripts.ts index f495ffe286e59..94dc3eeacac00 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/setup_required_scripts.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/service-connection-es-scripts.ts @@ -32,10 +32,6 @@ export const mapServiceConnsScript = { s.destination = doc['destination.address'].value; } - if (!doc['_index'].empty) { // TODO is this ever empty? - s._index = doc['_index'].value; - } - if (!doc['span.type'].empty) { s.span_type = doc['span.type'].value; } diff --git a/x-pack/legacy/plugins/apm/server/lib/settings/apm_indices/get_apm_indices.ts b/x-pack/legacy/plugins/apm/server/lib/settings/apm_indices/get_apm_indices.ts index e7b2330b472d8..fb528e0599b2c 100644 --- a/x-pack/legacy/plugins/apm/server/lib/settings/apm_indices/get_apm_indices.ts +++ b/x-pack/legacy/plugins/apm/server/lib/settings/apm_indices/get_apm_indices.ts @@ -23,6 +23,7 @@ export interface ApmIndicesConfig { 'apm_oss.transactionIndices': string; 'apm_oss.metricsIndices': string; apmAgentConfigurationIndex: string; + apmServiceConnectionsIndex: string; } export type ApmIndicesName = keyof ApmIndicesConfig; @@ -50,7 +51,8 @@ export function getApmIndicesConfig(config: APMConfig): ApmIndicesConfig { 'apm_oss.transactionIndices': config['apm_oss.transactionIndices'], 'apm_oss.metricsIndices': config['apm_oss.metricsIndices'], // system indices, not configurable - apmAgentConfigurationIndex: '.apm-agent-configuration' + apmAgentConfigurationIndex: '.apm-agent-configuration', + apmServiceConnectionsIndex: 'apm-service-connections' }; } diff --git a/x-pack/legacy/plugins/apm/server/lib/transaction_groups/fetcher.test.ts b/x-pack/legacy/plugins/apm/server/lib/transaction_groups/fetcher.test.ts index 4121ff74bfacc..8f8f2c6496619 100644 --- a/x-pack/legacy/plugins/apm/server/lib/transaction_groups/fetcher.test.ts +++ b/x-pack/legacy/plugins/apm/server/lib/transaction_groups/fetcher.test.ts @@ -28,7 +28,8 @@ function getSetup() { 'apm_oss.spanIndices': 'myIndex', 'apm_oss.transactionIndices': 'myIndex', 'apm_oss.metricsIndices': 'myIndex', - apmAgentConfigurationIndex: 'myIndex' + apmAgentConfigurationIndex: 'myIndex', + apmServiceConnectionsIndex: 'myIndex' }, dynamicIndexPattern: null as any }; diff --git a/x-pack/legacy/plugins/apm/server/lib/transactions/breakdown/index.test.ts b/x-pack/legacy/plugins/apm/server/lib/transactions/breakdown/index.test.ts index f49c1e022a070..94e3d65a8527d 100644 --- a/x-pack/legacy/plugins/apm/server/lib/transactions/breakdown/index.test.ts +++ b/x-pack/legacy/plugins/apm/server/lib/transactions/breakdown/index.test.ts @@ -17,7 +17,8 @@ const mockIndices = { 'apm_oss.spanIndices': 'myIndex', 'apm_oss.transactionIndices': 'myIndex', 'apm_oss.metricsIndices': 'myIndex', - apmAgentConfigurationIndex: 'myIndex' + apmAgentConfigurationIndex: 'myIndex', + apmServiceConnectionsIndex: 'myIndex' }; function getMockSetup(esResponse: any) { diff --git a/x-pack/legacy/plugins/apm/server/lib/transactions/charts/get_anomaly_data/index.test.ts b/x-pack/legacy/plugins/apm/server/lib/transactions/charts/get_anomaly_data/index.test.ts index 2a56f744f2f45..5416734f74873 100644 --- a/x-pack/legacy/plugins/apm/server/lib/transactions/charts/get_anomaly_data/index.test.ts +++ b/x-pack/legacy/plugins/apm/server/lib/transactions/charts/get_anomaly_data/index.test.ts @@ -42,7 +42,8 @@ describe('getAnomalySeries', () => { 'apm_oss.spanIndices': 'myIndex', 'apm_oss.transactionIndices': 'myIndex', 'apm_oss.metricsIndices': 'myIndex', - apmAgentConfigurationIndex: 'myIndex' + apmAgentConfigurationIndex: 'myIndex', + apmServiceConnectionsIndex: 'myIndex' }, dynamicIndexPattern: null as any } diff --git a/x-pack/legacy/plugins/apm/server/lib/transactions/charts/get_timeseries_data/fetcher.test.ts b/x-pack/legacy/plugins/apm/server/lib/transactions/charts/get_timeseries_data/fetcher.test.ts index 676ad4ded6b69..ad2342695f0c8 100644 --- a/x-pack/legacy/plugins/apm/server/lib/transactions/charts/get_timeseries_data/fetcher.test.ts +++ b/x-pack/legacy/plugins/apm/server/lib/transactions/charts/get_timeseries_data/fetcher.test.ts @@ -41,7 +41,8 @@ describe('timeseriesFetcher', () => { 'apm_oss.spanIndices': 'myIndex', 'apm_oss.transactionIndices': 'myIndex', 'apm_oss.metricsIndices': 'myIndex', - apmAgentConfigurationIndex: 'myIndex' + apmAgentConfigurationIndex: 'myIndex', + apmServiceConnectionsIndex: 'myIndex' }, dynamicIndexPattern: null as any } diff --git a/x-pack/plugins/apm/server/index.ts b/x-pack/plugins/apm/server/index.ts index 9c111626dd5a3..b0e10d245e0b9 100644 --- a/x-pack/plugins/apm/server/index.ts +++ b/x-pack/plugins/apm/server/index.ts @@ -16,9 +16,6 @@ export const config = { }, schema: schema.object({ serviceMapEnabled: schema.boolean({ defaultValue: false }), - serviceMapIndexPattern: schema.string({ defaultValue: 'apm-*' }), - serviceMapDestinationIndex: schema.maybe(schema.string()), - serviceMapDestinationPipeline: schema.maybe(schema.string()), autocreateApmIndexPattern: schema.boolean({ defaultValue: true }), ui: schema.object({ enabled: schema.boolean({ defaultValue: true }), @@ -44,9 +41,6 @@ export function mergeConfigs(apmOssConfig: APMOSSConfig, apmConfig: APMXPackConf 'xpack.apm.ui.maxTraceItems': apmConfig.ui.maxTraceItems, 'xpack.apm.ui.transactionGroupBucketSize': apmConfig.ui.transactionGroupBucketSize, 'xpack.apm.autocreateApmIndexPattern': apmConfig.autocreateApmIndexPattern, - 'xpack.apm.serviceMapIndexPattern': apmConfig.serviceMapIndexPattern, - 'xpack.apm.serviceMapDestinationIndex': apmConfig.serviceMapDestinationIndex, - 'xpack.apm.serviceMapDestinationPipeline': apmConfig.serviceMapDestinationPipeline, }; } From 14a231fdc0b13f90c305a4b75b2b1f8b5f6adc63 Mon Sep 17 00:00:00 2001 From: Oliver Gupte Date: Tue, 10 Dec 2019 17:02:11 -0800 Subject: [PATCH 09/15] generalize getEsClient to leverage it in run_service_map_task --- .../apm/server/lib/helpers/es_client.ts | 92 ++++++------------- .../apm/server/lib/helpers/setup_request.ts | 33 ++++++- .../get_next_transaction_samples.ts | 8 +- .../service_map/get_service_connections.ts | 8 +- .../lib/service_map/run_service_map_task.ts | 34 +++---- 5 files changed, 80 insertions(+), 95 deletions(-) diff --git a/x-pack/legacy/plugins/apm/server/lib/helpers/es_client.ts b/x-pack/legacy/plugins/apm/server/lib/helpers/es_client.ts index 406ca064ae1f9..d3af158687391 100644 --- a/x-pack/legacy/plugins/apm/server/lib/helpers/es_client.ts +++ b/x-pack/legacy/plugins/apm/server/lib/helpers/es_client.ts @@ -4,25 +4,23 @@ * you may not use this file except in compliance with the Elastic License. */ -/* eslint-disable no-console */ import { SearchParams, IndexDocumentParams, IndicesDeleteParams, - IndicesCreateParams + IndicesCreateParams, + BulkIndexDocumentsParams } from 'elasticsearch'; import { merge } from 'lodash'; import { cloneDeep, isString } from 'lodash'; -import { KibanaRequest } from 'src/core/server'; -import { CallCluster } from 'src/legacy/core_plugins/elasticsearch'; +import { IUiSettingsClient, IScopedClusterClient } from 'src/core/server'; import { OBSERVER_VERSION_MAJOR } from '../../../common/elasticsearch_fieldnames'; import { ESSearchResponse, ESSearchRequest } from '../../../typings/elasticsearch'; -import { APMRequestHandlerContext } from '../../routes/typings'; import { pickKeys } from '../../../public/utils/pickKeys'; -import { getApmIndices } from '../settings/apm_indices/get_apm_indices'; +import { ApmIndicesConfig } from '../settings/apm_indices/get_apm_indices'; // `type` was deprecated in 7.0 export type APMIndexDocumentParams = Omit, 'type'>; @@ -73,23 +71,17 @@ function addFilterForLegacyData( // add additional params for search (aka: read) requests async function getParamsForSearchRequest( - context: APMRequestHandlerContext, + apmIndices: ApmIndicesConfig, + uiSettingsClient: IUiSettingsClient, params: ESSearchRequest, apmOptions?: APMOptions ) { - const { uiSettings } = context.core; - const [indices, includeFrozen] = await Promise.all([ - getApmIndices({ - savedObjectsClient: context.core.savedObjects.client, - config: context.config - }), - uiSettings.client.get('search:includeFrozen') - ]); + const includeFrozen = await uiSettingsClient.get('search:includeFrozen'); // Get indices for legacy data filter (only those which apply) - const apmIndices = Object.values( + const legacyApmIndices = Object.values( pickKeys( - indices, + apmIndices, 'apm_oss.sourcemapIndices', 'apm_oss.errorIndices', 'apm_oss.onboardingIndices', @@ -99,7 +91,7 @@ async function getParamsForSearchRequest( ) ); return { - ...addFilterForLegacyData(apmIndices, params, apmOptions), // filter out pre-7.0 data + ...addFilterForLegacyData(legacyApmIndices, params, apmOptions), // filter out pre-7.0 data ignore_throttled: !includeFrozen // whether to query frozen indices or not }; } @@ -108,26 +100,16 @@ interface APMOptions { includeLegacyData: boolean; } -interface ClientCreateOptions { - clientAsInternalUser?: boolean; -} - export type ESClient = ReturnType; export function getESClient( - context: APMRequestHandlerContext, - request: KibanaRequest, - { clientAsInternalUser = false }: ClientCreateOptions = {} + apmIndices: ApmIndicesConfig, + uiSettingsClient: IUiSettingsClient, + esClient: + | IScopedClusterClient['callAsCurrentUser'] + | IScopedClusterClient['callAsInternalUser'], + onRequest: (endpoint: string, params: Record) => void = () => {} ) { - const { - callAsCurrentUser, - callAsInternalUser - } = context.core.elasticsearch.dataClient; - - const callMethod = clientAsInternalUser - ? callAsInternalUser - : callAsCurrentUser; - return { search: async < TDocument = unknown, @@ -137,47 +119,33 @@ export function getESClient( apmOptions?: APMOptions ): Promise> => { const nextParams = await getParamsForSearchRequest( - context, + apmIndices, + uiSettingsClient, params, apmOptions ); - if (context.params.query._debug) { - console.log(`--DEBUG ES QUERY--`); - console.log( - `${request.url.pathname} ${JSON.stringify(context.params.query)}` - ); - console.log(`GET ${nextParams.index}/_search`); - console.log(JSON.stringify(nextParams.body, null, 2)); - } + onRequest('search', nextParams); - return (callMethod('search', nextParams) as unknown) as Promise< + return (esClient('search', nextParams) as unknown) as Promise< ESSearchResponse >; }, index: (params: APMIndexDocumentParams) => { - return callMethod('index', params); + onRequest('index', params); + return esClient('index', params); }, delete: (params: IndicesDeleteParams) => { - return callMethod('delete', params); + onRequest('delete', params); + return esClient('delete', params); }, indicesCreate: (params: IndicesCreateParams) => { - return callMethod('indices.create', params); + onRequest('indices.create', params); + return esClient('indices.create', params); + }, + bulk: (params: BulkIndexDocumentsParams) => { + onRequest('bulk', params); + return esClient('bulk', params); } }; } - -export type SearchClient = ReturnType; -export function getSearchClient(callCluster: CallCluster) { - async function search< - TDocument = unknown, - TSearchRequest extends ESSearchRequest = {} - >( - params: TSearchRequest - ): Promise> { - return (callCluster('search', params) as unknown) as Promise< - ESSearchResponse - >; - } - return search; -} diff --git a/x-pack/legacy/plugins/apm/server/lib/helpers/setup_request.ts b/x-pack/legacy/plugins/apm/server/lib/helpers/setup_request.ts index 56c9255844009..9672c2e193ecc 100644 --- a/x-pack/legacy/plugins/apm/server/lib/helpers/setup_request.ts +++ b/x-pack/legacy/plugins/apm/server/lib/helpers/setup_request.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +/* eslint-disable no-console */ import moment from 'moment'; import { KibanaRequest } from 'src/core/server'; import { IIndexPattern } from 'src/plugins/data/common'; @@ -86,12 +87,36 @@ export async function setupRequest( const uiFiltersES = decodeUiFilters(dynamicIndexPattern, query.uiFilters); + const { + callAsCurrentUser, + callAsInternalUser + } = context.core.elasticsearch.dataClient; + + const onRequest = (endpoint: string, params: Record) => { + if (endpoint === 'search' && query._debug) { + console.log(`--DEBUG ES QUERY--`); + console.log( + `${request.url.pathname} ${JSON.stringify(context.params.query)}` + ); + console.log(`GET ${params.index}/_search`); + console.log(JSON.stringify(params.body, null, 2)); + } + }; + const coreSetupRequest = { indices, - client: getESClient(context, request, { clientAsInternalUser: false }), - internalClient: getESClient(context, request, { - clientAsInternalUser: true - }), + client: getESClient( + indices, + context.core.uiSettings.client, + callAsCurrentUser, + onRequest + ), + internalClient: getESClient( + indices, + context.core.uiSettings.client, + callAsInternalUser, + onRequest + ), config, dynamicIndexPattern }; diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/get_next_transaction_samples.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/get_next_transaction_samples.ts index 4950f37c399b1..b958279fc6b84 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/get_next_transaction_samples.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/get_next_transaction_samples.ts @@ -5,7 +5,7 @@ */ import { uniq } from 'lodash'; -import { SearchClient } from '../helpers/es_client'; +import { ESClient } from '../helpers/es_client'; import { Transaction } from '../../../typings/es_schemas/ui/Transaction'; import { SERVICE_NAME, @@ -22,12 +22,12 @@ export async function getNextTransactionSamples({ targetApmIndices, startTimeInterval, afterKey, - searchClient + esClient }: { targetApmIndices: string[]; startTimeInterval: string | number; afterKey?: object; - searchClient: SearchClient; + esClient: ESClient; }) { const params = { index: targetApmIndices, @@ -94,7 +94,7 @@ export async function getNextTransactionSamples({ } }; - const transactionsResponse = await searchClient(params); + const transactionsResponse = await esClient.search(params); const externalConnections = transactionsResponse.aggregations?.externalConnections; const buckets = externalConnections?.buckets ?? []; diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_connections.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_connections.ts index 124900bed9b53..43ede44598733 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_connections.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_connections.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { SearchClient } from '../helpers/es_client'; +import { ESClient } from '../helpers/es_client'; import { mapServiceConnsScript, combineServiceConnsScript, @@ -19,11 +19,11 @@ import { export async function getServiceConnections({ targetApmIndices, traceIds, - searchClient + esClient }: { targetApmIndices: string[]; traceIds: string[]; - searchClient: SearchClient; + esClient: ESClient; }) { const traceIdFilters = traceIds.map(traceId => ({ term: { [TRACE_ID]: traceId } @@ -63,7 +63,7 @@ export async function getServiceConnections({ } } }; - const serviceConnectionsResponse = await searchClient(params); + const serviceConnectionsResponse = await esClient.search(params); const traceConnectionsBuckets = serviceConnectionsResponse.aggregations?.traces.buckets ?? []; return traceConnectionsBuckets; diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/run_service_map_task.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/run_service_map_task.ts index 3c2013085b274..290331d10b562 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/run_service_map_task.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/run_service_map_task.ts @@ -6,9 +6,8 @@ import { Server } from 'hapi'; import { uniq } from 'lodash'; -import { BulkIndexDocumentsParams } from 'elasticsearch'; import { APMPluginContract } from '../../../../../../plugins/apm/server/plugin'; -import { getSearchClient, SearchClient } from '../helpers/es_client'; +import { getESClient, ESClient } from '../helpers/es_client'; import { Span } from '../../../typings/es_schemas/ui/Span'; import { getNextTransactionSamples } from './get_next_transaction_samples'; import { getServiceConnections } from './get_service_connections'; @@ -37,8 +36,7 @@ export interface TraceConnection { async function indexLatestConnections( apmIndices: ApmIndicesConfig, - searchClient: SearchClient, - bulkClient: (params: BulkIndexDocumentsParams) => Promise, + esClient: ESClient, startTimeInterval?: string | number, latestTransactionTime = 0, afterKey?: object @@ -55,7 +53,7 @@ async function indexLatestConnections( targetApmIndices, startTimeInterval: startTimeInterval || 'now-1h', afterKey, - searchClient + esClient }); if (traceIds.length === 0) { return { latestTransactionTime }; @@ -67,7 +65,7 @@ async function indexLatestConnections( const traceConnectionsBuckets = await getServiceConnections({ targetApmIndices, traceIds, - searchClient + esClient }); const bulkIndexConnectionDocs = traceConnectionsBuckets.flatMap(bucket => { const traceConnections = bucket.connections.value as TraceConnection[]; @@ -82,15 +80,14 @@ async function indexLatestConnections( mapTraceToBulkServiceConnection(apmIndices, servicesInTrace) ); }); - await bulkClient({ + await esClient.bulk({ body: bulkIndexConnectionDocs .map(bulkObject => JSON.stringify(bulkObject)) .join('\n') }); return await indexLatestConnections( apmIndices, - searchClient, - bulkClient, + esClient, startTimeInterval, nextLatestTransactionTime, nextAfterKey @@ -103,19 +100,14 @@ export async function runServiceMapTask( ) { const callCluster = server.plugins.elasticsearch.getCluster('data') .callWithInternalUser; - const searchClient = getSearchClient(callCluster); - const bulkClient = (params: BulkIndexDocumentsParams) => - callCluster('bulk', params); - const apmPlugin = server.newPlatform.setup.plugins.apm as APMPluginContract; - const apmIndices = await apmPlugin.getApmIndices( - getInternalSavedObjectsClient(server) - ); - - return await indexLatestConnections( + const savedObjectsClient = getInternalSavedObjectsClient(server); + const apmIndices = await apmPlugin.getApmIndices(savedObjectsClient); + const esClient: ESClient = getESClient( apmIndices, - searchClient, - bulkClient, - startTimeInterval + server.uiSettingsServiceFactory({ savedObjectsClient }), + callCluster ); + + return await indexLatestConnections(apmIndices, esClient, startTimeInterval); } From 4e2ce54969178057303414b133edb45b1cfd424d Mon Sep 17 00:00:00 2001 From: Oliver Gupte Date: Tue, 10 Dec 2019 17:06:59 -0800 Subject: [PATCH 10/15] remove development helper ingest pipeline which extracted destination.address from span names --- .../get_next_transaction_samples.ts | 3 +- .../service_map/initialize_service_maps.ts | 16 -- .../service-connection-es-scripts.ts | 148 ------------------ 3 files changed, 1 insertion(+), 166 deletions(-) diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/get_next_transaction_samples.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/get_next_transaction_samples.ts index b958279fc6b84..ba72d945af192 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/get_next_transaction_samples.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/get_next_transaction_samples.ts @@ -65,8 +65,7 @@ export async function getNextTransactionSamples({ } } ], - // TODO: needs to be balanced with the 20 below - size: 200, + size: 20, // determines size of bulk index requests to apm-service-connections after: afterKey }, aggs: { diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/initialize_service_maps.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/initialize_service_maps.ts index 4e0dde778974e..820b20b9f0d66 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/initialize_service_maps.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/initialize_service_maps.ts @@ -8,7 +8,6 @@ import { Server } from 'hapi'; // @ts-ignore import { TaskManager, RunContext } from '../legacy/plugins/task_manager'; import { runServiceMapTask } from './run_service_map_task'; -import { setupIngestPipeline } from './service-connection-es-scripts'; import { SERVICE_MAP_TASK_TYPE, SERVICE_MAP_TASK_ID @@ -16,21 +15,6 @@ import { import { createServiceConnectionsIndex } from './create_service_connections_index'; export async function initializeServiceMaps(server: Server) { - // TODO remove setupIngestPipeline when agents set destination.address (elastic/apm#115) - setupIngestPipeline(server) - .then(() => { - server.log( - ['info', 'plugins', 'apm'], - `Created ingest pipeline to extract destination.address from span names.` - ); - }) - .catch(error => { - server.log( - ['error', 'plugins', 'apm'], - `Unable to setup the ingest pipeline to extract destination.address from span names.\n${error.stack}` - ); - }); - await createServiceConnectionsIndex(server); const taskManager = server.plugins.task_manager; diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/service-connection-es-scripts.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/service-connection-es-scripts.ts index 94dc3eeacac00..f42c48da38315 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/service-connection-es-scripts.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/service-connection-es-scripts.ts @@ -4,12 +4,6 @@ * you may not use this file except in compliance with the Elastic License. */ -import { Legacy } from 'kibana'; -import { CallCluster } from 'src/legacy/core_plugins/elasticsearch'; - -const EXTRACT_DESTINATION_INGEST_PIPELINE_ID = 'apm_extract_destination'; -const APM_INGEST_PIPELINE_ID = 'apm'; - export const mapServiceConnsScript = { lang: 'painless', source: ` @@ -115,145 +109,3 @@ export const combineServiceConnsScript = { lang: 'painless', source: `return state.spans` }; - -async function putIngestPipelineExtractDestination(callCluster: CallCluster) { - return await callCluster('ingest.putPipeline', { - id: EXTRACT_DESTINATION_INGEST_PIPELINE_ID, - body: { - description: 'sets destination on ext spans based on their name', - processors: [ - { - set: { - if: "ctx.span != null && ctx.span.type == 'ext'", - field: 'span.type', - value: 'external' - } - }, - { - script: ` - if(ctx['span'] != null) { - if (ctx['span']['type'] == 'external') { - def spanName = ctx['span']['name']; - if (spanName.indexOf('/') > -1) { - spanName = spanName.substring(0, spanName.indexOf('/')); - } - - if (spanName.indexOf(' ') > -1) { - spanName = spanName.substring(spanName.indexOf(' ')+1, spanName.length()); - } - ctx['destination.address']=spanName; - } - - if (ctx['span']['type'] == 'resource') { - def spanName = ctx['span']['name']; - - if (spanName.indexOf('://') > -1) { - spanName = spanName.substring(spanName.indexOf('://')+3, spanName.length()); - } - if (spanName.indexOf('/') > -1) { - spanName = spanName.substring(0, spanName.indexOf('/')); - } - - ctx['destination.address']=spanName; - } - - if (ctx['span']['type'] == 'db') { - def dest = ctx['span']['subtype']; - ctx['destination.address']=dest; - } - - if (ctx['span']['type'] == 'cache') { - def dest = ctx['span']['subtype']; - ctx['destination.address']=dest; - } - } - ` - } - ] - } - }); -} - -interface ApmIngestPipeline { - [APM_INGEST_PIPELINE_ID]: { - description: string; - processors: Array<{ - pipeline: { - name: string; - }; - }>; - }; -} - -async function getIngestPipelineApm( - callCluster: CallCluster -): Promise { - return await callCluster('ingest.getPipeline', { - id: APM_INGEST_PIPELINE_ID - }); -} - -async function putIngestPipelineApm( - callCluster: CallCluster, - processors: ApmIngestPipeline[typeof APM_INGEST_PIPELINE_ID]['processors'] -) { - return await callCluster('ingest.putPipeline', { - id: APM_INGEST_PIPELINE_ID, - body: { - description: 'Default enrichment for APM events', - processors - } - }); -} - -async function applyExtractDestinationToApm(callCluster: CallCluster) { - let apmIngestPipeline: ApmIngestPipeline; - try { - // get current apm ingest pipeline - apmIngestPipeline = await getIngestPipelineApm(callCluster); - } catch (error) { - if (error.statusCode !== 404) { - throw error; - } - // create apm ingest pipeline if it doesn't exist - return await putIngestPipelineApm(callCluster, [ - { - pipeline: { - name: EXTRACT_DESTINATION_INGEST_PIPELINE_ID - } - } - ]); - } - - const { - apm: { processors } - } = apmIngestPipeline; - - // check if 'extract destination' processor is already applied - if ( - processors.find( - ({ pipeline: { name } }) => - name === EXTRACT_DESTINATION_INGEST_PIPELINE_ID - ) - ) { - return apmIngestPipeline; - } - - // append 'extract destination' to existing processors - return await putIngestPipelineApm(callCluster, [ - ...processors, - { - pipeline: { - name: EXTRACT_DESTINATION_INGEST_PIPELINE_ID - } - } - ]); -} - -// TODO remove this when agents set destination.address (elastic/apm#115) -export async function setupIngestPipeline(server: Legacy.Server) { - const callCluster = server.plugins.elasticsearch.getCluster('admin') - .callWithInternalUser; - await putIngestPipelineExtractDestination(callCluster); - return await applyExtractDestinationToApm(callCluster); -} From 945d8b04c5008b0962aa340ce727bea71ff72c60 Mon Sep 17 00:00:00 2001 From: Oliver Gupte Date: Tue, 10 Dec 2019 17:07:45 -0800 Subject: [PATCH 11/15] fix tests to snapshot ISO 8601 timestamps --- .../__snapshots__/elasticsearch_fieldnames.test.ts.snap | 6 +++--- .../plugins/apm/common/elasticsearch_fieldnames.test.ts | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/legacy/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap b/x-pack/legacy/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap index 72ee0abb5b394..741bb8a624895 100644 --- a/x-pack/legacy/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap +++ b/x-pack/legacy/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap @@ -96,7 +96,7 @@ exports[`Error SPAN_SUBTYPE 1`] = `undefined`; exports[`Error SPAN_TYPE 1`] = `undefined`; -exports[`Error TIMESTAMP 1`] = `"Wed Dec 31 1969 16:00:01 GMT-0800 (Pacific Standard Time)"`; +exports[`Error TIMESTAMP 1`] = `"1970-01-01T00:00:01.337Z"`; exports[`Error TRACE_ID 1`] = `"trace id"`; @@ -218,7 +218,7 @@ exports[`Span SPAN_SUBTYPE 1`] = `"my subtype"`; exports[`Span SPAN_TYPE 1`] = `"span type"`; -exports[`Span TIMESTAMP 1`] = `"Wed Dec 31 1969 16:00:01 GMT-0800 (Pacific Standard Time)"`; +exports[`Span TIMESTAMP 1`] = `"1970-01-01T00:00:01.337Z"`; exports[`Span TRACE_ID 1`] = `"trace id"`; @@ -340,7 +340,7 @@ exports[`Transaction SPAN_SUBTYPE 1`] = `undefined`; exports[`Transaction SPAN_TYPE 1`] = `undefined`; -exports[`Transaction TIMESTAMP 1`] = `"Wed Dec 31 1969 16:00:01 GMT-0800 (Pacific Standard Time)"`; +exports[`Transaction TIMESTAMP 1`] = `"1970-01-01T00:00:01.337Z"`; exports[`Transaction TRACE_ID 1`] = `"trace id"`; diff --git a/x-pack/legacy/plugins/apm/common/elasticsearch_fieldnames.test.ts b/x-pack/legacy/plugins/apm/common/elasticsearch_fieldnames.test.ts index 46b60542727f5..b4af9e97ec803 100644 --- a/x-pack/legacy/plugins/apm/common/elasticsearch_fieldnames.test.ts +++ b/x-pack/legacy/plugins/apm/common/elasticsearch_fieldnames.test.ts @@ -13,7 +13,7 @@ import * as fieldnames from './elasticsearch_fieldnames'; describe('Transaction', () => { const transaction: AllowUnknownProperties = { - '@timestamp': new Date(1337).toString(), + '@timestamp': new Date(1337).toISOString(), '@metadata': 'whatever', observer: 'whatever', agent: { @@ -61,7 +61,7 @@ describe('Transaction', () => { describe('Span', () => { const span: AllowUnknownProperties = { - '@timestamp': new Date(1337).toString(), + '@timestamp': new Date(1337).toISOString(), '@metadata': 'whatever', observer: 'whatever', agent: { @@ -125,7 +125,7 @@ describe('Error', () => { id: 'error id', grouping_key: 'grouping key' }, - '@timestamp': new Date(1337).toString(), + '@timestamp': new Date(1337).toISOString(), host: { hostname: 'my hostname' }, From 2121bcd577f9d247afd67f4046bc87512760187f Mon Sep 17 00:00:00 2001 From: Oliver Gupte Date: Sat, 14 Dec 2019 04:48:24 -0800 Subject: [PATCH 12/15] - code improvements to es scripts - conform to ESC fields - performance improvments to get service connections - add checks for empty bulk requests --- .../service_map/get_service_connections.ts | 30 ++-- .../service_map/initialize_service_maps.ts | 11 +- .../map_trace_to_bulk_service_connection.ts | 25 ++-- .../lib/service_map/run_service_map_task.ts | 47 ++++--- .../service-connection-es-scripts.ts | 133 ++++++++++-------- 5 files changed, 142 insertions(+), 104 deletions(-) diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_connections.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_connections.ts index 43ede44598733..f77a042020e36 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_connections.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_connections.ts @@ -6,14 +6,14 @@ import { ESClient } from '../helpers/es_client'; import { + initServiceConnsScript, mapServiceConnsScript, combineServiceConnsScript, reduceServiceConnsScript } from './service-connection-es-scripts'; import { - SPAN_ID, TRACE_ID, - TRANSACTION_TYPE + PROCESSOR_EVENT } from '../../../common/elasticsearch_fieldnames'; export async function getServiceConnections({ @@ -25,21 +25,24 @@ export async function getServiceConnections({ traceIds: string[]; esClient: ESClient; }) { - const traceIdFilters = traceIds.map(traceId => ({ - term: { [TRACE_ID]: traceId } - })); const params = { index: targetApmIndices, body: { size: 0, query: { bool: { - should: [ - { exists: { field: SPAN_ID } }, - { exists: { field: TRANSACTION_TYPE } }, - ...traceIdFilters - ], - minimum_should_match: 2 + filter: [ + { + terms: { + [PROCESSOR_EVENT]: ['transaction', 'span'] + } + }, + { + terms: { + [TRACE_ID]: traceIds + } + } + ] } }, aggs: { @@ -47,12 +50,13 @@ export async function getServiceConnections({ terms: { field: TRACE_ID, order: { _key: 'asc' as const }, - size: traceIds.length + size: traceIds.length, + execution_hint: 'map' }, aggs: { connections: { scripted_metric: { - init_script: 'state.spans = new HashMap();', + init_script: initServiceConnsScript, map_script: mapServiceConnsScript, combine_script: combineServiceConnsScript, reduce_script: reduceServiceConnsScript diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/initialize_service_maps.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/initialize_service_maps.ts index 820b20b9f0d66..b3ddb00a6e931 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/initialize_service_maps.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/initialize_service_maps.ts @@ -14,6 +14,11 @@ import { } from '../../../common/service_map_constants'; import { createServiceConnectionsIndex } from './create_service_connections_index'; +function isLessThan1Hour(unixTimestamp = 0) { + const hourMilliseconds = 60 * 60 * 1000; + return Date.now() - unixTimestamp < hourMilliseconds; +} + export async function initializeServiceMaps(server: Server) { await createServiceConnectionsIndex(server); @@ -29,12 +34,12 @@ export async function initializeServiceMaps(server: Server) { return { async run() { const { state } = taskInstance; - const { latestTransactionTime } = await runServiceMapTask( server, - state.latestTransactionTime + isLessThan1Hour(state.latestTransactionTime) + ? state.latestTransactionTime + : 'now-1h' ); - return { state: { latestTransactionTime } }; } }; diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/map_trace_to_bulk_service_connection.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/map_trace_to_bulk_service_connection.ts index 94bd437b9d05d..0a5dd837d0397 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/map_trace_to_bulk_service_connection.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/map_trace_to_bulk_service_connection.ts @@ -5,7 +5,14 @@ */ import { TraceConnection } from './run_service_map_task'; -import { TIMESTAMP } from '../../../common/elasticsearch_fieldnames'; +import { + TIMESTAMP, + SERVICE_NAME, + SERVICE_ENVIRONMENT, + SPAN_TYPE, + SPAN_SUBTYPE, + DESTINATION_ADDRESS +} from '../../../common/elasticsearch_fieldnames'; import { ApmIndicesConfig } from '../settings/apm_indices/get_apm_indices'; export function mapTraceToBulkServiceConnection( @@ -20,24 +27,24 @@ export function mapTraceToBulkServiceConnection( }; const source = { - [TIMESTAMP]: traceConnection.caller.timestamp, + [TIMESTAMP]: traceConnection.caller[TIMESTAMP], service: { - name: traceConnection.caller.service_name, - environment: traceConnection.caller.environment + name: traceConnection.caller[SERVICE_NAME], + environment: traceConnection.caller[SERVICE_ENVIRONMENT] }, callee: { - name: traceConnection.callee?.service_name, - environment: traceConnection.callee?.environment + name: traceConnection.callee?.[SERVICE_NAME], + environment: traceConnection.callee?.[SERVICE_ENVIRONMENT] }, connection: { upstream: { list: traceConnection.upstream }, in_trace: servicesInTrace, - type: traceConnection.caller.span_type, - subtype: traceConnection.caller.span_subtype + type: traceConnection.caller[SPAN_TYPE], + subtype: traceConnection.caller[SPAN_SUBTYPE] }, - destination: { address: traceConnection.caller.destination } + destination: { address: traceConnection.caller[DESTINATION_ADDRESS] } }; return [indexAction, source]; }; diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/run_service_map_task.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/run_service_map_task.ts index 290331d10b562..d55238ab6e3e4 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/run_service_map_task.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/run_service_map_task.ts @@ -8,24 +8,34 @@ import { Server } from 'hapi'; import { uniq } from 'lodash'; import { APMPluginContract } from '../../../../../../plugins/apm/server/plugin'; import { getESClient, ESClient } from '../helpers/es_client'; -import { Span } from '../../../typings/es_schemas/ui/Span'; import { getNextTransactionSamples } from './get_next_transaction_samples'; import { getServiceConnections } from './get_service_connections'; import { mapTraceToBulkServiceConnection } from './map_trace_to_bulk_service_connection'; import { ENVIRONMENT_NOT_DEFINED } from '../../../common/environment_filter_values'; import { getInternalSavedObjectsClient } from '../helpers/saved_objects_client'; import { ApmIndicesConfig } from '../settings/apm_indices/get_apm_indices'; +import { + SPAN_TYPE, + SPAN_SUBTYPE, + SPAN_ID, + TIMESTAMP, + TRANSACTION_ID, + PARENT_ID, + SERVICE_NAME, + SERVICE_ENVIRONMENT, + DESTINATION_ADDRESS +} from '../../../common/elasticsearch_fieldnames'; interface MappedDocument { - transaction?: boolean; - id: string; // span or transaction id - parent?: string; // parent.id - environment?: string; // service.environment - destination?: string; // destination.address - span_type?: Span['span']['type']; - span_subtype?: Span['span']['subtype']; - timestamp: string; - service_name: Span['service']['name']; + [TIMESTAMP]: string; + [TRANSACTION_ID]: string; + [PARENT_ID]?: string; + [SPAN_ID]?: string; + [SPAN_TYPE]: string; + [SPAN_SUBTYPE]?: string; + [SERVICE_NAME]: string; + [SERVICE_ENVIRONMENT]?: string; + [DESTINATION_ADDRESS]: string; } export interface TraceConnection { @@ -72,19 +82,22 @@ async function indexLatestConnections( const servicesInTrace = uniq( traceConnections.map( serviceConnection => - `${serviceConnection.caller.service_name}/${serviceConnection.caller - .environment || ENVIRONMENT_NOT_DEFINED}` + `${serviceConnection.caller[SERVICE_NAME]}/${serviceConnection.caller[ + SERVICE_ENVIRONMENT + ] || ENVIRONMENT_NOT_DEFINED}` ) ); return traceConnections.flatMap( mapTraceToBulkServiceConnection(apmIndices, servicesInTrace) ); }); - await esClient.bulk({ - body: bulkIndexConnectionDocs - .map(bulkObject => JSON.stringify(bulkObject)) - .join('\n') - }); + if (bulkIndexConnectionDocs.length > 0) { + await esClient.bulk({ + body: bulkIndexConnectionDocs + .map(bulkObject => JSON.stringify(bulkObject)) + .join('\n') + }); + } return await indexLatestConnections( apmIndices, esClient, diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/service-connection-es-scripts.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/service-connection-es-scripts.ts index f42c48da38315..5da7b302aada3 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/service-connection-es-scripts.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/service-connection-es-scripts.ts @@ -4,108 +4,117 @@ * you may not use this file except in compliance with the Elastic License. */ -export const mapServiceConnsScript = { +export const initServiceConnsScript = { lang: 'painless', source: ` - def s = new HashMap(); - - if (!doc['span.id'].empty) { - s.id = doc['span.id'].value - } else { - s.id = doc['transaction.id'].value; - s.transaction = true; - } - if (!doc['parent.id'].empty) { - s.parent = doc['parent.id'].value; - } - if (!doc['service.environment'].empty) { - s.environment = doc['service.environment'].value; - } - - if (!doc['destination.address'].empty) { - s.destination = doc['destination.address'].value; - } + state.mappedDocs = new HashMap(); + String[] docKeys = new String[] { + '@timestamp', + 'transaction.id', + 'parent.id', + 'span.id', + 'span.type', + 'span.subtype', + 'service.name', + 'service.environment', + 'destination.address' + }; + state.docKeys = docKeys; + ` +}; - if (!doc['span.type'].empty) { - s.span_type = doc['span.type'].value; +export const mapServiceConnsScript = { + lang: 'painless', + source: ` + def mappedDoc = new HashMap(); + for (docKey in state.docKeys) { + if (!doc[docKey].empty) { + mappedDoc[docKey] = doc[docKey].value; + } } - if (!doc['span.subtype'].empty) { - s.span_subtype = doc['span.subtype'].value; + if(!state.mappedDocs.containsKey(mappedDoc['parent.id'])) { + state.mappedDocs.put(mappedDoc['parent.id'], new ArrayList()); } - s.timestamp = doc['@timestamp'].value; - s.service_name = doc['service.name'].value; - if(!state.spans.containsKey(s.parent)) { - state.spans.put(s.parent, new ArrayList()) + def id; + if (!doc['span.id'].empty) { + id = doc['span.id'].value; + } else { + id = doc['transaction.id'].value; } - if (s.parent != s.id) { - state.spans[s.parent].add(s) + if (mappedDoc['parent.id'] != id) { + state.mappedDocs[mappedDoc['parent.id']].add(mappedDoc); } ` }; +export const combineServiceConnsScript = { + lang: 'painless', + source: `return state.mappedDocs` +}; + export const reduceServiceConnsScript = { lang: 'painless', source: ` - void extractChildren(def caller, def spans, def upstream, def conns, def count) { - // TODO: simplify this - if (spans.containsKey(caller.id)) { - for (s in spans[caller.id]) { - if (caller.span_type == 'external') { - upstream.add(caller.service_name + "/" + caller.environment); - def conn = new HashMap(); - conn.caller = caller; - conn.callee = s; - conn.upstream = new ArrayList(upstream); - conns.add(conn); - extractChildren(s, spans, upstream, conns, count); + void extractChildren(def caller, def mappedDocs, def upstream, def serviceConnections) { + def callerId; + if (caller.containsKey('span.id')) { + callerId = caller['span.id']; + } else { + callerId = caller['transaction.id']; + } + if (mappedDocs.containsKey(callerId)) { + for (mappedDoc in mappedDocs[callerId]) { + if (caller['span.type'] == 'external') { + upstream.add(caller['service.name'] + "/" + caller['service.environment']); + def serviceConnection = new HashMap(); + serviceConnection.caller = caller; + serviceConnection.callee = mappedDoc; + serviceConnection.upstream = new ArrayList(upstream); + serviceConnections.add(serviceConnection); + extractChildren(mappedDoc, mappedDocs, upstream, serviceConnections); upstream.remove(upstream.size() - 1); } else { - extractChildren(s, spans, upstream, conns, count); + extractChildren(mappedDoc, mappedDocs, upstream, serviceConnections); } } } else { // no connection found - def conn = new HashMap(); - conn.caller = caller; - conn.upstream = new ArrayList(upstream); - conn.upstream.add(caller.service_name + "/" + caller.environment); - conns.add(conn); + def serviceConnection = new HashMap(); + serviceConnection.caller = caller; + serviceConnection.upstream = new ArrayList(upstream); + serviceConnection.upstream.add(caller['service.name'] + "/" + caller['service.environment']); + serviceConnections.add(serviceConnection); } } - def conns = new HashSet(); - def spans = new HashMap(); + + def serviceConnections = new ArrayList(); + def mappedDocs = new HashMap(); // merge results from shards for (state in states) { for (s in state.entrySet()) { def v = s.getValue(); def k = s.getKey(); - if (!spans.containsKey(k)) { - spans[k] = v; + if (!mappedDocs.containsKey(k)) { + mappedDocs[k] = v; } else { for (p in v) { - spans[k].add(p); + mappedDocs[k].add(p); } } } } - if (spans.containsKey(null) && spans[null].size() > 0) { - def node = spans[null][0]; + if (mappedDocs.containsKey(null) && mappedDocs[null].size() > 0) { + def node = mappedDocs[null][0]; def upstream = new ArrayList(); - extractChildren(node, spans, upstream, conns, 0); - return new ArrayList(conns) + extractChildren(node, mappedDocs, upstream, serviceConnections); + return serviceConnections; } - return []; ` }; - -export const combineServiceConnsScript = { - lang: 'painless', - source: `return state.spans` -}; From 5e62db8271f1a4e6b016d18133259bf0f5d60c8b Mon Sep 17 00:00:00 2001 From: Oliver Gupte Date: Sat, 14 Dec 2019 05:16:57 -0800 Subject: [PATCH 13/15] fix import path failures --- .../service_map/create_service_connections_index.ts | 11 +++++++---- .../server/lib/service_map/run_service_map_task.ts | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/create_service_connections_index.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/create_service_connections_index.ts index 490647421c63f..34c7a413789cd 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/create_service_connections_index.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/create_service_connections_index.ts @@ -5,7 +5,7 @@ */ import { Server } from 'hapi'; -import { APMPluginContract } from '../../../../../../plugins/apm/server/plugin'; +import { APMPluginContract } from '../../../../../../plugins/apm/server'; import { getInternalSavedObjectsClient } from '../helpers/saved_objects_client'; import { CallCluster } from '../../../../../../../src/legacy/core_plugins/elasticsearch'; import { TIMESTAMP } from '../../../common/elasticsearch_fieldnames'; @@ -32,9 +32,12 @@ export async function createServiceConnectionsIndex(server: Server) { ); } } - } catch (e) { - // eslint-disable-next-line no-console - console.error('Could not create APM Service Connections:', e.message); + } catch (error) { + server.log( + ['apm', 'error'], + `Could not create APM Service Connections: ${error.message}` + ); + throw error; } } diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/run_service_map_task.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/run_service_map_task.ts index d55238ab6e3e4..87528d0a8ad10 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/run_service_map_task.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/run_service_map_task.ts @@ -6,7 +6,7 @@ import { Server } from 'hapi'; import { uniq } from 'lodash'; -import { APMPluginContract } from '../../../../../../plugins/apm/server/plugin'; +import { APMPluginContract } from '../../../../../../plugins/apm/server'; import { getESClient, ESClient } from '../helpers/es_client'; import { getNextTransactionSamples } from './get_next_transaction_samples'; import { getServiceConnections } from './get_service_connections'; From d7734ffe64967b616e27d43f3f8eed99f7c57026 Mon Sep 17 00:00:00 2001 From: Oliver Gupte Date: Tue, 17 Dec 2019 13:07:13 -0800 Subject: [PATCH 14/15] fixes typescript and task api errors --- .../apm/server/lib/service_map/get_service_connections.ts | 2 +- .../apm/server/lib/service_map/initialize_service_maps.ts | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_connections.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_connections.ts index f77a042020e36..7dc0719dbe92f 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_connections.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/get_service_connections.ts @@ -51,7 +51,7 @@ export async function getServiceConnections({ field: TRACE_ID, order: { _key: 'asc' as const }, size: traceIds.length, - execution_hint: 'map' + execution_hint: 'map' as const }, aggs: { connections: { diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/initialize_service_maps.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/initialize_service_maps.ts index b3ddb00a6e931..de4223cb0e7f7 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/initialize_service_maps.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/initialize_service_maps.ts @@ -50,7 +50,9 @@ export async function initializeServiceMaps(server: Server) { return await taskManager.ensureScheduled({ id: SERVICE_MAP_TASK_ID, taskType: SERVICE_MAP_TASK_TYPE, - interval: '1m', + schedule: { + interval: '1m' + }, scope: ['apm'], params: {}, state: {} From f0aec813a2a8c004abe09ad56f0be00b860a3c30 Mon Sep 17 00:00:00 2001 From: Oliver Gupte Date: Sat, 21 Dec 2019 01:32:49 -0800 Subject: [PATCH 15/15] Modify flow of the service map initialization to handle less permissive default security settings. The kibana user is responsible for creating and index to the `apm-service-connections` data index, while the apm user is resposible for kicking off the scheduled task and reading from `apm-*` indices. --- x-pack/legacy/plugins/apm/index.ts | 8 - .../components/app/ServiceMap/index.tsx | 53 ++++-- .../apm/server/lib/helpers/es_client.ts | 7 +- .../create_service_connections_index.ts | 47 ++--- .../service_map/initialize_service_maps.ts | 170 ++++++++++++++---- .../lib/service_map/run_service_map_task.ts | 28 ++- .../apm/server/routes/create_apm_api.ts | 6 +- .../plugins/apm/server/routes/service_map.ts | 33 ++++ .../plugins/apm/server/routes/services.ts | 23 --- x-pack/plugins/apm/server/plugin.ts | 2 + 10 files changed, 243 insertions(+), 134 deletions(-) create mode 100644 x-pack/legacy/plugins/apm/server/routes/service_map.ts diff --git a/x-pack/legacy/plugins/apm/index.ts b/x-pack/legacy/plugins/apm/index.ts index 1a78a61254bf0..f2f243cf62c24 100644 --- a/x-pack/legacy/plugins/apm/index.ts +++ b/x-pack/legacy/plugins/apm/index.ts @@ -11,7 +11,6 @@ import { APMPluginContract } from '../../../plugins/apm/server'; import { LegacyPluginInitializer } from '../../../../src/legacy/types'; import mappings from './mappings.json'; import { makeApmUsageCollector } from './server/lib/apm_telemetry'; -import { initializeServiceMaps } from './server/lib/service_map/initialize_service_maps'; export const apm: LegacyPluginInitializer = kibana => { return new kibana.Plugin({ @@ -78,15 +77,12 @@ export const apm: LegacyPluginInitializer = kibana => { autocreateApmIndexPattern: Joi.boolean().default(true), // service map - serviceMapEnabled: Joi.boolean().default(false) }).default(); }, // TODO: get proper types init(server: Server) { - const config = server.config(); - server.plugins.xpack_main.registerFeature({ id: 'apm', name: i18n.translate('xpack.apm.featureRegistry.apmFeatureName', { @@ -123,10 +119,6 @@ export const apm: LegacyPluginInitializer = kibana => { .apm as APMPluginContract; apmPlugin.registerLegacyAPI({ server }); - - if (config.get('xpack.apm.serviceMapEnabled')) { - initializeServiceMaps(server); - } } }); }; diff --git a/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/index.tsx b/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/index.tsx index cb4f93527d5c5..54decf15e160c 100644 --- a/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/index.tsx +++ b/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/index.tsx @@ -12,6 +12,8 @@ import { useUrlParams } from '../../../hooks/useUrlParams'; import { Controls } from './Controls'; import { Cytoscape } from './Cytoscape'; import { PlatinumLicensePrompt } from './PlatinumLicensePrompt'; +import { useApmPluginContext } from '../../../hooks/useApmPluginContext'; +import { callApi } from '../../../services/rest/callApi'; interface ServiceMapProps { serviceName?: string; @@ -51,9 +53,21 @@ export function ServiceMap({ serviceName }: ServiceMapProps) { [uiFilters] ); + const { http } = useApmPluginContext().core; + const { data: serviceMapStartResponse } = useFetcher(async () => { + const response = await callApi<{ + taskStatus: 'initialized' | 'active'; + }>(http, { + method: 'GET', + pathname: `/api/apm/service-map-start-task` + }); + return response; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [http]); + const { data } = useFetcher( callApmApi => { - if (start && end) { + if (start && end && serviceMapStartResponse) { return callApmApi({ pathname: '/api/apm/service-map', params: { @@ -68,23 +82,36 @@ export function ServiceMap({ serviceName }: ServiceMapProps) { }); } }, - [start, end, uiFiltersOmitEnv, environment, serviceName] + [ + start, + end, + uiFiltersOmitEnv, + environment, + serviceName, + serviceMapStartResponse + ] ); const elements = Array.isArray(data) ? data : []; const license = useLicense(); const isValidPlatinumLicense = - license?.isActive && license?.type === 'platinum'; + true || + (license?.isActive && + (license?.type === 'platinum' || license?.type === 'trial')); - return isValidPlatinumLicense ? ( - - - - ) : ( - + return ( + <> + {isValidPlatinumLicense ? ( + + + + ) : ( + + )} + ); } diff --git a/x-pack/legacy/plugins/apm/server/lib/helpers/es_client.ts b/x-pack/legacy/plugins/apm/server/lib/helpers/es_client.ts index d3af158687391..db371834b6a63 100644 --- a/x-pack/legacy/plugins/apm/server/lib/helpers/es_client.ts +++ b/x-pack/legacy/plugins/apm/server/lib/helpers/es_client.ts @@ -9,7 +9,8 @@ import { IndexDocumentParams, IndicesDeleteParams, IndicesCreateParams, - BulkIndexDocumentsParams + BulkIndexDocumentsParams, + IndicesExistsParams } from 'elasticsearch'; import { merge } from 'lodash'; import { cloneDeep, isString } from 'lodash'; @@ -143,6 +144,10 @@ export function getESClient( onRequest('indices.create', params); return esClient('indices.create', params); }, + indexExists: (params: IndicesExistsParams) => { + onRequest('indices.exists', params); + return esClient('indices.exists', params); + }, bulk: (params: BulkIndexDocumentsParams) => { onRequest('bulk', params); return esClient('bulk', params); diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/create_service_connections_index.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/create_service_connections_index.ts index 34c7a413789cd..eb6ee5d385063 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/create_service_connections_index.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/create_service_connections_index.ts @@ -4,45 +4,30 @@ * you may not use this file except in compliance with the Elastic License. */ -import { Server } from 'hapi'; -import { APMPluginContract } from '../../../../../../plugins/apm/server'; -import { getInternalSavedObjectsClient } from '../helpers/saved_objects_client'; -import { CallCluster } from '../../../../../../../src/legacy/core_plugins/elasticsearch'; import { TIMESTAMP } from '../../../common/elasticsearch_fieldnames'; +import { Setup } from '../helpers/setup_request'; -export async function createServiceConnectionsIndex(server: Server) { - const callCluster = server.plugins.elasticsearch.getCluster('data') - .callWithInternalUser; - const apmPlugin = server.newPlatform.setup.plugins.apm as APMPluginContract; +export async function createServiceConnectionsIndex(setup: Setup) { + const { internalClient, indices } = setup; + const index = indices.apmServiceConnectionsIndex; - try { - const apmIndices = await apmPlugin.getApmIndices( - getInternalSavedObjectsClient(server) - ); - const index = apmIndices.apmServiceConnectionsIndex; - const indexExists = await callCluster('indices.exists', { index }); - if (!indexExists) { - const result = await createNewIndex(index, callCluster); + const indexExists = await internalClient.indexExists({ index }); - if (!result.acknowledged) { - const resultError = - result && result.error && JSON.stringify(result.error); - throw new Error( - `Unable to create APM Service Connections index '${index}': ${resultError}` - ); - } + if (!indexExists) { + const result = await createNewIndex(index, internalClient); + + if (!result.acknowledged) { + const resultError = + result && result.error && JSON.stringify(result.error); + throw new Error( + `Unable to create APM Service Connections index '${index}': ${resultError}` + ); } - } catch (error) { - server.log( - ['apm', 'error'], - `Could not create APM Service Connections: ${error.message}` - ); - throw error; } } -function createNewIndex(index: string, callWithInternalUser: CallCluster) { - return callWithInternalUser('indices.create', { +function createNewIndex(index: string, client: Setup['client']) { + return client.indicesCreate({ index, body: { settings: { 'index.auto_expand_replicas': '0-1' }, diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/initialize_service_maps.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/initialize_service_maps.ts index de4223cb0e7f7..9ae076d5ade6c 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/initialize_service_maps.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/initialize_service_maps.ts @@ -5,57 +5,149 @@ */ import { Server } from 'hapi'; -// @ts-ignore -import { TaskManager, RunContext } from '../legacy/plugins/task_manager'; +import { CoreSetup, Logger } from 'src/core/server'; +import { Observable } from 'rxjs'; +import { PluginSetupContract as TaskManagerPluginContract } from '../../../../task_manager/plugin'; +import { RunContext, ConcreteTaskInstance } from '../../../../task_manager'; +import { RunFunction } from '../../../../task_manager/task'; +import { APMConfig } from '../../../../../../plugins/apm/server'; import { runServiceMapTask } from './run_service_map_task'; import { SERVICE_MAP_TASK_TYPE, SERVICE_MAP_TASK_ID } from '../../../common/service_map_constants'; import { createServiceConnectionsIndex } from './create_service_connections_index'; +import { setupRequest, Setup } from '../helpers/setup_request'; function isLessThan1Hour(unixTimestamp = 0) { const hourMilliseconds = 60 * 60 * 1000; return Date.now() - unixTimestamp < hourMilliseconds; } -export async function initializeServiceMaps(server: Server) { - await createServiceConnectionsIndex(server); - - const taskManager = server.plugins.task_manager; - if (taskManager) { - taskManager.registerTaskDefinitions({ - [SERVICE_MAP_TASK_TYPE]: { - title: 'ApmServiceMapTask', - type: SERVICE_MAP_TASK_TYPE, - description: 'Extract connections in traces for APM service maps', - timeout: '5m', - createTaskRunner({ taskInstance }: RunContext) { - return { - async run() { - const { state } = taskInstance; - const { latestTransactionTime } = await runServiceMapTask( - server, - isLessThan1Hour(state.latestTransactionTime) - ? state.latestTransactionTime - : 'now-1h' - ); - return { state: { latestTransactionTime } }; +let scopedRunFunction: + | ((taskInstance: ConcreteTaskInstance) => ReturnType) + | undefined; + +function isTaskActive() { + return scopedRunFunction !== undefined; +} + +async function runTask(setup: Setup, state: Record = {}) { + const nextState: typeof state = { + ...state, + isActive: true + }; + try { + const { latestTransactionTime } = await runServiceMapTask( + setup, + isLessThan1Hour(state.latestTransactionTime) + ? state.latestTransactionTime + : 'now-1h' + ); + nextState.latestTransactionTime = latestTransactionTime; + } catch (error) { + scopedRunFunction = undefined; + return { state: nextState, error }; + } + return { state: nextState }; +} + +async function scheduleTask( + taskManager: TaskManagerPluginContract, + runFn: NonNullable, + initialState: Record = {} +) { + scopedRunFunction = runFn; + return await taskManager.ensureScheduled({ + id: SERVICE_MAP_TASK_ID, + taskType: SERVICE_MAP_TASK_TYPE, + schedule: { interval: '1m' }, + scope: ['apm'], + params: {}, + state: initialState + }); +} + +export async function initializeServiceMaps( + core: CoreSetup, + { + config$, + logger, + __LEGACY + }: { + config$: Observable; + logger: Logger; + __LEGACY: { server: Server }; + } +) { + config$.subscribe(config => { + const server = __LEGACY.server; + const router = core.http.createRouter(); + + if (!config['xpack.apm.serviceMapEnabled']) { + return; + } + + const taskManager = server.plugins.task_manager; + if (taskManager) { + taskManager.registerTaskDefinitions({ + [SERVICE_MAP_TASK_TYPE]: { + title: 'ApmServiceMapTask', + type: SERVICE_MAP_TASK_TYPE, + description: 'Extract connections in traces for APM service maps', + timeout: '2m', + maxAttempts: 1, + createTaskRunner({ taskInstance }: RunContext) { + return { + run: async () => { + if (!scopedRunFunction) { + return; + } + return await scopedRunFunction(taskInstance); + } + }; + } + } + }); + + router.get( + { + path: '/api/apm/service-map-start-task', + validate: false + }, + async (context, request, response) => { + if (isTaskActive()) { + return response.ok({ body: { taskStatus: 'active' } }); + } + try { + const setup = await setupRequest( + { + ...context, + __LEGACY, + params: { query: { _debug: false } }, + config, + logger + }, + request + ); + await createServiceConnectionsIndex(setup); + const { state: initialState } = await runTask(setup); // initial task run + await scheduleTask( + taskManager, + (taskInstance: ConcreteTaskInstance) => + runTask(setup, taskInstance.state), // maintain scope in subsequent task runs + initialState + ); + return response.ok({ body: { taskStatus: 'initialized' } }); + } catch (error) { + logger.error(error); + if (error.statusCode === 403) { + return response.forbidden({ body: error }); } - }; + return response.internalError({ body: error }); + } } - } - }); - - return await taskManager.ensureScheduled({ - id: SERVICE_MAP_TASK_ID, - taskType: SERVICE_MAP_TASK_TYPE, - schedule: { - interval: '1m' - }, - scope: ['apm'], - params: {}, - state: {} - }); - } + ); + } + }); } diff --git a/x-pack/legacy/plugins/apm/server/lib/service_map/run_service_map_task.ts b/x-pack/legacy/plugins/apm/server/lib/service_map/run_service_map_task.ts index 87528d0a8ad10..998a52fa278b3 100644 --- a/x-pack/legacy/plugins/apm/server/lib/service_map/run_service_map_task.ts +++ b/x-pack/legacy/plugins/apm/server/lib/service_map/run_service_map_task.ts @@ -4,15 +4,12 @@ * you may not use this file except in compliance with the Elastic License. */ -import { Server } from 'hapi'; import { uniq } from 'lodash'; -import { APMPluginContract } from '../../../../../../plugins/apm/server'; -import { getESClient, ESClient } from '../helpers/es_client'; +import { ESClient } from '../helpers/es_client'; import { getNextTransactionSamples } from './get_next_transaction_samples'; import { getServiceConnections } from './get_service_connections'; import { mapTraceToBulkServiceConnection } from './map_trace_to_bulk_service_connection'; import { ENVIRONMENT_NOT_DEFINED } from '../../../common/environment_filter_values'; -import { getInternalSavedObjectsClient } from '../helpers/saved_objects_client'; import { ApmIndicesConfig } from '../settings/apm_indices/get_apm_indices'; import { SPAN_TYPE, @@ -25,6 +22,7 @@ import { SERVICE_ENVIRONMENT, DESTINATION_ADDRESS } from '../../../common/elasticsearch_fieldnames'; +import { Setup } from '../helpers/setup_request'; interface MappedDocument { [TIMESTAMP]: string; @@ -47,6 +45,7 @@ export interface TraceConnection { async function indexLatestConnections( apmIndices: ApmIndicesConfig, esClient: ESClient, + internalEsClient: ESClient, startTimeInterval?: string | number, latestTransactionTime = 0, afterKey?: object @@ -92,7 +91,7 @@ async function indexLatestConnections( ); }); if (bulkIndexConnectionDocs.length > 0) { - await esClient.bulk({ + await internalEsClient.bulk({ body: bulkIndexConnectionDocs .map(bulkObject => JSON.stringify(bulkObject)) .join('\n') @@ -101,6 +100,7 @@ async function indexLatestConnections( return await indexLatestConnections( apmIndices, esClient, + internalEsClient, startTimeInterval, nextLatestTransactionTime, nextAfterKey @@ -108,19 +108,13 @@ async function indexLatestConnections( } export async function runServiceMapTask( - server: Server, + setup: Setup, startTimeInterval?: string | number ) { - const callCluster = server.plugins.elasticsearch.getCluster('data') - .callWithInternalUser; - const apmPlugin = server.newPlatform.setup.plugins.apm as APMPluginContract; - const savedObjectsClient = getInternalSavedObjectsClient(server); - const apmIndices = await apmPlugin.getApmIndices(savedObjectsClient); - const esClient: ESClient = getESClient( - apmIndices, - server.uiSettingsServiceFactory({ savedObjectsClient }), - callCluster + return await indexLatestConnections( + setup.indices, + setup.client, + setup.internalClient, + startTimeInterval ); - - return await indexLatestConnections(apmIndices, esClient, startTimeInterval); } diff --git a/x-pack/legacy/plugins/apm/server/routes/create_apm_api.ts b/x-pack/legacy/plugins/apm/server/routes/create_apm_api.ts index a9da61c5c1f43..353b723289bf0 100644 --- a/x-pack/legacy/plugins/apm/server/routes/create_apm_api.ts +++ b/x-pack/legacy/plugins/apm/server/routes/create_apm_api.ts @@ -58,7 +58,7 @@ import { uiFiltersEnvironmentsRoute } from './ui_filters'; import { createApi } from './create_api'; -import { serviceMapRoute } from './services'; +import { serviceMapRoute } from './service_map'; const createApmApi = () => { const api = createApi() @@ -76,9 +76,11 @@ const createApmApi = () => { .add(serviceTransactionTypesRoute) .add(servicesRoute) .add(serviceNodeMetadataRoute) - .add(serviceMapRoute) .add(serviceAnnotationsRoute) + // Service Map + .add(serviceMapRoute) + // Agent configuration .add(agentConfigurationAgentNameRoute) .add(agentConfigurationRoute) diff --git a/x-pack/legacy/plugins/apm/server/routes/service_map.ts b/x-pack/legacy/plugins/apm/server/routes/service_map.ts new file mode 100644 index 0000000000000..cdb239362551c --- /dev/null +++ b/x-pack/legacy/plugins/apm/server/routes/service_map.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; +import Boom from 'boom'; +import { setupRequest } from '../lib/helpers/setup_request'; +import { createRoute } from './create_route'; +import { uiFiltersRt, rangeRt } from './default_api_types'; +import { getServiceMap } from '../lib/service_map/get_service_map'; + +export const serviceMapRoute = createRoute(() => ({ + path: '/api/apm/service-map', + params: { + query: t.intersection([ + t.partial({ environment: t.string, serviceName: t.string }), + uiFiltersRt, + rangeRt + ]) + }, + handler: async ({ context, request }) => { + if (!context.config['xpack.apm.serviceMapEnabled']) { + return new Boom('Not found', { statusCode: 404 }); + } + const setup = await setupRequest(context, request); + const { + query: { serviceName, environment } + } = context.params; + return getServiceMap({ setup, serviceName, environment }); + } +})); diff --git a/x-pack/legacy/plugins/apm/server/routes/services.ts b/x-pack/legacy/plugins/apm/server/routes/services.ts index af3dd025b70b9..18777183ea1de 100644 --- a/x-pack/legacy/plugins/apm/server/routes/services.ts +++ b/x-pack/legacy/plugins/apm/server/routes/services.ts @@ -5,7 +5,6 @@ */ import * as t from 'io-ts'; -import Boom from 'boom'; import { AgentName } from '../../typings/es_schemas/ui/fields/Agent'; import { createApmTelementry, @@ -18,7 +17,6 @@ import { getServiceTransactionTypes } from '../lib/services/get_service_transact import { getServiceNodeMetadata } from '../lib/services/get_service_node_metadata'; import { createRoute } from './create_route'; import { uiFiltersRt, rangeRt } from './default_api_types'; -import { getServiceMap } from '../lib/service_map/get_service_map'; import { getServiceAnnotations } from '../lib/services/annotations'; export const servicesRoute = createRoute(() => ({ @@ -87,27 +85,6 @@ export const serviceNodeMetadataRoute = createRoute(() => ({ } })); -export const serviceMapRoute = createRoute(() => ({ - path: '/api/apm/service-map', - params: { - query: t.intersection([ - t.partial({ environment: t.string, serviceName: t.string }), - uiFiltersRt, - rangeRt - ]) - }, - handler: async ({ context, request }) => { - if (!context.config['xpack.apm.serviceMapEnabled']) { - return new Boom('Not found', { statusCode: 404 }); - } - const setup = await setupRequest(context, request); - const { - query: { serviceName, environment } - } = context.params; - return getServiceMap({ setup, serviceName, environment }); - } -})); - export const serviceAnnotationsRoute = createRoute(() => ({ path: '/api/apm/services/{serviceName}/annotations', params: { diff --git a/x-pack/plugins/apm/server/plugin.ts b/x-pack/plugins/apm/server/plugin.ts index a1cf2ae4e8ead..c1e8e37a9f81c 100644 --- a/x-pack/plugins/apm/server/plugin.ts +++ b/x-pack/plugins/apm/server/plugin.ts @@ -18,6 +18,7 @@ import { createApmAgentConfigurationIndex } from '../../../legacy/plugins/apm/se import { createApmApi } from '../../../legacy/plugins/apm/server/routes/create_apm_api'; import { getApmIndices } from '../../../legacy/plugins/apm/server/lib/settings/apm_indices/get_apm_indices'; import { APMConfig, mergeConfigs, APMXPackConfig } from '.'; +import { initializeServiceMaps } from '../../../legacy/plugins/apm/server/lib/service_map/initialize_service_maps'; export interface LegacySetup { server: Server; @@ -55,6 +56,7 @@ export class APMPlugin implements Plugin { this.legacySetup$.subscribe(__LEGACY => { createApmApi().init(core, { config$: mergedConfig$, logger, __LEGACY }); + initializeServiceMaps(core, { config$: mergedConfig$, logger, __LEGACY }); }); await new Promise(resolve => {