From e5ea84c7e0632dd94f137486281ac3d650fcb1c2 Mon Sep 17 00:00:00 2001 From: Oliver Gupte Date: Wed, 8 Jul 2020 17:30:55 -0700 Subject: [PATCH] Reintroduces the previous anomaly detection ML integration back into the transaction duration chart in the service details screen. Support the latest APM anoamly detection by environment jobs. --- .../app/TransactionOverview/index.tsx | 3 +- .../shared/charts/TransactionCharts/index.tsx | 19 ++- .../apm/public/selectors/chartSelectors.ts | 2 + .../charts/get_anomaly_data/fetcher.ts | 105 ++++++++++++++ .../get_anomaly_data/get_ml_bucket_size.ts | 65 +++++++++ .../charts/get_anomaly_data/index.ts | 69 +++++++-- .../charts/get_anomaly_data/transform.ts | 136 ++++++++++++++++++ .../server/lib/transactions/charts/index.ts | 1 + .../server/lib/transactions/queries.test.ts | 3 + .../apm/server/routes/transaction_groups.ts | 14 +- 10 files changed, 396 insertions(+), 21 deletions(-) create mode 100644 x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/fetcher.ts create mode 100644 x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/get_ml_bucket_size.ts create mode 100644 x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/transform.ts diff --git a/x-pack/plugins/apm/public/components/app/TransactionOverview/index.tsx b/x-pack/plugins/apm/public/components/app/TransactionOverview/index.tsx index a1e01b61d5c1b..22a0337ebc96e 100644 --- a/x-pack/plugins/apm/public/components/app/TransactionOverview/index.tsx +++ b/x-pack/plugins/apm/public/components/app/TransactionOverview/index.tsx @@ -130,8 +130,7 @@ export function TransactionOverview() { { }; public renderMLHeader(hasValidMlLicense: boolean | undefined) { - const { hasMLJob } = this.props; - if (!hasValidMlLicense || !hasMLJob) { + const { + hasMLJob, + charts: { mlJobId }, + } = this.props; + if (!hasValidMlLicense || !hasMLJob || !mlJobId) { return null; } - const { serviceName, kuery } = this.props.urlParams; + const { serviceName, kuery, transactionType } = this.props.urlParams; if (!serviceName) { return null; } - const linkedJobId = ''; // TODO [APM ML] link to ML job id for the selected environment - const hasKuery = !isEmpty(kuery); const icon = hasKuery ? ( { } )}{' '} - View Job + + View Job + ); diff --git a/x-pack/plugins/apm/public/selectors/chartSelectors.ts b/x-pack/plugins/apm/public/selectors/chartSelectors.ts index 714d62a703f51..26c2365ed77e1 100644 --- a/x-pack/plugins/apm/public/selectors/chartSelectors.ts +++ b/x-pack/plugins/apm/public/selectors/chartSelectors.ts @@ -33,6 +33,7 @@ export interface ITpmBucket { export interface ITransactionChartData { tpmSeries: ITpmBucket[]; responseTimeSeries: TimeSeries[]; + mlJobId: string | undefined; } const INITIAL_DATA = { @@ -62,6 +63,7 @@ export function getTransactionCharts( return { tpmSeries, responseTimeSeries, + mlJobId: anomalyTimeseries?.jobId, }; } diff --git a/x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/fetcher.ts b/x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/fetcher.ts new file mode 100644 index 0000000000000..ea6a3b66b9f3e --- /dev/null +++ b/x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/fetcher.ts @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { PromiseReturnType } from '../../../../../../observability/typings/common'; +import { Setup, SetupTimeRange } from '../../../helpers/setup_request'; + +export type ESResponse = Exclude< + PromiseReturnType, + undefined +>; + +export async function anomalySeriesFetcher({ + serviceName, + transactionType, + intervalString, + mlBucketSize, + setup, + jobId, +}: { + serviceName: string; + transactionType: string; + intervalString: string; + mlBucketSize: number; + setup: Setup & SetupTimeRange; + jobId: string; +}) { + const { ml, start, end } = setup; + if (!ml) { + return; + } + + // move the start back with one bucket size, to ensure to get anomaly data in the beginning + // this is required because ML has a minimum bucket size (default is 900s) so if our buckets are smaller, we might have several null buckets in the beginning + const newStart = start - mlBucketSize * 1000; + + const params = { + body: { + size: 0, + query: { + bool: { + filter: [ + { term: { job_id: jobId } }, + { exists: { field: 'bucket_span' } }, + { + term: { + result_type: 'model_plot', + }, + }, + { + term: { + partition_field_value: serviceName, + }, + }, + { + term: { + by_field_value: transactionType, + }, + }, + { + range: { + timestamp: { + gte: newStart, + lte: end, + format: 'epoch_millis', + }, + }, + }, + ], + }, + }, + aggs: { + ml_avg_response_times: { + date_histogram: { + field: 'timestamp', + fixed_interval: intervalString, + min_doc_count: 0, + extended_bounds: { + min: newStart, + max: end, + }, + }, + aggs: { + anomaly_score: { max: { field: 'anomaly_score' } }, + lower: { min: { field: 'model_lower' } }, + upper: { max: { field: 'model_upper' } }, + }, + }, + }, + }, + }; + + try { + const response = await ml.mlSystem.mlAnomalySearch(params); + return response; + } catch (err) { + const isHttpError = 'statusCode' in err; + if (isHttpError) { + return; + } + throw err; + } +} diff --git a/x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/get_ml_bucket_size.ts b/x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/get_ml_bucket_size.ts new file mode 100644 index 0000000000000..26301b8cd8742 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/get_ml_bucket_size.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Setup, SetupTimeRange } from '../../../helpers/setup_request'; + +interface IOptions { + serviceName: string; + transactionType: string; + setup: Setup & SetupTimeRange; + jobId: string; +} + +interface ESResponse { + bucket_span: number; +} + +export async function getMlBucketSize({ + serviceName, + transactionType, + setup, + jobId, +}: IOptions): Promise { + const { ml, start, end } = setup; + if (!ml) { + return 0; + } + + const params = { + body: { + _source: 'bucket_span', + size: 1, + query: { + bool: { + filter: [ + { term: { job_id: jobId } }, + { exists: { field: 'bucket_span' } }, + { + range: { + timestamp: { + gte: start, + lte: end, + format: 'epoch_millis', + }, + }, + }, + ], + }, + }, + }, + }; + + try { + const resp = await ml.mlSystem.mlAnomalySearch(params); + return resp.hits.hits[0]?._source.bucket_span || 0; + } catch (err) { + const isHttpError = 'statusCode' in err; + if (isHttpError) { + return 0; + } + throw err; + } +} diff --git a/x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/index.ts b/x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/index.ts index b2d11f2ffe19a..6a6005039b08a 100644 --- a/x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/index.ts +++ b/x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/index.ts @@ -3,32 +3,36 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ - +import { getBucketSize } from '../../../helpers/get_bucket_size'; import { Setup, SetupTimeRange, SetupUIFilters, } from '../../../helpers/setup_request'; -import { Coordinate, RectCoordinate } from '../../../../../typings/timeseries'; - -interface AnomalyTimeseries { - anomalyBoundaries: Coordinate[]; - anomalyScore: RectCoordinate[]; -} +import { anomalySeriesFetcher } from './fetcher'; +import { getMlBucketSize } from './get_ml_bucket_size'; +import { anomalySeriesTransform } from './transform'; +import { getMLJobIds } from '../../../service_map/get_service_anomalies'; +import { + SERVICE_ENVIRONMENT, + SERVICE_NAME, +} from '../../../../../common/elasticsearch_fieldnames'; export async function getAnomalySeries({ serviceName, transactionType, transactionName, + environment, timeSeriesDates, setup, }: { serviceName: string; transactionType: string | undefined; transactionName: string | undefined; + environment: string | undefined; timeSeriesDates: number[]; setup: Setup & SetupTimeRange & SetupUIFilters; -}): Promise { +}) { // don't fetch anomalies for transaction details page if (transactionName) { return; @@ -39,9 +43,16 @@ export async function getAnomalySeries({ return; } - // don't fetch anomalies if uiFilters are applied if (setup.uiFiltersES.length > 0) { - return; + // filter out known uiFilters like service.environment & service.name + const unknownFilters = setup.uiFiltersES.filter( + (uiFilter) => + !uiFilter.term?.[SERVICE_ENVIRONMENT] && !uiFilter.terms?.[SERVICE_NAME] + ); + // don't fetch anomalies if unknown uiFilters are applied + if (unknownFilters.length > 0) { + return; + } } // don't fetch anomalies if the ML plugin is not setup @@ -55,6 +66,40 @@ export async function getAnomalySeries({ return; } - // TODO [APM ML] return a series of anomaly scores, upper & lower bounds for the given timeSeriesDates - return; + const mlJobIds = await getMLJobIds(setup.ml, environment); + + // don't fetch anomalies if there are more than 1 ML jobs for the given environment + if (mlJobIds.length > 1) { + return; + } + const jobId = mlJobIds[0]; + + const mlBucketSize = await getMlBucketSize({ + serviceName, + transactionType, + setup, + jobId, + }); + + const { start, end } = setup; + const { intervalString, bucketSize } = getBucketSize(start, end, 'auto'); + + const esResponse = await anomalySeriesFetcher({ + serviceName, + transactionType, + intervalString, + mlBucketSize, + setup, + jobId, + }); + + return esResponse + ? anomalySeriesTransform( + esResponse, + mlBucketSize, + bucketSize, + timeSeriesDates, + jobId + ) + : undefined; } diff --git a/x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/transform.ts b/x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/transform.ts new file mode 100644 index 0000000000000..393a73f7c1ccd --- /dev/null +++ b/x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/transform.ts @@ -0,0 +1,136 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { first, last } from 'lodash'; +import { Coordinate, RectCoordinate } from '../../../../../typings/timeseries'; +import { ESResponse } from './fetcher'; + +type IBucket = ReturnType; +function getBucket( + bucket: Required< + ESResponse + >['aggregations']['ml_avg_response_times']['buckets'][0] +) { + return { + x: bucket.key, + anomalyScore: bucket.anomaly_score.value, + lower: bucket.lower.value, + upper: bucket.upper.value, + }; +} + +export type AnomalyTimeSeriesResponse = ReturnType< + typeof anomalySeriesTransform +>; +export function anomalySeriesTransform( + response: ESResponse, + mlBucketSize: number, + bucketSize: number, + timeSeriesDates: number[], + jobId: string +) { + const buckets = + response.aggregations?.ml_avg_response_times.buckets.map(getBucket) || []; + + const bucketSizeInMillis = Math.max(bucketSize, mlBucketSize) * 1000; + + return { + jobId, + anomalyScore: getAnomalyScoreDataPoints( + buckets, + timeSeriesDates, + bucketSizeInMillis + ), + anomalyBoundaries: getAnomalyBoundaryDataPoints(buckets, timeSeriesDates), + }; +} + +export function getAnomalyScoreDataPoints( + buckets: IBucket[], + timeSeriesDates: number[], + bucketSizeInMillis: number +): RectCoordinate[] { + const ANOMALY_THRESHOLD = 75; + const firstDate = first(timeSeriesDates); + const lastDate = last(timeSeriesDates); + + if (firstDate === undefined || lastDate === undefined) { + return []; + } + + return buckets + .filter( + (bucket) => + bucket.anomalyScore !== null && bucket.anomalyScore > ANOMALY_THRESHOLD + ) + .filter(isInDateRange(firstDate, lastDate)) + .map((bucket) => { + return { + x0: bucket.x, + x: Math.min(bucket.x + bucketSizeInMillis, lastDate), // don't go beyond last date + }; + }); +} + +export function getAnomalyBoundaryDataPoints( + buckets: IBucket[], + timeSeriesDates: number[] +): Coordinate[] { + return replaceFirstAndLastBucket(buckets, timeSeriesDates) + .filter((bucket) => bucket.lower !== null) + .map((bucket) => { + return { + x: bucket.x, + y0: bucket.lower, + y: bucket.upper, + }; + }); +} + +export function replaceFirstAndLastBucket( + buckets: IBucket[], + timeSeriesDates: number[] +) { + const firstDate = first(timeSeriesDates); + const lastDate = last(timeSeriesDates); + + if (firstDate === undefined || lastDate === undefined) { + return buckets; + } + + const preBucketWithValue = buckets + .filter((p) => p.x <= firstDate) + .reverse() + .find((p) => p.lower !== null); + + const bucketsInRange = buckets.filter(isInDateRange(firstDate, lastDate)); + + // replace first bucket if it is null + const firstBucket = first(bucketsInRange); + if (preBucketWithValue && firstBucket && firstBucket.lower === null) { + firstBucket.lower = preBucketWithValue.lower; + firstBucket.upper = preBucketWithValue.upper; + } + + const lastBucketWithValue = [...buckets] + .reverse() + .find((p) => p.lower !== null); + + // replace last bucket if it is null + const lastBucket = last(bucketsInRange); + if (lastBucketWithValue && lastBucket && lastBucket.lower === null) { + lastBucket.lower = lastBucketWithValue.lower; + lastBucket.upper = lastBucketWithValue.upper; + } + + return bucketsInRange; +} + +// anomaly time series contain one or more buckets extra in the beginning +// these extra buckets should be removed +function isInDateRange(firstDate: number, lastDate: number) { + return (p: IBucket) => p.x >= firstDate && p.x <= lastDate; +} diff --git a/x-pack/plugins/apm/server/lib/transactions/charts/index.ts b/x-pack/plugins/apm/server/lib/transactions/charts/index.ts index 2ec049002d605..e6f975e8eb78e 100644 --- a/x-pack/plugins/apm/server/lib/transactions/charts/index.ts +++ b/x-pack/plugins/apm/server/lib/transactions/charts/index.ts @@ -25,6 +25,7 @@ export async function getTransactionCharts(options: { serviceName: string; transactionType: string | undefined; transactionName: string | undefined; + environment: string | undefined; setup: Setup & SetupTimeRange & SetupUIFilters; }) { const apmTimeseries = await getApmTimeseriesData(options); diff --git a/x-pack/plugins/apm/server/lib/transactions/queries.test.ts b/x-pack/plugins/apm/server/lib/transactions/queries.test.ts index 713635cff2fbf..7fd156a20e688 100644 --- a/x-pack/plugins/apm/server/lib/transactions/queries.test.ts +++ b/x-pack/plugins/apm/server/lib/transactions/queries.test.ts @@ -52,6 +52,7 @@ describe('transaction queries', () => { transactionName: undefined, transactionType: undefined, setup, + environment: 'test', }) ); expect(mock.params).toMatchSnapshot(); @@ -64,6 +65,7 @@ describe('transaction queries', () => { transactionName: 'bar', transactionType: undefined, setup, + environment: 'test', }) ); expect(mock.params).toMatchSnapshot(); @@ -76,6 +78,7 @@ describe('transaction queries', () => { transactionName: 'bar', transactionType: 'baz', setup, + environment: 'test', }) ); diff --git a/x-pack/plugins/apm/server/routes/transaction_groups.ts b/x-pack/plugins/apm/server/routes/transaction_groups.ts index 9ad281159fca5..b3ee81d09cd35 100644 --- a/x-pack/plugins/apm/server/routes/transaction_groups.ts +++ b/x-pack/plugins/apm/server/routes/transaction_groups.ts @@ -14,6 +14,7 @@ import { createRoute } from './create_route'; import { uiFiltersRt, rangeRt } from './default_api_types'; import { getTransactionAvgDurationByBrowser } from '../lib/transactions/avg_duration_by_browser'; import { getTransactionAvgDurationByCountry } from '../lib/transactions/avg_duration_by_country'; +import { UIFilters } from '../../typings/ui_filters'; export const transactionGroupsRoute = createRoute(() => ({ path: '/api/apm/services/{serviceName}/transaction_groups', @@ -63,13 +64,24 @@ export const transactionGroupsChartsRoute = createRoute(() => ({ handler: async ({ context, request }) => { const setup = await setupRequest(context, request); const { serviceName } = context.params.path; - const { transactionType, transactionName } = context.params.query; + const { + transactionType, + transactionName, + uiFilters: uiFiltersJson, + } = context.params.query; + let uiFilters: UIFilters = {}; + try { + uiFilters = JSON.parse(uiFiltersJson); + } catch (error) { + context.logger.error(error); + } return getTransactionCharts({ serviceName, transactionType, transactionName, setup, + environment: uiFilters.environment, }); }, }));