diff --git a/x-pack/plugins/apm/public/components/app/TransactionDetails/index.tsx b/x-pack/plugins/apm/public/components/app/TransactionDetails/index.tsx index 620ae6708eda0..c56b7b9aaa720 100644 --- a/x-pack/plugins/apm/public/components/app/TransactionDetails/index.tsx +++ b/x-pack/plugins/apm/public/components/app/TransactionDetails/index.tsx @@ -89,7 +89,6 @@ export function TransactionDetails() { { }; public renderMLHeader(hasValidMlLicense: boolean | undefined) { - const { hasMLJob } = this.props; - if (!hasValidMlLicense || !hasMLJob) { + const { mlJobId } = this.props.charts; + + if (!hasValidMlLicense || !mlJobId) { return null; } - const { serviceName, kuery } = this.props.urlParams; + const { serviceName, kuery, transactionType } = this.props.urlParams; if (!serviceName) { return null; } - const linkedJobId = ''; // TODO [APM ML] link to ML job id for the selected environment - const hasKuery = !isEmpty(kuery); const icon = hasKuery ? ( { } )}{' '} - View Job + + View Job + ); diff --git a/x-pack/plugins/apm/public/selectors/chartSelectors.ts b/x-pack/plugins/apm/public/selectors/chartSelectors.ts index 714d62a703f51..26c2365ed77e1 100644 --- a/x-pack/plugins/apm/public/selectors/chartSelectors.ts +++ b/x-pack/plugins/apm/public/selectors/chartSelectors.ts @@ -33,6 +33,7 @@ export interface ITpmBucket { export interface ITransactionChartData { tpmSeries: ITpmBucket[]; responseTimeSeries: TimeSeries[]; + mlJobId: string | undefined; } const INITIAL_DATA = { @@ -62,6 +63,7 @@ export function getTransactionCharts( return { tpmSeries, responseTimeSeries, + mlJobId: anomalyTimeseries?.jobId, }; } diff --git a/x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/fetcher.ts b/x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/fetcher.ts new file mode 100644 index 0000000000000..3cf9a54e3fe9b --- /dev/null +++ b/x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/fetcher.ts @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Logger } from 'kibana/server'; +import { PromiseReturnType } from '../../../../../../observability/typings/common'; +import { Setup, SetupTimeRange } from '../../../helpers/setup_request'; + +export type ESResponse = Exclude< + PromiseReturnType, + undefined +>; + +export async function anomalySeriesFetcher({ + serviceName, + transactionType, + intervalString, + mlBucketSize, + setup, + jobId, + logger, +}: { + serviceName: string; + transactionType: string; + intervalString: string; + mlBucketSize: number; + setup: Setup & SetupTimeRange; + jobId: string; + logger: Logger; +}) { + const { ml, start, end } = setup; + if (!ml) { + return; + } + + // move the start back with one bucket size, to ensure to get anomaly data in the beginning + // this is required because ML has a minimum bucket size (default is 900s) so if our buckets are smaller, we might have several null buckets in the beginning + const newStart = start - mlBucketSize * 1000; + + const params = { + body: { + size: 0, + query: { + bool: { + filter: [ + { term: { job_id: jobId } }, + { exists: { field: 'bucket_span' } }, + { term: { result_type: 'model_plot' } }, + { term: { partition_field_value: serviceName } }, + { term: { by_field_value: transactionType } }, + { + range: { + timestamp: { gte: newStart, lte: end, format: 'epoch_millis' }, + }, + }, + ], + }, + }, + aggs: { + ml_avg_response_times: { + date_histogram: { + field: 'timestamp', + fixed_interval: intervalString, + min_doc_count: 0, + extended_bounds: { min: newStart, max: end }, + }, + aggs: { + anomaly_score: { max: { field: 'anomaly_score' } }, + lower: { min: { field: 'model_lower' } }, + upper: { max: { field: 'model_upper' } }, + }, + }, + }, + }, + }; + + try { + const response = await ml.mlSystem.mlAnomalySearch(params); + return response; + } catch (err) { + const isHttpError = 'statusCode' in err; + if (isHttpError) { + logger.info( + `Status code "${err.statusCode}" while retrieving ML anomalies for APM` + ); + return; + } + logger.error('An error occurred while retrieving ML anomalies for APM'); + logger.error(err); + } +} diff --git a/x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/get_ml_bucket_size.ts b/x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/get_ml_bucket_size.ts new file mode 100644 index 0000000000000..2f5e703251c03 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/get_ml_bucket_size.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Logger } from 'kibana/server'; +import { Setup, SetupTimeRange } from '../../../helpers/setup_request'; + +interface IOptions { + setup: Setup & SetupTimeRange; + jobId: string; + logger: Logger; +} + +interface ESResponse { + bucket_span: number; +} + +export async function getMlBucketSize({ + setup, + jobId, + logger, +}: IOptions): Promise { + const { ml, start, end } = setup; + if (!ml) { + return; + } + + const params = { + body: { + _source: 'bucket_span', + size: 1, + terminateAfter: 1, + query: { + bool: { + filter: [ + { term: { job_id: jobId } }, + { exists: { field: 'bucket_span' } }, + { + range: { + timestamp: { gte: start, lte: end, format: 'epoch_millis' }, + }, + }, + ], + }, + }, + }, + }; + + try { + const resp = await ml.mlSystem.mlAnomalySearch(params); + return resp.hits.hits[0]?._source.bucket_span; + } catch (err) { + const isHttpError = 'statusCode' in err; + if (isHttpError) { + return; + } + logger.error(err); + } +} diff --git a/x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/index.ts b/x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/index.ts index b2d11f2ffe19a..072099bc9553c 100644 --- a/x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/index.ts +++ b/x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/index.ts @@ -3,18 +3,19 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ - +import { Logger } from 'kibana/server'; +import { isNumber } from 'lodash'; +import { getBucketSize } from '../../../helpers/get_bucket_size'; import { Setup, SetupTimeRange, SetupUIFilters, } from '../../../helpers/setup_request'; -import { Coordinate, RectCoordinate } from '../../../../../typings/timeseries'; - -interface AnomalyTimeseries { - anomalyBoundaries: Coordinate[]; - anomalyScore: RectCoordinate[]; -} +import { anomalySeriesFetcher } from './fetcher'; +import { getMlBucketSize } from './get_ml_bucket_size'; +import { anomalySeriesTransform } from './transform'; +import { getMLJobIds } from '../../../service_map/get_service_anomalies'; +import { UIFilters } from '../../../../../typings/ui_filters'; export async function getAnomalySeries({ serviceName, @@ -22,13 +23,17 @@ export async function getAnomalySeries({ transactionName, timeSeriesDates, setup, + logger, + uiFilters, }: { serviceName: string; transactionType: string | undefined; transactionName: string | undefined; timeSeriesDates: number[]; setup: Setup & SetupTimeRange & SetupUIFilters; -}): Promise { + logger: Logger; + uiFilters: UIFilters; +}) { // don't fetch anomalies for transaction details page if (transactionName) { return; @@ -39,8 +44,12 @@ export async function getAnomalySeries({ return; } - // don't fetch anomalies if uiFilters are applied - if (setup.uiFiltersES.length > 0) { + // don't fetch anomalies if unknown uiFilters are applied + const knownFilters = ['environment', 'serviceName']; + const uiFilterNames = Object.keys(uiFilters); + if ( + uiFilterNames.some((uiFilterName) => !knownFilters.includes(uiFilterName)) + ) { return; } @@ -55,6 +64,45 @@ export async function getAnomalySeries({ return; } - // TODO [APM ML] return a series of anomaly scores, upper & lower bounds for the given timeSeriesDates - return; + let mlJobIds: string[] = []; + try { + mlJobIds = await getMLJobIds(setup.ml, uiFilters.environment); + } catch (error) { + logger.error(error); + return; + } + + // don't fetch anomalies if there are isn't exaclty 1 ML job match for the given environment + if (mlJobIds.length !== 1) { + return; + } + const jobId = mlJobIds[0]; + + const mlBucketSize = await getMlBucketSize({ setup, jobId, logger }); + if (!isNumber(mlBucketSize)) { + return; + } + + const { start, end } = setup; + const { intervalString, bucketSize } = getBucketSize(start, end, 'auto'); + + const esResponse = await anomalySeriesFetcher({ + serviceName, + transactionType, + intervalString, + mlBucketSize, + setup, + jobId, + logger, + }); + + if (esResponse && mlBucketSize > 0) { + return anomalySeriesTransform( + esResponse, + mlBucketSize, + bucketSize, + timeSeriesDates, + jobId + ); + } } diff --git a/x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/transform.ts b/x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/transform.ts new file mode 100644 index 0000000000000..393a73f7c1ccd --- /dev/null +++ b/x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/transform.ts @@ -0,0 +1,136 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { first, last } from 'lodash'; +import { Coordinate, RectCoordinate } from '../../../../../typings/timeseries'; +import { ESResponse } from './fetcher'; + +type IBucket = ReturnType; +function getBucket( + bucket: Required< + ESResponse + >['aggregations']['ml_avg_response_times']['buckets'][0] +) { + return { + x: bucket.key, + anomalyScore: bucket.anomaly_score.value, + lower: bucket.lower.value, + upper: bucket.upper.value, + }; +} + +export type AnomalyTimeSeriesResponse = ReturnType< + typeof anomalySeriesTransform +>; +export function anomalySeriesTransform( + response: ESResponse, + mlBucketSize: number, + bucketSize: number, + timeSeriesDates: number[], + jobId: string +) { + const buckets = + response.aggregations?.ml_avg_response_times.buckets.map(getBucket) || []; + + const bucketSizeInMillis = Math.max(bucketSize, mlBucketSize) * 1000; + + return { + jobId, + anomalyScore: getAnomalyScoreDataPoints( + buckets, + timeSeriesDates, + bucketSizeInMillis + ), + anomalyBoundaries: getAnomalyBoundaryDataPoints(buckets, timeSeriesDates), + }; +} + +export function getAnomalyScoreDataPoints( + buckets: IBucket[], + timeSeriesDates: number[], + bucketSizeInMillis: number +): RectCoordinate[] { + const ANOMALY_THRESHOLD = 75; + const firstDate = first(timeSeriesDates); + const lastDate = last(timeSeriesDates); + + if (firstDate === undefined || lastDate === undefined) { + return []; + } + + return buckets + .filter( + (bucket) => + bucket.anomalyScore !== null && bucket.anomalyScore > ANOMALY_THRESHOLD + ) + .filter(isInDateRange(firstDate, lastDate)) + .map((bucket) => { + return { + x0: bucket.x, + x: Math.min(bucket.x + bucketSizeInMillis, lastDate), // don't go beyond last date + }; + }); +} + +export function getAnomalyBoundaryDataPoints( + buckets: IBucket[], + timeSeriesDates: number[] +): Coordinate[] { + return replaceFirstAndLastBucket(buckets, timeSeriesDates) + .filter((bucket) => bucket.lower !== null) + .map((bucket) => { + return { + x: bucket.x, + y0: bucket.lower, + y: bucket.upper, + }; + }); +} + +export function replaceFirstAndLastBucket( + buckets: IBucket[], + timeSeriesDates: number[] +) { + const firstDate = first(timeSeriesDates); + const lastDate = last(timeSeriesDates); + + if (firstDate === undefined || lastDate === undefined) { + return buckets; + } + + const preBucketWithValue = buckets + .filter((p) => p.x <= firstDate) + .reverse() + .find((p) => p.lower !== null); + + const bucketsInRange = buckets.filter(isInDateRange(firstDate, lastDate)); + + // replace first bucket if it is null + const firstBucket = first(bucketsInRange); + if (preBucketWithValue && firstBucket && firstBucket.lower === null) { + firstBucket.lower = preBucketWithValue.lower; + firstBucket.upper = preBucketWithValue.upper; + } + + const lastBucketWithValue = [...buckets] + .reverse() + .find((p) => p.lower !== null); + + // replace last bucket if it is null + const lastBucket = last(bucketsInRange); + if (lastBucketWithValue && lastBucket && lastBucket.lower === null) { + lastBucket.lower = lastBucketWithValue.lower; + lastBucket.upper = lastBucketWithValue.upper; + } + + return bucketsInRange; +} + +// anomaly time series contain one or more buckets extra in the beginning +// these extra buckets should be removed +function isInDateRange(firstDate: number, lastDate: number) { + return (p: IBucket) => p.x >= firstDate && p.x <= lastDate; +} diff --git a/x-pack/plugins/apm/server/lib/transactions/charts/index.ts b/x-pack/plugins/apm/server/lib/transactions/charts/index.ts index 2ec049002d605..e862982145f77 100644 --- a/x-pack/plugins/apm/server/lib/transactions/charts/index.ts +++ b/x-pack/plugins/apm/server/lib/transactions/charts/index.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import { Logger } from 'kibana/server'; import { PromiseReturnType } from '../../../../../observability/typings/common'; import { Setup, @@ -13,6 +14,7 @@ import { import { getAnomalySeries } from './get_anomaly_data'; import { getApmTimeseriesData } from './get_timeseries_data'; import { ApmTimeSeriesResponse } from './get_timeseries_data/transform'; +import { UIFilters } from '../../../../typings/ui_filters'; function getDates(apmTimeseries: ApmTimeSeriesResponse) { return apmTimeseries.responseTimes.avg.map((p) => p.x); @@ -26,6 +28,8 @@ export async function getTransactionCharts(options: { transactionType: string | undefined; transactionName: string | undefined; setup: Setup & SetupTimeRange & SetupUIFilters; + logger: Logger; + uiFilters: UIFilters; }) { const apmTimeseries = await getApmTimeseriesData(options); const anomalyTimeseries = await getAnomalySeries({ diff --git a/x-pack/plugins/apm/server/lib/transactions/queries.test.ts b/x-pack/plugins/apm/server/lib/transactions/queries.test.ts index 713635cff2fbf..586fa1798b7bc 100644 --- a/x-pack/plugins/apm/server/lib/transactions/queries.test.ts +++ b/x-pack/plugins/apm/server/lib/transactions/queries.test.ts @@ -12,6 +12,8 @@ import { SearchParamsMock, inspectSearchParams, } from '../../../public/utils/testHelpers'; +// eslint-disable-next-line @kbn/eslint/no-restricted-paths +import { loggerMock } from '../../../../../../src/core/server/logging/logger.mock'; describe('transaction queries', () => { let mock: SearchParamsMock; @@ -52,6 +54,8 @@ describe('transaction queries', () => { transactionName: undefined, transactionType: undefined, setup, + logger: loggerMock.create(), + uiFilters: {}, }) ); expect(mock.params).toMatchSnapshot(); @@ -64,6 +68,8 @@ describe('transaction queries', () => { transactionName: 'bar', transactionType: undefined, setup, + logger: loggerMock.create(), + uiFilters: {}, }) ); expect(mock.params).toMatchSnapshot(); @@ -76,6 +82,8 @@ describe('transaction queries', () => { transactionName: 'bar', transactionType: 'baz', setup, + logger: loggerMock.create(), + uiFilters: {}, }) ); diff --git a/x-pack/plugins/apm/server/routes/transaction_groups.ts b/x-pack/plugins/apm/server/routes/transaction_groups.ts index 9ad281159fca5..3d939b04795c6 100644 --- a/x-pack/plugins/apm/server/routes/transaction_groups.ts +++ b/x-pack/plugins/apm/server/routes/transaction_groups.ts @@ -14,6 +14,7 @@ import { createRoute } from './create_route'; import { uiFiltersRt, rangeRt } from './default_api_types'; import { getTransactionAvgDurationByBrowser } from '../lib/transactions/avg_duration_by_browser'; import { getTransactionAvgDurationByCountry } from '../lib/transactions/avg_duration_by_country'; +import { UIFilters } from '../../typings/ui_filters'; export const transactionGroupsRoute = createRoute(() => ({ path: '/api/apm/services/{serviceName}/transaction_groups', @@ -62,14 +63,27 @@ export const transactionGroupsChartsRoute = createRoute(() => ({ }, handler: async ({ context, request }) => { const setup = await setupRequest(context, request); + const logger = context.logger; const { serviceName } = context.params.path; - const { transactionType, transactionName } = context.params.query; + const { + transactionType, + transactionName, + uiFilters: uiFiltersJson, + } = context.params.query; + let uiFilters: UIFilters = {}; + try { + uiFilters = JSON.parse(uiFiltersJson); + } catch (error) { + logger.error(error); + } return getTransactionCharts({ serviceName, transactionType, transactionName, setup, + logger, + uiFilters, }); }, }));