From 125eee0e7d49c8f08ab797691d522ab3c02315e1 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Thu, 15 Jul 2021 12:49:13 +0200 Subject: [PATCH] [7.x] [ML] API integration tests for APM latency correlation. (#104644) (#105719) Adds API integration tests for APM Latency Correlations code. Writing the tests surfaced some glitches fixed as part of this PR: - If the applied filters don't return any docs, we won't throw an error anymore. Instead, the async search service finishes early and just returns no results. - If for whatever reason the async search service throws an error, it will also set its state now to isRunning = false. - If the client triggers a request with a service ID we now make sure that async search service still exists. We throw an error if that service no longer exists. This avoids re-instantiating async search services when they've already finished or failed and for whatever reason a client triggers another request with the same ID. - Refactored requests to reuse APM's own getCorrelationsFilters(). We now require start/end to be set and it will be converted from ISO (client side) to epochmillis (server side) to be more in line with APM's existing code. - The async search service now creates a simple internal log. This gets exposed via the API and we assert it using the API tests. In the future, we might also expose it in the UI to allow for better problem investigation for users and support. - Use 8.0.0 dataset instead of ml_8.0.0 --- .../correlations/ml_latency_correlations.tsx | 34 ++- .../app/correlations/use_correlations.ts | 1 + .../correlations/async_search_service.ts | 66 ++++- .../get_query_with_params.test.ts | 55 +++- .../correlations/get_query_with_params.ts | 65 +++-- .../correlations/query_correlation.test.ts | 2 +- .../query_field_candidates.test.ts | 11 +- .../query_field_value_pairs.test.ts | 2 +- .../correlations/query_fractions.test.ts | 2 +- .../correlations/query_histogram.test.ts | 11 +- .../query_histogram_interval.test.ts | 11 +- ...ts => query_histogram_range_steps.test.ts} | 22 +- ...teps.ts => query_histogram_range_steps.ts} | 25 +- .../correlations/query_percentiles.test.ts | 16 +- .../correlations/query_percentiles.ts | 21 +- .../correlations/query_ranges.test.ts | 11 +- .../correlations/search_strategy.test.ts | 30 +- .../correlations/search_strategy.ts | 35 ++- .../utils/aggregation_utils.test.ts | 94 +++++++ .../tests/correlations/latency_ml.ts | 266 ++++++++++++++++++ .../test/apm_api_integration/tests/index.ts | 4 + 21 files changed, 675 insertions(+), 109 deletions(-) rename x-pack/plugins/apm/server/lib/search_strategies/correlations/{query_histogram_rangesteps.test.ts => query_histogram_range_steps.test.ts} (77%) rename x-pack/plugins/apm/server/lib/search_strategies/correlations/{query_histogram_rangesteps.ts => query_histogram_range_steps.ts} (83%) create mode 100644 x-pack/test/apm_api_integration/tests/correlations/latency_ml.ts diff --git a/x-pack/plugins/apm/public/components/app/correlations/ml_latency_correlations.tsx b/x-pack/plugins/apm/public/components/app/correlations/ml_latency_correlations.tsx index 4bd20f51977c6..03fab3e788639 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/ml_latency_correlations.tsx +++ b/x-pack/plugins/apm/public/components/app/correlations/ml_latency_correlations.tsx @@ -61,17 +61,16 @@ export function MlLatencyCorrelations({ onClose }: Props) { } = useApmPluginContext(); const { serviceName } = useParams<{ serviceName: string }>(); - const { urlParams } = useUrlParams(); - - const fetchOptions = useMemo( - () => ({ - ...{ - serviceName, - ...urlParams, - }, - }), - [serviceName, urlParams] - ); + const { + urlParams: { + environment, + kuery, + transactionName, + transactionType, + start, + end, + }, + } = useUrlParams(); const { error, @@ -85,7 +84,15 @@ export function MlLatencyCorrelations({ onClose }: Props) { } = useCorrelations({ index: 'apm-*', ...{ - ...fetchOptions, + ...{ + environment, + kuery, + serviceName, + transactionName, + transactionType, + start, + end, + }, percentileThreshold: DEFAULT_PERCENTILE_THRESHOLD, }, }); @@ -322,8 +329,7 @@ export function MlLatencyCorrelations({ onClose }: Props) { { defaultMessage: 'Latency distribution for {name}', values: { - name: - fetchOptions.transactionName ?? fetchOptions.serviceName, + name: transactionName ?? serviceName, }, } )} diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_correlations.ts index 8c874571d23db..2baeb63fa4a23 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_correlations.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_correlations.ts @@ -36,6 +36,7 @@ interface RawResponse { took: number; values: SearchServiceValue[]; overallHistogram: HistogramItem[]; + log: string[]; } export const useCorrelations = (params: CorrelationsOptions) => { diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/async_search_service.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/async_search_service.ts index 7a511fc60fd06..155cb1f4615bd 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/async_search_service.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/async_search_service.ts @@ -11,7 +11,7 @@ import { fetchTransactionDurationFieldCandidates } from './query_field_candidate import { fetchTransactionDurationFieldValuePairs } from './query_field_value_pairs'; import { fetchTransactionDurationPercentiles } from './query_percentiles'; import { fetchTransactionDurationCorrelation } from './query_correlation'; -import { fetchTransactionDurationHistogramRangesteps } from './query_histogram_rangesteps'; +import { fetchTransactionDurationHistogramRangeSteps } from './query_histogram_range_steps'; import { fetchTransactionDurationRanges, HistogramItem } from './query_ranges'; import type { AsyncSearchProviderProgress, @@ -24,6 +24,8 @@ import { fetchTransactionDurationFractions } from './query_fractions'; const CORRELATION_THRESHOLD = 0.3; const KS_TEST_THRESHOLD = 0.1; +const currentTimeAsString = () => new Date().toISOString(); + export const asyncSearchServiceProvider = ( esClient: ElasticsearchClient, params: SearchServiceParams @@ -31,6 +33,9 @@ export const asyncSearchServiceProvider = ( let isCancelled = false; let isRunning = true; let error: Error; + const log: string[] = []; + const logMessage = (message: string) => + log.push(`${currentTimeAsString()}: ${message}`); const progress: AsyncSearchProviderProgress = { started: Date.now(), @@ -53,13 +58,17 @@ export const asyncSearchServiceProvider = ( let percentileThresholdValue: number; const cancel = () => { + logMessage(`Service cancelled.`); isCancelled = true; }; const fetchCorrelations = async () => { try { // 95th percentile to be displayed as a marker in the log log chart - const percentileThreshold = await fetchTransactionDurationPercentiles( + const { + totalDocs, + percentiles: percentileThreshold, + } = await fetchTransactionDurationPercentiles( esClient, params, params.percentileThreshold ? [params.percentileThreshold] : undefined @@ -67,12 +76,32 @@ export const asyncSearchServiceProvider = ( percentileThresholdValue = percentileThreshold[`${params.percentileThreshold}.0`]; - const histogramRangeSteps = await fetchTransactionDurationHistogramRangesteps( + logMessage( + `Fetched ${params.percentileThreshold}th percentile value of ${percentileThresholdValue} based on ${totalDocs} documents.` + ); + + // finish early if we weren't able to identify the percentileThresholdValue. + if (percentileThresholdValue === undefined) { + logMessage( + `Abort service since percentileThresholdValue could not be determined.` + ); + progress.loadedHistogramStepsize = 1; + progress.loadedOverallHistogram = 1; + progress.loadedFieldCanditates = 1; + progress.loadedFieldValuePairs = 1; + progress.loadedHistograms = 1; + isRunning = false; + return; + } + + const histogramRangeSteps = await fetchTransactionDurationHistogramRangeSteps( esClient, params ); progress.loadedHistogramStepsize = 1; + logMessage(`Loaded histogram range steps.`); + if (isCancelled) { isRunning = false; return; @@ -86,6 +115,8 @@ export const asyncSearchServiceProvider = ( progress.loadedOverallHistogram = 1; overallHistogram = overallLogHistogramChartData; + logMessage(`Loaded overall histogram chart data.`); + if (isCancelled) { isRunning = false; return; @@ -93,13 +124,13 @@ export const asyncSearchServiceProvider = ( // Create an array of ranges [2, 4, 6, ..., 98] const percents = Array.from(range(2, 100, 2)); - const percentilesRecords = await fetchTransactionDurationPercentiles( - esClient, - params, - percents - ); + const { + percentiles: percentilesRecords, + } = await fetchTransactionDurationPercentiles(esClient, params, percents); const percentiles = Object.values(percentilesRecords); + logMessage(`Loaded percentiles.`); + if (isCancelled) { isRunning = false; return; @@ -110,6 +141,8 @@ export const asyncSearchServiceProvider = ( params ); + logMessage(`Identified ${fieldCandidates.length} fieldCandidates.`); + progress.loadedFieldCanditates = 1; const fieldValuePairs = await fetchTransactionDurationFieldValuePairs( @@ -119,6 +152,8 @@ export const asyncSearchServiceProvider = ( progress ); + logMessage(`Identified ${fieldValuePairs.length} fieldValuePairs.`); + if (isCancelled) { isRunning = false; return; @@ -133,6 +168,8 @@ export const asyncSearchServiceProvider = ( totalDocCount, } = await fetchTransactionDurationFractions(esClient, params, ranges); + logMessage(`Loaded fractions and totalDocCount of ${totalDocCount}.`); + async function* fetchTransactionDurationHistograms() { for (const item of shuffle(fieldValuePairs)) { if (item === undefined || isCancelled) { @@ -185,7 +222,11 @@ export const asyncSearchServiceProvider = ( yield undefined; } } catch (e) { - error = e; + // don't fail the whole process for individual correlation queries, just add the error to the internal log. + logMessage( + `Failed to fetch correlation/kstest for '${item.field}/${item.value}'` + ); + yield undefined; } } } @@ -199,10 +240,14 @@ export const asyncSearchServiceProvider = ( progress.loadedHistograms = loadedHistograms / fieldValuePairs.length; } - isRunning = false; + logMessage( + `Identified ${values.length} significant correlations out of ${fieldValuePairs.length} field/value pairs.` + ); } catch (e) { error = e; } + + isRunning = false; }; fetchCorrelations(); @@ -212,6 +257,7 @@ export const asyncSearchServiceProvider = ( return { error, + log, isRunning, loaded: Math.round(progress.getOverallProgress() * 100), overallHistogram, diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/get_query_with_params.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/get_query_with_params.test.ts index 12e897ab3eec9..016355b3a6415 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/get_query_with_params.test.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/get_query_with_params.test.ts @@ -10,10 +10,23 @@ import { getQueryWithParams } from './get_query_with_params'; describe('correlations', () => { describe('getQueryWithParams', () => { it('returns the most basic query filtering on processor.event=transaction', () => { - const query = getQueryWithParams({ params: { index: 'apm-*' } }); + const query = getQueryWithParams({ + params: { index: 'apm-*', start: '2020', end: '2021' }, + }); expect(query).toEqual({ bool: { - filter: [{ term: { 'processor.event': 'transaction' } }], + filter: [ + { term: { 'processor.event': 'transaction' } }, + { + range: { + '@timestamp': { + format: 'epoch_millis', + gte: 1577836800000, + lte: 1609459200000, + }, + }, + }, + ], }, }); }); @@ -24,8 +37,8 @@ describe('correlations', () => { index: 'apm-*', serviceName: 'actualServiceName', transactionName: 'actualTransactionName', - start: '01-01-2021', - end: '31-01-2021', + start: '2020', + end: '2021', environment: 'dev', percentileThresholdValue: 75, }, @@ -33,22 +46,17 @@ describe('correlations', () => { expect(query).toEqual({ bool: { filter: [ - { term: { 'processor.event': 'transaction' } }, - { - term: { - 'service.name': 'actualServiceName', - }, - }, { term: { - 'transaction.name': 'actualTransactionName', + 'processor.event': 'transaction', }, }, { range: { '@timestamp': { - gte: '01-01-2021', - lte: '31-01-2021', + format: 'epoch_millis', + gte: 1577836800000, + lte: 1609459200000, }, }, }, @@ -57,6 +65,16 @@ describe('correlations', () => { 'service.environment': 'dev', }, }, + { + term: { + 'service.name': 'actualServiceName', + }, + }, + { + term: { + 'transaction.name': 'actualTransactionName', + }, + }, { range: { 'transaction.duration.us': { @@ -71,7 +89,7 @@ describe('correlations', () => { it('returns a query considering a custom field/value pair', () => { const query = getQueryWithParams({ - params: { index: 'apm-*' }, + params: { index: 'apm-*', start: '2020', end: '2021' }, fieldName: 'actualFieldName', fieldValue: 'actualFieldValue', }); @@ -79,6 +97,15 @@ describe('correlations', () => { bool: { filter: [ { term: { 'processor.event': 'transaction' } }, + { + range: { + '@timestamp': { + format: 'epoch_millis', + gte: 1577836800000, + lte: 1609459200000, + }, + }, + }, { term: { actualFieldName: 'actualFieldValue', diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/get_query_with_params.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/get_query_with_params.ts index a13b156242f59..e0ddfc1b053b5 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/get_query_with_params.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/get_query_with_params.ts @@ -5,16 +5,19 @@ * 2.0. */ +import { pipe } from 'fp-ts/lib/pipeable'; +import { getOrElse } from 'fp-ts/lib/Either'; +import { failure } from 'io-ts/lib/PathReporter'; +import * as t from 'io-ts'; + import type { estypes } from '@elastic/elasticsearch'; -import { - PROCESSOR_EVENT, - SERVICE_NAME, - TRANSACTION_DURATION, - TRANSACTION_NAME, -} from '../../../../common/elasticsearch_fieldnames'; +import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames'; import type { SearchServiceParams } from '../../../../common/search_strategies/correlations/types'; -import { environmentQuery as getEnvironmentQuery } from '../../../../common/utils/environment_query'; -import { ProcessorEvent } from '../../../../common/processor_event'; +import { rangeRt } from '../../../routes/default_api_types'; + +import { Setup, SetupTimeRange } from '../../helpers/setup_request'; + +import { getCorrelationsFilters } from '../../correlations/get_filters'; const getPercentileThresholdValueQuery = ( percentileThresholdValue: number | undefined @@ -39,26 +42,6 @@ export const getTermsQuery = ( return fieldName && fieldValue ? [{ term: { [fieldName]: fieldValue } }] : []; }; -const getRangeQuery = ( - start?: string, - end?: string -): estypes.QueryDslQueryContainer[] => { - if (start === undefined && end === undefined) { - return []; - } - - return [ - { - range: { - '@timestamp': { - ...(start !== undefined ? { gte: start } : {}), - ...(end !== undefined ? { lte: end } : {}), - }, - }, - }, - ]; -}; - interface QueryParams { params: SearchServiceParams; fieldName?: string; @@ -71,21 +54,37 @@ export const getQueryWithParams = ({ }: QueryParams) => { const { environment, + kuery, serviceName, start, end, percentileThresholdValue, + transactionType, transactionName, } = params; + + // converts string based start/end to epochmillis + const setup = pipe( + rangeRt.decode({ start, end }), + getOrElse((errors) => { + throw new Error(failure(errors).join('\n')); + }) + ) as Setup & SetupTimeRange; + + const filters = getCorrelationsFilters({ + setup, + environment, + kuery, + serviceName, + transactionType, + transactionName, + }); + return { bool: { filter: [ - ...getTermsQuery(PROCESSOR_EVENT, ProcessorEvent.transaction), - ...getTermsQuery(SERVICE_NAME, serviceName), - ...getTermsQuery(TRANSACTION_NAME, transactionName), + ...filters, ...getTermsQuery(fieldName, fieldValue), - ...getRangeQuery(start, end), - ...getEnvironmentQuery(environment), ...getPercentileThresholdValueQuery(percentileThresholdValue), ] as estypes.QueryDslQueryContainer[], }, diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_correlation.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_correlation.test.ts index 24741ebaa2dae..678328dce1a19 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_correlation.test.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_correlation.test.ts @@ -15,7 +15,7 @@ import { BucketCorrelation, } from './query_correlation'; -const params = { index: 'apm-*' }; +const params = { index: 'apm-*', start: '2020', end: '2021' }; const expectations = [1, 3, 5]; const ranges = [{ to: 1 }, { from: 1, to: 3 }, { from: 3, to: 5 }, { from: 5 }]; const fractions = [1, 2, 4, 5]; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_candidates.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_candidates.test.ts index 89bdd4280d324..8929b31b3ecb1 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_candidates.test.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_candidates.test.ts @@ -16,7 +16,7 @@ import { shouldBeExcluded, } from './query_field_candidates'; -const params = { index: 'apm-*' }; +const params = { index: 'apm-*', start: '2020', end: '2021' }; describe('query_field_candidates', () => { describe('shouldBeExcluded', () => { @@ -61,6 +61,15 @@ describe('query_field_candidates', () => { 'processor.event': 'transaction', }, }, + { + range: { + '@timestamp': { + format: 'epoch_millis', + gte: 1577836800000, + lte: 1609459200000, + }, + }, + }, ], }, }, diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_value_pairs.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_value_pairs.test.ts index ea5a1f55bc924..7ffbc5208e41e 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_value_pairs.test.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_value_pairs.test.ts @@ -16,7 +16,7 @@ import { getTermsAggRequest, } from './query_field_value_pairs'; -const params = { index: 'apm-*' }; +const params = { index: 'apm-*', start: '2020', end: '2021' }; describe('query_field_value_pairs', () => { describe('getTermsAggRequest', () => { diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_fractions.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_fractions.test.ts index 6052841d277c3..3e7d4a52e4de2 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_fractions.test.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_fractions.test.ts @@ -14,7 +14,7 @@ import { getTransactionDurationRangesRequest, } from './query_fractions'; -const params = { index: 'apm-*' }; +const params = { index: 'apm-*', start: '2020', end: '2021' }; const ranges = [{ to: 1 }, { from: 1, to: 3 }, { from: 3, to: 5 }, { from: 5 }]; describe('query_fractions', () => { diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram.test.ts index 2be9446352260..ace9177947960 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram.test.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram.test.ts @@ -14,7 +14,7 @@ import { getTransactionDurationHistogramRequest, } from './query_histogram'; -const params = { index: 'apm-*' }; +const params = { index: 'apm-*', start: '2020', end: '2021' }; const interval = 100; describe('query_histogram', () => { @@ -40,6 +40,15 @@ describe('query_histogram', () => { 'processor.event': 'transaction', }, }, + { + range: { + '@timestamp': { + format: 'epoch_millis', + gte: 1577836800000, + lte: 1609459200000, + }, + }, + }, ], }, }, diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_interval.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_interval.test.ts index 9ed529ccabddb..ebd78f1248510 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_interval.test.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_interval.test.ts @@ -14,7 +14,7 @@ import { getHistogramIntervalRequest, } from './query_histogram_interval'; -const params = { index: 'apm-*' }; +const params = { index: 'apm-*', start: '2020', end: '2021' }; describe('query_histogram_interval', () => { describe('getHistogramIntervalRequest', () => { @@ -43,6 +43,15 @@ describe('query_histogram_interval', () => { 'processor.event': 'transaction', }, }, + { + range: { + '@timestamp': { + format: 'epoch_millis', + gte: 1577836800000, + lte: 1609459200000, + }, + }, + }, ], }, }, diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_rangesteps.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_range_steps.test.ts similarity index 77% rename from x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_rangesteps.test.ts rename to x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_range_steps.test.ts index bb366ea29fed4..76aab1cd979c9 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_rangesteps.test.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_range_steps.test.ts @@ -10,13 +10,13 @@ import type { estypes } from '@elastic/elasticsearch'; import type { ElasticsearchClient } from 'src/core/server'; import { - fetchTransactionDurationHistogramRangesteps, + fetchTransactionDurationHistogramRangeSteps, getHistogramIntervalRequest, -} from './query_histogram_rangesteps'; +} from './query_histogram_range_steps'; -const params = { index: 'apm-*' }; +const params = { index: 'apm-*', start: '2020', end: '2021' }; -describe('query_histogram_rangesteps', () => { +describe('query_histogram_range_steps', () => { describe('getHistogramIntervalRequest', () => { it('returns the request body for the histogram interval request', () => { const req = getHistogramIntervalRequest(params); @@ -43,6 +43,15 @@ describe('query_histogram_rangesteps', () => { 'processor.event': 'transaction', }, }, + { + range: { + '@timestamp': { + format: 'epoch_millis', + gte: 1577836800000, + lte: 1609459200000, + }, + }, + }, ], }, }, @@ -53,13 +62,14 @@ describe('query_histogram_rangesteps', () => { }); }); - describe('fetchTransactionDurationHistogramRangesteps', () => { + describe('fetchTransactionDurationHistogramRangeSteps', () => { it('fetches the range steps for the log histogram', async () => { const esClientSearchMock = jest.fn((req: estypes.SearchRequest): { body: estypes.SearchResponse; } => { return { body: ({ + hits: { total: { value: 10 } }, aggregations: { transaction_duration_max: { value: 10000, @@ -76,7 +86,7 @@ describe('query_histogram_rangesteps', () => { search: esClientSearchMock, } as unknown) as ElasticsearchClient; - const resp = await fetchTransactionDurationHistogramRangesteps( + const resp = await fetchTransactionDurationHistogramRangeSteps( esClientMock, params ); diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_rangesteps.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_range_steps.ts similarity index 83% rename from x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_rangesteps.ts rename to x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_range_steps.ts index e537165ca53f3..6ee5dd6bcdf83 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_rangesteps.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_range_steps.ts @@ -23,6 +23,14 @@ import type { SearchServiceParams } from '../../../../common/search_strategies/c import { getQueryWithParams } from './get_query_with_params'; +const getHistogramRangeSteps = (min: number, max: number, steps: number) => { + // A d3 based scale function as a helper to get equally distributed bins on a log scale. + const logFn = scaleLog().domain([min, max]).range([1, steps]); + return [...Array(steps).keys()] + .map(logFn.invert) + .map((d) => (isNaN(d) ? 0 : d)); +}; + export const getHistogramIntervalRequest = ( params: SearchServiceParams ): estypes.SearchRequest => ({ @@ -37,19 +45,24 @@ export const getHistogramIntervalRequest = ( }, }); -export const fetchTransactionDurationHistogramRangesteps = async ( +export const fetchTransactionDurationHistogramRangeSteps = async ( esClient: ElasticsearchClient, params: SearchServiceParams ): Promise => { + const steps = 100; + const resp = await esClient.search(getHistogramIntervalRequest(params)); + if ((resp.body.hits.total as estypes.SearchTotalHits).value === 0) { + return getHistogramRangeSteps(0, 1, 100); + } + if (resp.body.aggregations === undefined) { throw new Error( - 'fetchTransactionDurationHistogramInterval failed, did not return aggregations.' + 'fetchTransactionDurationHistogramRangeSteps failed, did not return aggregations.' ); } - const steps = 100; const min = (resp.body.aggregations .transaction_duration_min as estypes.AggregationsValueAggregate).value; const max = @@ -57,9 +70,5 @@ export const fetchTransactionDurationHistogramRangesteps = async ( .transaction_duration_max as estypes.AggregationsValueAggregate).value * 2; - // A d3 based scale function as a helper to get equally distributed bins on a log scale. - const logFn = scaleLog().domain([min, max]).range([1, steps]); - return [...Array(steps).keys()] - .map(logFn.invert) - .map((d) => (isNaN(d) ? 0 : d)); + return getHistogramRangeSteps(min, max, steps); }; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_percentiles.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_percentiles.test.ts index 0c319aee0fb2b..f0d01a4849f9f 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_percentiles.test.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_percentiles.test.ts @@ -14,7 +14,7 @@ import { getTransactionDurationPercentilesRequest, } from './query_percentiles'; -const params = { index: 'apm-*' }; +const params = { index: 'apm-*', start: '2020', end: '2021' }; describe('query_percentiles', () => { describe('getTransactionDurationPercentilesRequest', () => { @@ -41,10 +41,20 @@ describe('query_percentiles', () => { 'processor.event': 'transaction', }, }, + { + range: { + '@timestamp': { + format: 'epoch_millis', + gte: 1577836800000, + lte: 1609459200000, + }, + }, + }, ], }, }, size: 0, + track_total_hits: true, }, index: params.index, }); @@ -53,6 +63,7 @@ describe('query_percentiles', () => { describe('fetchTransactionDurationPercentiles', () => { it('fetches the percentiles', async () => { + const totalDocs = 10; const percentilesValues = { '1.0': 5.0, '5.0': 25.0, @@ -68,6 +79,7 @@ describe('query_percentiles', () => { } => { return { body: ({ + hits: { total: { value: totalDocs } }, aggregations: { transaction_duration_percentiles: { values: percentilesValues, @@ -86,7 +98,7 @@ describe('query_percentiles', () => { params ); - expect(resp).toEqual(percentilesValues); + expect(resp).toEqual({ percentiles: percentilesValues, totalDocs }); expect(esClientSearchMock).toHaveBeenCalledTimes(1); }); }); diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_percentiles.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_percentiles.ts index 18dcefb59a11a..c80f5d836c0ef 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_percentiles.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_percentiles.ts @@ -38,6 +38,7 @@ export const getTransactionDurationPercentilesRequest = ( return { index: params.index, body: { + track_total_hits: true, query, size: 0, aggs: { @@ -61,7 +62,7 @@ export const fetchTransactionDurationPercentiles = async ( percents?: number[], fieldName?: string, fieldValue?: string -): Promise> => { +): Promise<{ totalDocs: number; percentiles: Record }> => { const resp = await esClient.search( getTransactionDurationPercentilesRequest( params, @@ -71,14 +72,22 @@ export const fetchTransactionDurationPercentiles = async ( ) ); + // return early with no results if the search didn't return any documents + if ((resp.body.hits.total as estypes.SearchTotalHits).value === 0) { + return { totalDocs: 0, percentiles: {} }; + } + if (resp.body.aggregations === undefined) { throw new Error( 'fetchTransactionDurationPercentiles failed, did not return aggregations.' ); } - return ( - (resp.body.aggregations - .transaction_duration_percentiles as estypes.AggregationsTDigestPercentilesAggregate) - .values ?? {} - ); + + return { + totalDocs: (resp.body.hits.total as estypes.SearchTotalHits).value, + percentiles: + (resp.body.aggregations + .transaction_duration_percentiles as estypes.AggregationsTDigestPercentilesAggregate) + .values ?? {}, + }; }; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_ranges.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_ranges.test.ts index 9451928e47ded..7d18efc360563 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_ranges.test.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_ranges.test.ts @@ -14,7 +14,7 @@ import { getTransactionDurationRangesRequest, } from './query_ranges'; -const params = { index: 'apm-*' }; +const params = { index: 'apm-*', start: '2020', end: '2021' }; const rangeSteps = [1, 3, 5]; describe('query_ranges', () => { @@ -59,6 +59,15 @@ describe('query_ranges', () => { 'processor.event': 'transaction', }, }, + { + range: { + '@timestamp': { + format: 'epoch_millis', + gte: 1577836800000, + lte: 1609459200000, + }, + }, + }, ], }, }, diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/search_strategy.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/search_strategy.test.ts index 6d4bfcdde9994..09775cb2eb034 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/search_strategy.test.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/search_strategy.test.ts @@ -122,6 +122,8 @@ describe('APM Correlations search strategy', () => { } as unknown) as SearchStrategyDependencies; params = { index: 'apm-*', + start: '2020', + end: '2021', }; }); @@ -154,10 +156,22 @@ describe('APM Correlations search strategy', () => { }, query: { bool: { - filter: [{ term: { 'processor.event': 'transaction' } }], + filter: [ + { term: { 'processor.event': 'transaction' } }, + { + range: { + '@timestamp': { + format: 'epoch_millis', + gte: 1577836800000, + lte: 1609459200000, + }, + }, + }, + ], }, }, size: 0, + track_total_hits: true, }) ); }); @@ -167,11 +181,17 @@ describe('APM Correlations search strategy', () => { it('retrieves the current request', async () => { const searchStrategy = await apmCorrelationsSearchStrategyProvider(); const response = await searchStrategy - .search({ id: 'my-search-id', params }, {}, mockDeps) + .search({ params }, {}, mockDeps) .toPromise(); - expect(response).toEqual( - expect.objectContaining({ id: 'my-search-id' }) + const searchStrategyId = response.id; + + const response2 = await searchStrategy + .search({ id: searchStrategyId, params }, {}, mockDeps) + .toPromise(); + + expect(response2).toEqual( + expect.objectContaining({ id: searchStrategyId }) ); }); }); @@ -226,7 +246,7 @@ describe('APM Correlations search strategy', () => { expect(response2.id).toEqual(response1.id); expect(response2).toEqual( - expect.objectContaining({ loaded: 10, isRunning: false }) + expect.objectContaining({ loaded: 100, isRunning: false }) ); }); }); diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/search_strategy.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/search_strategy.ts index d6b4e0e7094b3..8f2e6913c0d06 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/search_strategy.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/search_strategy.ts @@ -41,14 +41,40 @@ export const apmCorrelationsSearchStrategyProvider = (): ISearchStrategy< throw new Error('Invalid request parameters.'); } - const id = request.id ?? uuid(); + // The function to fetch the current state of the async search service. + // This will be either an existing service for a follow up fetch or a new one for new requests. + let getAsyncSearchServiceState: ReturnType< + typeof asyncSearchServiceProvider + >; + + // If the request includes an ID, we require that the async search service already exists + // otherwise we throw an error. The client should never poll a service that's been cancelled or finished. + // This also avoids instantiating async search services when the service gets called with random IDs. + if (typeof request.id === 'string') { + const existingGetAsyncSearchServiceState = asyncSearchServiceMap.get( + request.id + ); - const getAsyncSearchServiceState = - asyncSearchServiceMap.get(id) ?? - asyncSearchServiceProvider(deps.esClient.asCurrentUser, request.params); + if (typeof existingGetAsyncSearchServiceState === 'undefined') { + throw new Error( + `AsyncSearchService with ID '${request.id}' does not exist.` + ); + } + + getAsyncSearchServiceState = existingGetAsyncSearchServiceState; + } else { + getAsyncSearchServiceState = asyncSearchServiceProvider( + deps.esClient.asCurrentUser, + request.params + ); + } + + // Reuse the request's id or create a new one. + const id = request.id ?? uuid(); const { error, + log, isRunning, loaded, started, @@ -76,6 +102,7 @@ export const apmCorrelationsSearchStrategyProvider = (): ISearchStrategy< isRunning, isPartial: isRunning, rawResponse: { + log, took, values, percentileThresholdValue, diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/utils/aggregation_utils.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/utils/aggregation_utils.test.ts index 63de0a59d4894..4313ad58ecbc0 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/utils/aggregation_utils.test.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/utils/aggregation_utils.test.ts @@ -14,6 +14,7 @@ describe('aggregation utils', () => { expect(expectations).toEqual([0, 0.5, 1]); expect(ranges).toEqual([{ to: 0 }, { from: 0, to: 1 }, { from: 1 }]); }); + it('returns expectations and ranges based on given percentiles #2', async () => { const { expectations, ranges } = computeExpectationsAndRanges([1, 3, 5]); expect(expectations).toEqual([1, 2, 4, 5]); @@ -24,6 +25,7 @@ describe('aggregation utils', () => { { from: 5 }, ]); }); + it('returns expectations and ranges with adjusted fractions', async () => { const { expectations, ranges } = computeExpectationsAndRanges([ 1, @@ -45,5 +47,97 @@ describe('aggregation utils', () => { { from: 5 }, ]); }); + + // TODO identify these results derived from the array of percentiles are usable with the ES correlation aggregation + it('returns expectation and ranges adjusted when percentiles have equal values', async () => { + const { expectations, ranges } = computeExpectationsAndRanges([ + 5000, + 5000, + 3090428, + 3090428, + 3090428, + 3618812, + 3618812, + 3618812, + 3618812, + 3696636, + 3696636, + 3696636, + 3696636, + 3696636, + 3696636, + ]); + expect(expectations).toEqual([ + 5000, + 1856256.7999999998, + 3392361.714285714, + 3665506.4, + 3696636, + ]); + expect(ranges).toEqual([ + { + to: 5000, + }, + { + from: 5000, + to: 5000, + }, + { + from: 5000, + to: 3090428, + }, + { + from: 3090428, + to: 3090428, + }, + { + from: 3090428, + to: 3090428, + }, + { + from: 3090428, + to: 3618812, + }, + { + from: 3618812, + to: 3618812, + }, + { + from: 3618812, + to: 3618812, + }, + { + from: 3618812, + to: 3618812, + }, + { + from: 3618812, + to: 3696636, + }, + { + from: 3696636, + to: 3696636, + }, + { + from: 3696636, + to: 3696636, + }, + { + from: 3696636, + to: 3696636, + }, + { + from: 3696636, + to: 3696636, + }, + { + from: 3696636, + to: 3696636, + }, + { + from: 3696636, + }, + ]); + }); }); }); diff --git a/x-pack/test/apm_api_integration/tests/correlations/latency_ml.ts b/x-pack/test/apm_api_integration/tests/correlations/latency_ml.ts new file mode 100644 index 0000000000000..38e2ab5449837 --- /dev/null +++ b/x-pack/test/apm_api_integration/tests/correlations/latency_ml.ts @@ -0,0 +1,266 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import request from 'superagent'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { registry } from '../../common/registry'; + +import { PartialSearchRequest } from '../../../../plugins/apm/server/lib/search_strategies/correlations/search_strategy'; + +function parseBfetchResponse(resp: request.Response): Array> { + return resp.text + .trim() + .split('\n') + .map((item) => JSON.parse(item)); +} + +export default function ApiTest({ getService }: FtrProviderContext) { + const retry = getService('retry'); + const supertest = getService('supertest'); + + const getRequestBody = () => { + const partialSearchRequest: PartialSearchRequest = { + params: { + index: 'apm-*', + environment: 'ENVIRONMENT_ALL', + start: '2020', + end: '2021', + percentileThreshold: 95, + }, + }; + + return { + batch: [ + { + request: partialSearchRequest, + options: { strategy: 'apmCorrelationsSearchStrategy' }, + }, + ], + }; + }; + + registry.when( + 'correlations latency_ml overall without data', + { config: 'trial', archives: [] }, + () => { + it('handles the empty state', async () => { + const intialResponse = await supertest + .post(`/internal/bsearch`) + .set('kbn-xsrf', 'foo') + .send(getRequestBody()); + + expect(intialResponse.status).to.eql( + 200, + `Expected status to be '200', got '${intialResponse.status}'` + ); + expect(intialResponse.body).to.eql( + {}, + `Expected response body to be an empty object, actual response is in the text attribute. Got: '${JSON.stringify( + intialResponse.body + )}'` + ); + + const body = parseBfetchResponse(intialResponse)[0]; + + expect(typeof body.result).to.be('object'); + const { result } = body; + + expect(typeof result?.id).to.be('string'); + + // pass on id for follow up queries + const searchStrategyId = result.id; + + // follow up request body including search strategy ID + const reqBody = getRequestBody(); + reqBody.batch[0].request.id = searchStrategyId; + + let followUpResponse: Record = {}; + + // continues querying until the search strategy finishes + await retry.waitForWithTimeout( + 'search strategy eventually completes and returns full results', + 5000, + async () => { + const response = await supertest + .post(`/internal/bsearch`) + .set('kbn-xsrf', 'foo') + .send(reqBody); + + followUpResponse = parseBfetchResponse(response)[0]; + + return ( + followUpResponse?.result?.isRunning === false || followUpResponse?.error !== undefined + ); + } + ); + + expect(followUpResponse?.error).to.eql( + undefined, + `search strategy should not return an error, got: ${JSON.stringify( + followUpResponse?.error + )}` + ); + + const followUpResult = followUpResponse.result; + expect(followUpResult?.isRunning).to.eql(false, 'search strategy should not be running'); + expect(followUpResult?.isPartial).to.eql( + false, + 'search strategy result should not be partial' + ); + expect(followUpResult?.id).to.eql( + searchStrategyId, + 'search strategy id should match original id' + ); + expect(followUpResult?.isRestored).to.eql( + true, + 'search strategy response should be restored' + ); + expect(followUpResult?.loaded).to.eql(100, 'loaded state should be 100'); + expect(followUpResult?.total).to.eql(100, 'total state should be 100'); + + expect(typeof followUpResult?.rawResponse).to.be('object'); + + const { rawResponse: finalRawResponse } = followUpResult; + + expect(typeof finalRawResponse?.took).to.be('number'); + expect(finalRawResponse?.percentileThresholdValue).to.be(undefined); + expect(finalRawResponse?.overallHistogram).to.be(undefined); + expect(finalRawResponse?.values.length).to.be(0); + expect(finalRawResponse?.log.map((d: string) => d.split(': ')[1])).to.eql([ + 'Fetched 95th percentile value of undefined based on 0 documents.', + 'Abort service since percentileThresholdValue could not be determined.', + ]); + }); + } + ); + + registry.when( + 'Correlations latency_ml with data and opbeans-node args', + { config: 'trial', archives: ['8.0.0'] }, + () => { + // putting this into a single `it` because the responses depend on each other + it('queries the search strategy and returns results', async () => { + const intialResponse = await supertest + .post(`/internal/bsearch`) + .set('kbn-xsrf', 'foo') + .send(getRequestBody()); + + expect(intialResponse.status).to.eql( + 200, + `Expected status to be '200', got '${intialResponse.status}'` + ); + expect(intialResponse.body).to.eql( + {}, + `Expected response body to be an empty object, actual response is in the text attribute. Got: '${JSON.stringify( + intialResponse.body + )}'` + ); + + const body = parseBfetchResponse(intialResponse)[0]; + + expect(typeof body?.result).to.be('object'); + const { result } = body; + + expect(typeof result?.id).to.be('string'); + + // pass on id for follow up queries + const searchStrategyId = result.id; + + expect(result?.loaded).to.be(0); + expect(result?.total).to.be(100); + expect(result?.isRunning).to.be(true); + expect(result?.isPartial).to.be(true); + expect(result?.isRestored).to.eql( + false, + `Expected response result to be not restored. Got: '${result?.isRestored}'` + ); + expect(typeof result?.rawResponse).to.be('object'); + + const { rawResponse } = result; + + expect(typeof rawResponse?.took).to.be('number'); + expect(rawResponse?.values).to.eql([]); + + // follow up request body including search strategy ID + const reqBody = getRequestBody(); + reqBody.batch[0].request.id = searchStrategyId; + + let followUpResponse: Record = {}; + + // continues querying until the search strategy finishes + await retry.waitForWithTimeout( + 'search strategy eventually completes and returns full results', + 5000, + async () => { + const response = await supertest + .post(`/internal/bsearch`) + .set('kbn-xsrf', 'foo') + .send(reqBody); + followUpResponse = parseBfetchResponse(response)[0]; + + return ( + followUpResponse?.result?.isRunning === false || followUpResponse?.error !== undefined + ); + } + ); + + expect(followUpResponse?.error).to.eql( + undefined, + `Finished search strategy should not return an error, got: ${JSON.stringify( + followUpResponse?.error + )}` + ); + + const followUpResult = followUpResponse.result; + expect(followUpResult?.isRunning).to.eql( + false, + `Expected finished result not to be running. Got: ${followUpResult?.isRunning}` + ); + expect(followUpResult?.isPartial).to.eql( + false, + `Expected finished result not to be partial. Got: ${followUpResult?.isPartial}` + ); + expect(followUpResult?.id).to.be(searchStrategyId); + expect(followUpResult?.isRestored).to.be(true); + expect(followUpResult?.loaded).to.be(100); + expect(followUpResult?.total).to.be(100); + + expect(typeof followUpResult?.rawResponse).to.be('object'); + + const { rawResponse: finalRawResponse } = followUpResult; + + expect(typeof finalRawResponse?.took).to.be('number'); + expect(finalRawResponse?.percentileThresholdValue).to.be(1309695.875); + expect(finalRawResponse?.overallHistogram.length).to.be(101); + + expect(finalRawResponse?.values.length).to.eql( + 13, + `Expected 13 identified correlations, got ${finalRawResponse?.values.length}.` + ); + expect(finalRawResponse?.log.map((d: string) => d.split(': ')[1])).to.eql([ + 'Fetched 95th percentile value of 1309695.875 based on 1244 documents.', + 'Loaded histogram range steps.', + 'Loaded overall histogram chart data.', + 'Loaded percentiles.', + 'Identified 69 fieldCandidates.', + 'Identified 379 fieldValuePairs.', + 'Loaded fractions and totalDocCount of 1244.', + 'Identified 13 significant correlations out of 379 field/value pairs.', + ]); + + const correlation = finalRawResponse?.values[0]; + expect(typeof correlation).to.be('object'); + expect(correlation?.field).to.be('transaction.result'); + expect(correlation?.value).to.be('success'); + expect(correlation?.correlation).to.be(0.6275246559191225); + expect(correlation?.ksTest).to.be(4.806503252860024e-13); + expect(correlation?.histogram.length).to.be(101); + }); + } + ); +} diff --git a/x-pack/test/apm_api_integration/tests/index.ts b/x-pack/test/apm_api_integration/tests/index.ts index 813e0e4f3cdb8..a00fa1723fa3e 100644 --- a/x-pack/test/apm_api_integration/tests/index.ts +++ b/x-pack/test/apm_api_integration/tests/index.ts @@ -32,6 +32,10 @@ export default function apmApiIntegrationTests(providerContext: FtrProviderConte loadTestFile(require.resolve('./correlations/latency_slow_transactions')); }); + describe('correlations/latency_ml', function () { + loadTestFile(require.resolve('./correlations/latency_ml')); + }); + describe('correlations/latency_overall', function () { loadTestFile(require.resolve('./correlations/latency_overall')); });