diff --git a/x-pack/solutions/observability/plugins/apm/common/connections.ts b/x-pack/solutions/observability/plugins/apm/common/connections.ts index 1253d10a94842..9bf303a1bc74f 100644 --- a/x-pack/solutions/observability/plugins/apm/common/connections.ts +++ b/x-pack/solutions/observability/plugins/apm/common/connections.ts @@ -36,19 +36,19 @@ export type Node = ServiceNode | DependencyNode; export interface ConnectionStats { latency: { value: number | null; - timeseries: Coordinate[]; + timeseries?: Coordinate[]; }; throughput: { value: number | null; - timeseries: Coordinate[]; + timeseries?: Coordinate[]; }; errorRate: { value: number | null; - timeseries: Coordinate[]; + timeseries?: Coordinate[]; }; totalTime: { value: number | null; - timeseries: Coordinate[]; + timeseries?: Coordinate[]; }; } diff --git a/x-pack/solutions/observability/plugins/apm/public/components/app/dependencies_inventory/dependencies_inventory_table/index.tsx b/x-pack/solutions/observability/plugins/apm/public/components/app/dependencies_inventory/dependencies_inventory_table/index.tsx index 1d007f67921e4..284d1993e4e57 100644 --- a/x-pack/solutions/observability/plugins/apm/public/components/app/dependencies_inventory/dependencies_inventory_table/index.tsx +++ b/x-pack/solutions/observability/plugins/apm/public/components/app/dependencies_inventory/dependencies_inventory_table/index.tsx @@ -4,26 +4,45 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - -import { METRIC_TYPE } from '@kbn/analytics'; -import { i18n } from '@kbn/i18n'; -import React, { useEffect } from 'react'; import { useEuiTheme } from '@elastic/eui'; import { css } from '@emotion/react'; -import { useUiTracker } from '@kbn/observability-shared-plugin/public'; +import { METRIC_TYPE } from '@kbn/analytics'; import { usePerformanceContext } from '@kbn/ebt-tools'; -import { isTimeComparison } from '../../../shared/time_comparison/get_comparison_options'; -import { getNodeName, NodeType } from '../../../../../common/connections'; +import { i18n } from '@kbn/i18n'; +import { useUiTracker } from '@kbn/observability-shared-plugin/public'; +import { orderBy } from 'lodash'; +import React, { useEffect, useMemo } from 'react'; +import { v4 as uuidv4 } from 'uuid'; +import { NodeType, getNodeName } from '../../../../../common/connections'; import { useApmParams } from '../../../../hooks/use_apm_params'; -import { FETCH_STATUS, useFetcher } from '../../../../hooks/use_fetcher'; +import { FETCH_STATUS, isPending, useFetcher } from '../../../../hooks/use_fetcher'; import { useTimeRange } from '../../../../hooks/use_time_range'; +import type { DependenciesItem } from '../../../shared/dependencies_table'; +import { + DependenciesTable, + INITIAL_SORTING_FIELD, + INITIAL_SORTING_DIRECTION, +} from '../../../shared/dependencies_table'; import { DependencyLink } from '../../../shared/links/dependency_link'; -import { DependenciesTable } from '../../../shared/dependencies_table'; +import { isTimeComparison } from '../../../shared/time_comparison/get_comparison_options'; import { RandomSamplerBadge } from '../random_sampler_badge'; +const INITIAL_PAGE_SIZE = 25; + export function DependenciesInventoryTable() { const { - query: { rangeFrom, rangeTo, environment, kuery, comparisonEnabled, offset }, + query: { + rangeFrom, + rangeTo, + environment, + kuery, + comparisonEnabled, + offset, + page = 0, + pageSize = INITIAL_PAGE_SIZE, + sortDirection = INITIAL_SORTING_DIRECTION, + sortField = INITIAL_SORTING_FIELD, + }, } = useApmParams('/dependencies/inventory'); const { onPageReady } = usePerformanceContext(); const { start, end } = useTimeRange({ rangeFrom, rangeTo }); @@ -38,19 +57,63 @@ export function DependenciesInventoryTable() { } return callApmApi('GET /internal/apm/dependencies/top_dependencies', { - params: { - query: { - start, - end, - environment, - numBuckets: 8, - offset: comparisonEnabled && isTimeComparison(offset) ? offset : undefined, - kuery, - }, - }, + params: { query: { start, end, environment, numBuckets: 8, kuery } }, + }).then((response) => { + return { + ...response, + requestId: uuidv4(), + }; }); }, - [start, end, environment, offset, kuery, comparisonEnabled] + [start, end, environment, kuery] + ); + + const visibleDependenciesNames = useMemo( + () => + data?.dependencies + ? orderBy( + data.dependencies.map((item) => ({ + name: getNodeName(item.location), + impact: item.currentStats.impact, + latency: item.currentStats.latency.value, + throughput: item.currentStats.throughput.value, + failureRate: item.currentStats.errorRate.value, + })), + sortField, + sortDirection + ) + .slice(page * pageSize, (page + 1) * pageSize) + .map(({ name }) => name) + .sort() + : undefined, + [data?.dependencies, page, pageSize, sortDirection, sortField] + ); + + const { data: timeseriesData, status: timeseriesStatus } = useFetcher( + (callApmApi) => { + if (data?.requestId && visibleDependenciesNames?.length) { + return callApmApi('POST /internal/apm/dependencies/top_dependencies/statistics', { + params: { + query: { + start, + end, + environment, + numBuckets: 8, + offset: comparisonEnabled && isTimeComparison(offset) ? offset : undefined, + kuery, + }, + body: { + dependencyNames: JSON.stringify(visibleDependenciesNames), + }, + }, + }); + } + }, + // Disables exhaustive deps because the statistics api must only be called when the rendered items changed or when comparison is toggled or changed. + // eslint-disable-next-line react-hooks/exhaustive-deps + [data?.requestId, visibleDependenciesNames, comparisonEnabled, offset], + // Do not invalidate this API call when the refresh button is clicked + { skipTimeRangeRefreshUpdate: true } ); useEffect(() => { @@ -64,46 +127,91 @@ export function DependenciesInventoryTable() { } }, [status, onPageReady, rangeFrom, rangeTo]); - const dependencies = - data?.dependencies.map((dependency) => { - const { location } = dependency; - const name = getNodeName(location); - - if (location.type !== NodeType.dependency) { - throw new Error('Expected a dependency node'); - } - const link = ( - { - trackEvent({ - app: 'apm', - metricType: METRIC_TYPE.CLICK, - metric: 'dependencies_inventory_to_dependency_detail', - }); - }} - /> - ); + const dependencies: DependenciesItem[] = useMemo( + () => + data?.dependencies.map((dependency) => { + const { location } = dependency; + const name = getNodeName(location); - return { - name, - currentStats: dependency.currentStats, - previousStats: dependency.previousStats, - link, - }; - }) ?? []; + if (location.type !== NodeType.dependency) { + throw new Error('Expected a dependency node'); + } + const link = ( + { + trackEvent({ + app: 'apm', + metricType: METRIC_TYPE.CLICK, + metric: 'dependencies_inventory_to_dependency_detail', + }); + }} + /> + ); + return { + name, + currentStats: { + impact: dependency.currentStats.impact, + totalTime: { value: dependency.currentStats.totalTime.value }, + latency: { + value: dependency.currentStats.latency.value, + timeseries: timeseriesData?.currentTimeseries[name]?.latency, + }, + throughput: { + value: dependency.currentStats.throughput.value, + timeseries: timeseriesData?.currentTimeseries[name]?.throughput, + }, + errorRate: { + value: dependency.currentStats.errorRate.value, + timeseries: timeseriesData?.currentTimeseries[name]?.errorRate, + }, + }, + previousStats: { + impact: dependency.previousStats?.impact ?? 0, + totalTime: { value: dependency.previousStats?.totalTime.value ?? null }, + latency: { + value: dependency.previousStats?.latency.value ?? null, + timeseries: timeseriesData?.comparisonTimeseries?.[name]?.latency, + }, + throughput: { + value: dependency.previousStats?.throughput.value ?? null, + timeseries: timeseriesData?.comparisonTimeseries?.[name]?.throughput, + }, + errorRate: { + value: dependency.previousStats?.errorRate.value ?? null, + timeseries: timeseriesData?.comparisonTimeseries?.[name]?.errorRate, + }, + }, + link, + }; + }) ?? [], + [ + comparisonEnabled, + data?.dependencies, + environment, + kuery, + offset, + rangeFrom, + rangeTo, + timeseriesData?.comparisonTimeseries, + timeseriesData?.currentTimeseries, + trackEvent, + ] + ); const showRandomSamplerBadge = data?.sampled && status === FETCH_STATUS.SUCCESS; + const fetchingStatus = + isPending(status) || isPending(timeseriesStatus) ? FETCH_STATUS.LOADING : FETCH_STATUS.SUCCESS; return ( <> @@ -121,9 +229,9 @@ export function DependenciesInventoryTable() { nameColumnTitle={i18n.translate('xpack.apm.dependenciesInventory.dependencyTableColumn', { defaultMessage: 'Dependency', })} - status={status} + status={fetchingStatus} compact={false} - initialPageSize={25} + initialPageSize={INITIAL_PAGE_SIZE} /> ); diff --git a/x-pack/solutions/observability/plugins/apm/public/components/shared/dependencies_table/get_span_metric_columns.tsx b/x-pack/solutions/observability/plugins/apm/public/components/shared/dependencies_table/get_span_metric_columns.tsx index f6a33c9afa3cb..fb7d409333480 100644 --- a/x-pack/solutions/observability/plugins/apm/public/components/shared/dependencies_table/get_span_metric_columns.tsx +++ b/x-pack/solutions/observability/plugins/apm/public/components/shared/dependencies_table/get_span_metric_columns.tsx @@ -28,17 +28,17 @@ export interface SpanMetricGroup { impact: number | null; currentStats: | { - latency: Coordinate[]; - throughput: Coordinate[]; - failureRate: Coordinate[]; + latency?: Coordinate[]; + throughput?: Coordinate[]; + failureRate?: Coordinate[]; } | undefined; previousStats: | { - latency: Coordinate[]; - throughput: Coordinate[]; - failureRate: Coordinate[]; - impact: number; + latency?: Coordinate[]; + throughput?: Coordinate[]; + failureRate?: Coordinate[]; + impact?: number; } | undefined; } diff --git a/x-pack/solutions/observability/plugins/apm/public/components/shared/dependencies_table/index.tsx b/x-pack/solutions/observability/plugins/apm/public/components/shared/dependencies_table/index.tsx index 6266091a3c21b..bfde5668ff2d9 100644 --- a/x-pack/solutions/observability/plugins/apm/public/components/shared/dependencies_table/index.tsx +++ b/x-pack/solutions/observability/plugins/apm/public/components/shared/dependencies_table/index.tsx @@ -7,7 +7,7 @@ import { EuiFlexGroup, EuiFlexItem, EuiTitle } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; -import React from 'react'; +import React, { useMemo } from 'react'; import type { ConnectionStatsItemWithComparisonData } from '../../../../common/connections'; import { useBreakpoints } from '../../../hooks/use_breakpoints'; import { FETCH_STATUS } from '../../../hooks/use_fetcher'; @@ -24,6 +24,8 @@ export type DependenciesItem = Omit void; } -type FormattedSpanMetricGroup = SpanMetricGroup & { +export type FormattedSpanMetricGroup = SpanMetricGroup & { name: string; link: React.ReactElement; }; -export function DependenciesTable(props: Props) { - const { - dependencies, - fixedHeight, - link, - title, - nameColumnTitle, - status, - compact = true, - showPerPageOptions = true, - initialPageSize, - showSparkPlots, - } = props; - +export function DependenciesTable({ + dependencies, + fixedHeight, + link, + title, + nameColumnTitle, + status, + compact = true, + showPerPageOptions = true, + initialPageSize, + showSparkPlots, + onChangeRenderedItems, +}: Props) { const { isLarge } = useBreakpoints(); const shouldShowSparkPlots = showSparkPlots ?? !isLarge; - const items: FormattedSpanMetricGroup[] = dependencies.map((dependency) => ({ - name: dependency.name, - link: dependency.link, - latency: dependency.currentStats.latency.value, - throughput: dependency.currentStats.throughput.value, - failureRate: dependency.currentStats.errorRate.value, - impact: dependency.currentStats.impact, - currentStats: { - latency: dependency.currentStats.latency.timeseries, - throughput: dependency.currentStats.throughput.timeseries, - failureRate: dependency.currentStats.errorRate.timeseries, - }, - previousStats: dependency.previousStats - ? { - latency: dependency.previousStats.latency.timeseries, - throughput: dependency.previousStats.throughput.timeseries, - failureRate: dependency.previousStats.errorRate.timeseries, - impact: dependency.previousStats.impact, - } - : undefined, - })); + const items: FormattedSpanMetricGroup[] = useMemo( + () => + dependencies.map((dependency) => ({ + name: dependency.name, + link: dependency.link, + latency: dependency.currentStats.latency.value, + throughput: dependency.currentStats.throughput.value, + failureRate: dependency.currentStats.errorRate.value, + impact: dependency.currentStats.impact, + currentStats: { + latency: dependency.currentStats.latency.timeseries, + throughput: dependency.currentStats.throughput.timeseries, + failureRate: dependency.currentStats.errorRate.timeseries, + }, + previousStats: dependency.previousStats + ? { + latency: dependency.previousStats.latency.timeseries, + throughput: dependency.previousStats.throughput.timeseries, + failureRate: dependency.previousStats.errorRate.timeseries, + impact: dependency.previousStats.impact, + } + : undefined, + })), + [dependencies] + ); const columns: Array> = [ { @@ -133,11 +139,12 @@ export function DependenciesTable(props: Props) { columns={columns} items={items} noItemsMessage={noItemsMessage} - initialSortField="impact" - initialSortDirection="desc" + initialSortField={INITIAL_SORTING_FIELD} + initialSortDirection={INITIAL_SORTING_DIRECTION} pagination={true} showPerPageOptions={showPerPageOptions} initialPageSize={initialPageSize} + onChangeRenderedItems={onChangeRenderedItems} /> diff --git a/x-pack/solutions/observability/plugins/apm/public/hooks/use_fetcher.tsx b/x-pack/solutions/observability/plugins/apm/public/hooks/use_fetcher.tsx index 419b540c24d3c..498b623f760ec 100644 --- a/x-pack/solutions/observability/plugins/apm/public/hooks/use_fetcher.tsx +++ b/x-pack/solutions/observability/plugins/apm/public/hooks/use_fetcher.tsx @@ -87,6 +87,7 @@ export function useFetcher( options: { preservePreviousData?: boolean; showToastOnError?: boolean; + skipTimeRangeRefreshUpdate?: boolean; } = {} ): FetcherResult> & { refetch: () => void } { const { notifications } = useKibana(); @@ -99,6 +100,21 @@ export function useFetcher( const { timeRangeId } = useTimeRangeId(); const { addInspectorRequest } = useInspectorContext(); + const deps = useMemo(() => { + const _deps = [counter, preservePreviousData, showToastOnError, ...fnDeps]; + if (options.skipTimeRangeRefreshUpdate !== true) { + _deps.push(timeRangeId); + } + return _deps; + }, [ + counter, + fnDeps, + options.skipTimeRangeRefreshUpdate, + preservePreviousData, + showToastOnError, + timeRangeId, + ]); + useEffect(() => { let controller: AbortController = new AbortController(); @@ -176,14 +192,7 @@ export function useFetcher( controller.abort(); }; /* eslint-disable react-hooks/exhaustive-deps */ - }, [ - counter, - preservePreviousData, - timeRangeId, - showToastOnError, - ...fnDeps, - /* eslint-enable react-hooks/exhaustive-deps */ - ]); + }, deps); return useMemo(() => { return { diff --git a/x-pack/solutions/observability/plugins/apm/server/lib/connections/get_connection_stats/get_stats.ts b/x-pack/solutions/observability/plugins/apm/server/lib/connections/get_connection_stats/get_stats.ts index 8f55d921ec63d..5fe4fe2d4bf2b 100644 --- a/x-pack/solutions/observability/plugins/apm/server/lib/connections/get_connection_stats/get_stats.ts +++ b/x-pack/solutions/observability/plugins/apm/server/lib/connections/get_connection_stats/get_stats.ts @@ -5,7 +5,6 @@ * 2.0. */ -import { sum } from 'lodash'; import objectHash from 'object-hash'; import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; import { rangeQuery } from '@kbn/observability-plugin/server'; @@ -41,6 +40,7 @@ export const getStats = async ({ filter, numBuckets, offset, + withTimeseries, }: { apmEventClient: APMEventClient; start: number; @@ -48,6 +48,7 @@ export const getStats = async ({ filter: QueryDslQueryContainer[]; numBuckets: number; offset?: string; + withTimeseries: boolean; }) => { const { offsetInMs, startWithOffset, endWithOffset } = getOffsetInMs({ start, @@ -61,6 +62,7 @@ export const getStats = async ({ endWithOffset, filter, numBuckets, + withTimeseries, }); return ( @@ -85,27 +87,15 @@ export const getStats = async ({ type: NodeType.dependency as const, }, value: { - count: sum(bucket.timeseries.buckets.map((dateBucket) => dateBucket.count.value ?? 0)), - latency_sum: sum( - bucket.timeseries.buckets.map((dateBucket) => dateBucket.latency_sum.value ?? 0) - ), - error_count: sum( - bucket.timeseries.buckets.flatMap( - (dateBucket) => - dateBucket[EVENT_OUTCOME].buckets.find( - (outcomeBucket) => outcomeBucket.key === EventOutcome.failure - )?.count.value ?? 0 - ) - ), + count: bucket.doc_count ?? 0, + latency_sum: bucket.total_latency_sum.value ?? 0, + error_count: bucket.error_count.doc_count ?? 0, }, - timeseries: bucket.timeseries.buckets.map((dateBucket) => ({ + timeseries: bucket.timeseries?.buckets.map((dateBucket) => ({ x: dateBucket.key + offsetInMs, - count: dateBucket.count.value ?? 0, - latency_sum: dateBucket.latency_sum.value ?? 0, - error_count: - dateBucket[EVENT_OUTCOME].buckets.find( - (outcomeBucket) => outcomeBucket.key === EventOutcome.failure - )?.count.value ?? 0, + count: dateBucket.doc_count ?? 0, + latency_sum: dateBucket.total_latency_sum.value ?? 0, + error_count: dateBucket.error_count.doc_count ?? 0, })), }; }) ?? [] @@ -118,6 +108,7 @@ async function getConnectionStats({ endWithOffset, filter, numBuckets, + withTimeseries, }: { apmEventClient: APMEventClient; startWithOffset: number; @@ -125,7 +116,27 @@ async function getConnectionStats({ filter: QueryDslQueryContainer[]; numBuckets: number; after?: { serviceName: string | number; dependencyName: string | number }; + withTimeseries: boolean; + dependencyNames?: string[]; }) { + const statsAggs = { + total_latency_sum: { + sum: { + field: SPAN_DESTINATION_SERVICE_RESPONSE_TIME_SUM, + }, + }, + total_latency_count: { + sum: { + field: SPAN_DESTINATION_SERVICE_RESPONSE_TIME_COUNT, + }, + }, + error_count: { + filter: { + bool: { filter: [{ terms: { [EVENT_OUTCOME]: [EventOutcome.failure] } }] }, + }, + }, + }; + return apmEventClient.search('get_connection_stats', { apm: { sources: [ @@ -191,55 +202,27 @@ async function getConnectionStats({ }, }, }, - total_latency_sum: { - sum: { - field: SPAN_DESTINATION_SERVICE_RESPONSE_TIME_SUM, - }, - }, - total_latency_count: { - sum: { - field: SPAN_DESTINATION_SERVICE_RESPONSE_TIME_COUNT, - }, - }, - timeseries: { - date_histogram: { - field: '@timestamp', - fixed_interval: getBucketSize({ - start: startWithOffset, - end: endWithOffset, - numBuckets, - minBucketSize: 60, - }).intervalString, - extended_bounds: { - min: startWithOffset, - max: endWithOffset, - }, - }, - aggs: { - latency_sum: { - sum: { - field: SPAN_DESTINATION_SERVICE_RESPONSE_TIME_SUM, - }, - }, - count: { - sum: { - field: SPAN_DESTINATION_SERVICE_RESPONSE_TIME_COUNT, - }, - }, - [EVENT_OUTCOME]: { - terms: { - field: EVENT_OUTCOME, - }, - aggs: { - count: { - sum: { - field: SPAN_DESTINATION_SERVICE_RESPONSE_TIME_COUNT, + ...statsAggs, + ...(withTimeseries + ? { + timeseries: { + date_histogram: { + field: '@timestamp', + fixed_interval: getBucketSize({ + start: startWithOffset, + end: endWithOffset, + numBuckets, + minBucketSize: 60, + }).intervalString, + extended_bounds: { + min: startWithOffset, + max: endWithOffset, }, }, + aggs: statsAggs, }, - }, - }, - }, + } + : undefined), }, }, }, diff --git a/x-pack/solutions/observability/plugins/apm/server/lib/connections/get_connection_stats/index.ts b/x-pack/solutions/observability/plugins/apm/server/lib/connections/get_connection_stats/index.ts index 5833c1518fd23..713a6aa5c86de 100644 --- a/x-pack/solutions/observability/plugins/apm/server/lib/connections/get_connection_stats/index.ts +++ b/x-pack/solutions/observability/plugins/apm/server/lib/connections/get_connection_stats/index.ts @@ -5,16 +5,16 @@ * 2.0. */ -import type { ValuesType } from 'utility-types'; -import { merge } from 'lodash'; import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; +import { merge } from 'lodash'; +import type { ValuesType } from 'utility-types'; import { joinByKey } from '../../../../common/utils/join_by_key'; -import { getStats } from './get_stats'; -import { getDestinationMap } from './get_destination_map'; -import { calculateThroughputWithRange } from '../../helpers/calculate_throughput'; import { withApmSpan } from '../../../utils/with_apm_span'; +import { calculateThroughputWithRange } from '../../helpers/calculate_throughput'; import type { APMEventClient } from '../../helpers/create_es_client/create_apm_event_client'; import type { RandomSampler } from '../../helpers/get_random_sampler'; +import { getDestinationMap } from './get_destination_map'; +import { getStats } from './get_stats'; export function getConnectionStats({ apmEventClient, @@ -25,6 +25,7 @@ export function getConnectionStats({ collapseBy, offset, randomSampler, + withTimeseries = true, }: { apmEventClient: APMEventClient; start: number; @@ -34,6 +35,7 @@ export function getConnectionStats({ collapseBy: 'upstream' | 'downstream'; offset?: string; randomSampler: RandomSampler; + withTimeseries?: boolean; }) { return withApmSpan('get_connection_stats_and_map', async () => { const [allMetrics, { nodesBydependencyName: destinationMap, sampled }] = await Promise.all([ @@ -44,6 +46,7 @@ export function getConnectionStats({ filter, numBuckets, offset, + withTimeseries, }), getDestinationMap({ apmEventClient, @@ -84,12 +87,15 @@ export function getConnectionStats({ latency_sum: prev.value.latency_sum + current.value.latency_sum, error_count: prev.value.error_count + current.value.error_count, }, - timeseries: joinByKey([...prev.timeseries, ...current.timeseries], 'x', (a, b) => ({ - x: a.x, - count: a.count + b.count, - latency_sum: a.latency_sum + b.latency_sum, - error_count: a.error_count + b.error_count, - })), + timeseries: + prev.timeseries && current.timeseries + ? joinByKey([...prev.timeseries, ...current.timeseries], 'x', (a, b) => ({ + x: a.x, + count: a.count + b.count, + latency_sum: a.latency_sum + b.latency_sum, + error_count: a.error_count + b.error_count, + })) + : undefined, }; }, { @@ -108,14 +114,14 @@ export function getConnectionStats({ mergedStats.value.count > 0 ? mergedStats.value.latency_sum / mergedStats.value.count : null, - timeseries: mergedStats.timeseries.map((point) => ({ + timeseries: mergedStats.timeseries?.map((point) => ({ x: point.x, y: point.count > 0 ? point.latency_sum / point.count : null, })), }, totalTime: { value: mergedStats.value.latency_sum, - timeseries: mergedStats.timeseries.map((point) => ({ + timeseries: mergedStats.timeseries?.map((point) => ({ x: point.x, y: point.latency_sum, })), @@ -129,7 +135,7 @@ export function getConnectionStats({ value: mergedStats.value.count, }) : null, - timeseries: mergedStats.timeseries.map((point) => ({ + timeseries: mergedStats.timeseries?.map((point) => ({ x: point.x, y: point.count > 0 @@ -146,7 +152,7 @@ export function getConnectionStats({ mergedStats.value.count > 0 ? (mergedStats.value.error_count ?? 0) / mergedStats.value.count : null, - timeseries: mergedStats.timeseries.map((point) => ({ + timeseries: mergedStats.timeseries?.map((point) => ({ x: point.x, y: point.count > 0 ? (point.error_count ?? 0) / point.count : null, })), diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/dependencies/get_dependencies_timeseries_statistics.test.ts b/x-pack/solutions/observability/plugins/apm/server/routes/dependencies/get_dependencies_timeseries_statistics.test.ts new file mode 100644 index 0000000000000..37582a8b2f56d --- /dev/null +++ b/x-pack/solutions/observability/plugins/apm/server/routes/dependencies/get_dependencies_timeseries_statistics.test.ts @@ -0,0 +1,109 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { DependenciesTimeseriesBuckes } from './get_dependencies_timeseries_statistics'; +import { parseDependenciesStats } from './get_dependencies_timeseries_statistics'; + +describe('parseDependenciesStats', () => { + const offsetInMs = 1000; + + test('should parse dependency stats correctly with all values', () => { + const dependencies = [ + { + key: 'service-A', + timeseries: { + buckets: [ + { + key: 1700000000000, + doc_count: 10, + total_count: { value: 10 }, + failures: { total_count: { value: 2 }, doc_count: 2 }, + latency_sum: { value: 5000 }, + latency_count: { value: 10 }, + throughput: { value: 50 }, + }, + ], + }, + }, + ] as DependenciesTimeseriesBuckes; + + const result = parseDependenciesStats({ dependencies, offsetInMs }); + + expect(result).toEqual({ + 'service-A': { + latency: [{ x: 1700000001000, y: 500 }], + errorRate: [{ x: 1700000001000, y: 0.2 }], + throughput: [{ x: 1700000001000, y: 50 }], + }, + }); + }); + + test('should handle missing optional values correctly', () => { + const dependencies = [ + { + key: 'service-B', + timeseries: { + buckets: [ + { + key: 1700000000000, + doc_count: 5, + failures: { doc_count: 1 }, + latency_sum: { value: 2000 }, + latency_count: { value: 5 }, + throughput: {}, + }, + ], + }, + }, + ] as DependenciesTimeseriesBuckes; + + const result = parseDependenciesStats({ dependencies, offsetInMs }); + + expect(result).toEqual({ + 'service-B': { + latency: [{ x: 1700000001000, y: 400 }], + errorRate: [{ x: 1700000001000, y: 0.2 }], + throughput: [{ x: 1700000001000, y: undefined }], + }, + }); + }); + + test('should handle missing failures field', () => { + const dependencies = [ + { + key: 'service-C', + timeseries: { + buckets: [ + { + key: 1700000000000, + doc_count: 8, + failures: { doc_count: 0 }, + total_count: { value: 8 }, + latency_sum: { value: 4000 }, + latency_count: { value: 8 }, + throughput: { value: 30 }, + }, + ], + }, + }, + ] as DependenciesTimeseriesBuckes; + + const result = parseDependenciesStats({ dependencies, offsetInMs }); + + expect(result).toEqual({ + 'service-C': { + latency: [{ x: 1700000001000, y: 500 }], + errorRate: [{ x: 1700000001000, y: 0 }], + throughput: [{ x: 1700000001000, y: 30 }], + }, + }); + }); + + test('should return an empty object when dependencies are empty', () => { + const result = parseDependenciesStats({ dependencies: [], offsetInMs }); + expect(result).toEqual({}); + }); +}); diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/dependencies/get_dependencies_timeseries_statistics.ts b/x-pack/solutions/observability/plugins/apm/server/routes/dependencies/get_dependencies_timeseries_statistics.ts new file mode 100644 index 0000000000000..af9a33935128b --- /dev/null +++ b/x-pack/solutions/observability/plugins/apm/server/routes/dependencies/get_dependencies_timeseries_statistics.ts @@ -0,0 +1,256 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server'; +import { EVENT_OUTCOME, SPAN_DESTINATION_SERVICE_RESOURCE } from '../../../common/es_fields/apm'; +import { EventOutcome } from '../../../common/event_outcome'; +import { getBucketSize } from '../../../common/utils/get_bucket_size'; +import { environmentQuery } from '../../../common/utils/environment_query'; +import { getOffsetInMs } from '../../../common/utils/get_offset_in_ms'; +import type { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; +import { + getDocCountFieldForServiceDestinationStatistics, + getDocumentTypeFilterForServiceDestinationStatistics, + getLatencyFieldForServiceDestinationStatistics, + getProcessorEventForServiceDestinationStatistics, +} from '../../lib/helpers/spans/get_is_using_service_destination_metrics'; + +interface Options { + dependencyNames: string[]; + searchServiceDestinationMetrics: boolean; + apmEventClient: APMEventClient; + start: number; + end: number; + environment: string; + kuery: string; + offset?: string; + numBuckets: number; +} + +interface Statistics { + latency: Array<{ x: number; y: number }>; + errorRate: Array<{ x: number; y: number }>; + throughput: Array<{ x: number; y: number | null }>; +} + +async function fetchDependenciesTimeseriesStatistics({ + dependencyNames, + searchServiceDestinationMetrics, + apmEventClient, + start, + end, + environment, + kuery, + numBuckets, +}: Options) { + const response = await apmEventClient.search('get_latency_for_dependency', { + apm: { + events: [getProcessorEventForServiceDestinationStatistics(searchServiceDestinationMetrics)], + }, + track_total_hits: false, + size: 0, + query: { + bool: { + filter: [ + ...environmentQuery(environment), + ...kqlQuery(kuery), + ...rangeQuery(start, end), + ...getDocumentTypeFilterForServiceDestinationStatistics(searchServiceDestinationMetrics), + { terms: { [SPAN_DESTINATION_SERVICE_RESOURCE]: dependencyNames } }, + ], + }, + }, + aggs: { + dependencies: { + terms: { + field: SPAN_DESTINATION_SERVICE_RESOURCE, + }, + aggs: { + timeseries: { + date_histogram: { + field: '@timestamp', + fixed_interval: getBucketSize({ + start, + end, + numBuckets, + minBucketSize: 60, + }).intervalString, + extended_bounds: { + min: start, + max: end, + }, + }, + aggs: { + // latency + latency_sum: { + sum: { + field: getLatencyFieldForServiceDestinationStatistics( + searchServiceDestinationMetrics + ), + }, + }, + ...(searchServiceDestinationMetrics + ? { + latency_count: { + sum: { + field: getDocCountFieldForServiceDestinationStatistics( + searchServiceDestinationMetrics + ), + }, + }, + } + : {}), + // error + ...(searchServiceDestinationMetrics + ? { + total_count: { + sum: { + field: getDocCountFieldForServiceDestinationStatistics( + searchServiceDestinationMetrics + ), + }, + }, + } + : {}), + failures: { + filter: { + term: { + [EVENT_OUTCOME]: EventOutcome.failure, + }, + }, + aggs: { + ...(searchServiceDestinationMetrics + ? { + total_count: { + sum: { + field: getDocCountFieldForServiceDestinationStatistics( + searchServiceDestinationMetrics + ), + }, + }, + } + : {}), + }, + }, + // throughput + throughput: { + rate: { + ...(searchServiceDestinationMetrics + ? { + field: getDocCountFieldForServiceDestinationStatistics( + searchServiceDestinationMetrics + ), + } + : {}), + unit: 'minute', + }, + }, + }, + }, + }, + }, + }, + }); + + return response.aggregations?.dependencies.buckets || []; +} + +export type DependenciesTimeseriesBuckes = Awaited< + ReturnType +>; + +export function parseDependenciesStats({ + dependencies, + offsetInMs, +}: { + dependencies: DependenciesTimeseriesBuckes; + offsetInMs: number; +}) { + return ( + dependencies.reduce>((acc, bucket) => { + const stats: Statistics = { + latency: [], + errorRate: [], + throughput: [], + }; + + for (const statsBucket of bucket.timeseries.buckets) { + const totalCount = statsBucket.total_count?.value ?? statsBucket.doc_count; + const failureCount = + statsBucket.failures.total_count?.value ?? statsBucket.failures.doc_count; + const x = statsBucket.key + offsetInMs; + + stats.latency.push({ + x, + y: + (statsBucket.latency_sum.value ?? 0) / + (statsBucket.latency_count?.value ?? statsBucket.doc_count), + }); + stats.errorRate.push({ x, y: failureCount / totalCount }); + stats.throughput.push({ x, y: statsBucket.throughput.value }); + } + + acc[bucket.key] = stats; + return acc; + }, {}) ?? {} + ); +} + +export interface DependenciesTimeseriesStatisticsResponse { + currentTimeseries: Record; + comparisonTimeseries: Record | null; +} + +export async function getDependenciesTimeseriesStatistics({ + apmEventClient, + dependencyNames, + start, + end, + environment, + kuery, + searchServiceDestinationMetrics, + offset, + numBuckets, +}: Options): Promise { + const { offsetInMs, startWithOffset, endWithOffset } = getOffsetInMs({ + start, + end, + offset, + }); + + const [currentTimeseries, comparisonTimeseries] = await Promise.all([ + fetchDependenciesTimeseriesStatistics({ + dependencyNames, + searchServiceDestinationMetrics, + apmEventClient, + start, + end, + kuery, + environment, + numBuckets, + }), + offset + ? fetchDependenciesTimeseriesStatistics({ + dependencyNames, + searchServiceDestinationMetrics, + apmEventClient, + start: startWithOffset, + end: endWithOffset, + kuery, + environment, + numBuckets, + }) + : null, + ]); + + return { + currentTimeseries: parseDependenciesStats({ dependencies: currentTimeseries, offsetInMs: 0 }), + comparisonTimeseries: comparisonTimeseries?.length + ? parseDependenciesStats({ dependencies: comparisonTimeseries, offsetInMs }) + : null, + }; +} diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/dependencies/get_top_dependencies.ts b/x-pack/solutions/observability/plugins/apm/server/routes/dependencies/get_top_dependencies.ts index 6d23b9c871755..9168ef6729118 100644 --- a/x-pack/solutions/observability/plugins/apm/server/routes/dependencies/get_top_dependencies.ts +++ b/x-pack/solutions/observability/plugins/apm/server/routes/dependencies/get_top_dependencies.ts @@ -27,6 +27,7 @@ interface Options { offset?: string; kuery: string; randomSampler: RandomSampler; + withTimeseries: boolean; } interface TopDependenciesForTimeRange { @@ -44,6 +45,7 @@ async function getTopDependenciesForTimeRange({ offset, kuery, randomSampler, + withTimeseries, }: Options): Promise { const { statsItems, sampled } = await getConnectionStats({ apmEventClient, @@ -54,6 +56,7 @@ async function getTopDependenciesForTimeRange({ offset, collapseBy: 'downstream', randomSampler, + withTimeseries, }); return { diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/dependencies/route.ts b/x-pack/solutions/observability/plugins/apm/server/routes/dependencies/route.ts index 36780980cc0bd..225091b5da0ad 100644 --- a/x-pack/solutions/observability/plugins/apm/server/routes/dependencies/route.ts +++ b/x-pack/solutions/observability/plugins/apm/server/routes/dependencies/route.ts @@ -5,13 +5,15 @@ * 2.0. */ -import { toBooleanRt, toNumberRt } from '@kbn/io-ts-utils'; +import { jsonRt, toBooleanRt, toNumberRt } from '@kbn/io-ts-utils'; import * as t from 'io-ts'; import { offsetRt } from '../../../common/comparison_rt'; import { getApmEventClient } from '../../lib/helpers/get_apm_event_client'; import { getRandomSampler } from '../../lib/helpers/get_random_sampler'; import { createApmServerRoute } from '../apm_routes/create_apm_server_route'; import { environmentRt, kueryRt, rangeRt } from '../default_api_types'; +import type { DependenciesTimeseriesStatisticsResponse } from './get_dependencies_timeseries_statistics'; +import { getDependenciesTimeseriesStatistics } from './get_dependencies_timeseries_statistics'; import type { DependencyLatencyDistributionResponse } from './get_dependency_latency_distribution'; import { getDependencyLatencyDistribution } from './get_dependency_latency_distribution'; import { getErrorRateChartsForDependency } from './get_error_rate_charts_for_dependency'; @@ -32,14 +34,9 @@ import { getUpstreamServicesForDependency } from './get_upstream_services_for_de const topDependenciesRoute = createApmServerRoute({ endpoint: 'GET /internal/apm/dependencies/top_dependencies', - params: t.intersection([ - t.type({ - query: t.intersection([rangeRt, environmentRt, kueryRt, t.type({ numBuckets: toNumberRt })]), - }), - t.partial({ - query: offsetRt, - }), - ]), + params: t.type({ + query: t.intersection([rangeRt, environmentRt, kueryRt, t.type({ numBuckets: toNumberRt })]), + }), security: { authz: { requiredPrivileges: ['apm'] } }, handler: async (resources): Promise => { const { request, core } = resources; @@ -49,7 +46,7 @@ const topDependenciesRoute = createApmServerRoute({ getApmEventClient(resources), getRandomSampler({ coreStart, request, probability: 1 }), ]); - const { environment, offset, numBuckets, kuery, start, end } = resources.params.query; + const { environment, numBuckets, kuery, start, end } = resources.params.query; return getTopDependencies({ apmEventClient, @@ -58,8 +55,37 @@ const topDependenciesRoute = createApmServerRoute({ numBuckets, environment, kuery, - offset, randomSampler, + withTimeseries: false, + }); + }, +}); + +const topDependenciesStatisticsRoute = createApmServerRoute({ + endpoint: 'POST /internal/apm/dependencies/top_dependencies/statistics', + params: t.type({ + query: t.intersection([ + t.intersection([environmentRt, kueryRt, rangeRt, offsetRt]), + t.type({ numBuckets: toNumberRt }), + ]), + body: t.type({ dependencyNames: jsonRt.pipe(t.array(t.string)) }), + }), + security: { authz: { requiredPrivileges: ['apm'] } }, + handler: async (resources): Promise => { + const apmEventClient = await getApmEventClient(resources); + const { environment, offset, numBuckets, kuery, start, end } = resources.params.query; + const { dependencyNames } = resources.params.body; + + return getDependenciesTimeseriesStatistics({ + apmEventClient, + start, + end, + environment, + kuery, + offset, + dependencyNames, + searchServiceDestinationMetrics: true, + numBuckets, }); }, }); @@ -403,4 +429,5 @@ export const dependencisRouteRepository = { ...dependencyOperationsRoute, ...dependencyLatencyDistributionChartsRoute, ...topDependencySpansRoute, + ...topDependenciesStatisticsRoute, }; diff --git a/x-pack/solutions/observability/plugins/apm/server/routes/services/get_service_dependencies_breakdown.ts b/x-pack/solutions/observability/plugins/apm/server/routes/services/get_service_dependencies_breakdown.ts index d69b55a46a1ca..d629852fd9788 100644 --- a/x-pack/solutions/observability/plugins/apm/server/routes/services/get_service_dependencies_breakdown.ts +++ b/x-pack/solutions/observability/plugins/apm/server/routes/services/get_service_dependencies_breakdown.ts @@ -54,7 +54,7 @@ export async function getServiceDependenciesBreakdown({ return { title: getNodeName(location), - data: stats.totalTime.timeseries, + data: stats.totalTime.timeseries || [], }; }); } diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/dependencies/generate_data.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/dependencies/generate_data.ts index 58b708f0ab253..e306f52653c51 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/dependencies/generate_data.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/dependencies/generate_data.ts @@ -9,6 +9,7 @@ import type { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace'; export const dataConfig = { rate: 20, + errorRate: 5, transaction: { name: 'GET /api/product/list', duration: 1000, @@ -33,26 +34,45 @@ export async function generateData({ const instance = apm .service({ name: 'synth-go', environment: 'production', agentName: 'go' }) .instance('instance-a'); - const { rate, transaction, span } = dataConfig; + const { rate, transaction, span, errorRate } = dataConfig; - await apmSynthtraceEsClient.index( - timerange(start, end) - .interval('1m') - .rate(rate) - .generator((timestamp) => - instance - .transaction({ transactionName: transaction.name }) - .timestamp(timestamp) - .duration(transaction.duration) - .success() - .children( - instance - .span({ spanName: span.name, spanType: span.type, spanSubtype: span.subType }) - .duration(transaction.duration) - .success() - .destination(span.destination) - .timestamp(timestamp) - ) - ) - ); + const successfulEvents = timerange(start, end) + .interval('1m') + .rate(rate) + .generator((timestamp) => + instance + .transaction({ transactionName: transaction.name }) + .timestamp(timestamp) + .duration(transaction.duration) + .success() + .children( + instance + .span({ spanName: span.name, spanType: span.type, spanSubtype: span.subType }) + .duration(transaction.duration) + .success() + .destination(span.destination) + .timestamp(timestamp) + ) + ); + + const failureEvents = timerange(start, end) + .interval('1m') + .rate(errorRate) + .generator((timestamp) => + instance + .transaction({ transactionName: transaction.name }) + .timestamp(timestamp) + .duration(transaction.duration) + .failure() + .children( + instance + .span({ spanName: span.name, spanType: span.type, spanSubtype: span.subType }) + .duration(transaction.duration) + .failure() + .destination(span.destination) + .timestamp(timestamp) + ) + ); + + await apmSynthtraceEsClient.index([successfulEvents, failureEvents]); } diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/dependencies/top_dependencies.spec.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/dependencies/top_dependencies.spec.ts index 21e990a1dbb52..76a07c0755de4 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/dependencies/top_dependencies.spec.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/dependencies/top_dependencies.spec.ts @@ -13,6 +13,8 @@ import { dataConfig, generateData } from './generate_data'; import { roundNumber } from '../utils/common'; type TopDependencies = APIReturnType<'GET /internal/apm/dependencies/top_dependencies'>; +type TopDependenciesStatistics = + APIReturnType<'POST /internal/apm/dependencies/top_dependencies/statistics'>; export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderContext) { const apmApiClient = getService('apmApi'); @@ -20,6 +22,7 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon const start = new Date('2021-01-01T00:00:00.000Z').getTime(); const end = new Date('2021-01-01T00:15:00.000Z').getTime() - 1; + const bucketSize = Math.round((end - start) / (60 * 1000)); async function callApi() { return await apmApiClient.readUser({ @@ -31,7 +34,24 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon environment: 'ENVIRONMENT_ALL', kuery: '', numBuckets: 20, - offset: '', + }, + }, + }); + } + + async function callStatisticsApi() { + return await apmApiClient.readUser({ + endpoint: 'POST /internal/apm/dependencies/top_dependencies/statistics', + params: { + query: { + start: new Date(start).toISOString(), + end: new Date(end).toISOString(), + environment: 'ENVIRONMENT_ALL', + kuery: '', + numBuckets: 20, + }, + body: { + dependencyNames: JSON.stringify([dataConfig.span.destination]), }, }, }); @@ -48,13 +68,15 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon describe('when data is generated', () => { let topDependencies: TopDependencies; + let topDependenciesStats: TopDependenciesStatistics; let apmSynthtraceEsClient: ApmSynthtraceEsClient; before(async () => { apmSynthtraceEsClient = await synthtrace.createApmSynthtraceEsClient(); await generateData({ apmSynthtraceEsClient, start, end }); - const response = await callApi(); + const [response, statisticsResponse] = await Promise.all([callApi(), callStatisticsApi()]); topDependencies = response.body; + topDependenciesStats = statisticsResponse.body; }); after(() => apmSynthtraceEsClient.clean()); @@ -90,6 +112,13 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon expect(dependencies.currentStats).to.have.property('impact'); }); + it("doesn't have timeseries stats", () => { + expect(dependencies.currentStats.latency).to.not.have.property('timeseries'); + expect(dependencies.currentStats.totalTime).to.not.have.property('timeseries'); + expect(dependencies.currentStats.throughput).to.not.have.property('timeseries'); + expect(dependencies.currentStats.errorRate).to.not.have.property('timeseries'); + }); + it('returns the correct latency', () => { const { currentStats: { latency }, @@ -97,38 +126,52 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon const { transaction } = dataConfig; - expect(latency.value).to.be(transaction.duration * 1000); - expect(latency.timeseries.every(({ y }) => y === transaction.duration * 1000)).to.be( - true - ); + const expectedValue = transaction.duration * 1000; + expect(latency.value).to.be(expectedValue); + expect( + topDependenciesStats.currentTimeseries[dataConfig.span.destination].latency.every( + ({ y }) => y === expectedValue + ) + ).to.be(true); }); it('returns the correct throughput', () => { const { currentStats: { throughput }, } = dependencies; - const { rate } = dataConfig; + const { rate, errorRate } = dataConfig; - expect(roundNumber(throughput.value)).to.be(roundNumber(rate)); + const totalRate = rate + errorRate; + expect(roundNumber(throughput.value)).to.be(roundNumber(totalRate)); + expect( + topDependenciesStats.currentTimeseries[dataConfig.span.destination].throughput.every( + ({ y }) => roundNumber(y) === roundNumber(totalRate) + ) + ).to.be(true); }); it('returns the correct total time', () => { const { currentStats: { totalTime }, } = dependencies; - const { rate, transaction } = dataConfig; + const { rate, transaction, errorRate } = dataConfig; - expect( - totalTime.timeseries.every(({ y }) => y === rate * transaction.duration * 1000) - ).to.be(true); + const expectedValuePerBucket = (rate + errorRate) * transaction.duration * 1000; + expect(totalTime.value).to.be(expectedValuePerBucket * bucketSize); }); it('returns the correct error rate', () => { const { currentStats: { errorRate }, } = dependencies; - expect(errorRate.value).to.be(0); - expect(errorRate.timeseries.every(({ y }) => y === 0)).to.be(true); + const { rate, errorRate: dataConfigErroRate } = dataConfig; + const expectedValue = dataConfigErroRate / (rate + dataConfigErroRate); + expect(errorRate.value).to.be(expectedValue); + expect( + topDependenciesStats.currentTimeseries[dataConfig.span.destination].errorRate.every( + ({ y }) => y === expectedValue + ) + ).to.be(true); }); }); }); diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/dependencies/upstream_services.spec.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/dependencies/upstream_services.spec.ts index 42d0a66c31a89..99e6da8f321d7 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/dependencies/upstream_services.spec.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/dependencies/upstream_services.spec.ts @@ -63,7 +63,7 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon ]); const currentStatsLatencyValues = body.services[0].currentStats.latency.timeseries; - expect(currentStatsLatencyValues.every(({ y }) => y === 1000000)).to.be(true); + expect(currentStatsLatencyValues?.every(({ y }) => y === 1000000)).to.be(true); }); }); }); diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/service_overview/dependencies/generate_data.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/service_overview/dependencies/generate_data.ts new file mode 100644 index 0000000000000..e306f52653c51 --- /dev/null +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/service_overview/dependencies/generate_data.ts @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { apm, timerange } from '@kbn/apm-synthtrace-client'; +import type { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace'; + +export const dataConfig = { + rate: 20, + errorRate: 5, + transaction: { + name: 'GET /api/product/list', + duration: 1000, + }, + span: { + name: 'GET apm-*/_search', + type: 'db', + subType: 'elasticsearch', + destination: 'elasticsearch', + }, +}; + +export async function generateData({ + apmSynthtraceEsClient, + start, + end, +}: { + apmSynthtraceEsClient: ApmSynthtraceEsClient; + start: number; + end: number; +}) { + const instance = apm + .service({ name: 'synth-go', environment: 'production', agentName: 'go' }) + .instance('instance-a'); + const { rate, transaction, span, errorRate } = dataConfig; + + const successfulEvents = timerange(start, end) + .interval('1m') + .rate(rate) + .generator((timestamp) => + instance + .transaction({ transactionName: transaction.name }) + .timestamp(timestamp) + .duration(transaction.duration) + .success() + .children( + instance + .span({ spanName: span.name, spanType: span.type, spanSubtype: span.subType }) + .duration(transaction.duration) + .success() + .destination(span.destination) + .timestamp(timestamp) + ) + ); + + const failureEvents = timerange(start, end) + .interval('1m') + .rate(errorRate) + .generator((timestamp) => + instance + .transaction({ transactionName: transaction.name }) + .timestamp(timestamp) + .duration(transaction.duration) + .failure() + .children( + instance + .span({ spanName: span.name, spanType: span.type, spanSubtype: span.subType }) + .duration(transaction.duration) + .failure() + .destination(span.destination) + .timestamp(timestamp) + ) + ); + + await apmSynthtraceEsClient.index([successfulEvents, failureEvents]); +} diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/service_overview/dependencies/index.spec.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/service_overview/dependencies/index.spec.ts index 29f7aaa7e0d08..d5164a0ea90c8 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/service_overview/dependencies/index.spec.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/service_overview/dependencies/index.spec.ts @@ -5,33 +5,22 @@ * 2.0. */ import expect from '@kbn/expect'; -import { last, pick } from 'lodash'; import { DependencyNode } from '@kbn/apm-plugin/common/connections'; -import type { ValuesType } from 'utility-types'; import type { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace'; import type { APIReturnType } from '@kbn/apm-plugin/public/services/rest/create_call_apm_api'; -import { type Node, NodeType } from '@kbn/apm-plugin/common/connections'; -import { - ENVIRONMENT_ALL, - ENVIRONMENT_NOT_DEFINED, -} from '@kbn/apm-plugin/common/environment_filter_values'; +import { NodeType } from '@kbn/apm-plugin/common/connections'; import type { DeploymentAgnosticFtrProviderContext } from '../../../../../ftr_provider_context'; import { roundNumber } from '../../utils/common'; -import { generateDependencyData } from '../generate_data'; -import { apmDependenciesMapping, createServiceDependencyDocs } from './es_utils'; +import { generateData, dataConfig } from './generate_data'; export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderContext) { const apmApiClient = getService('apmApi'); const synthtrace = getService('synthtrace'); - const es = getService('es'); const start = new Date('2021-01-01T00:00:00.000Z').getTime(); const end = new Date('2021-01-01T00:15:00.000Z').getTime() - 1; - const dependencyName = 'elasticsearch'; - const serviceName = 'synth-go'; + const bucketSize = Math.round((end - start) / (60 * 1000)); - function getName(node: Node) { - return node.type === NodeType.service ? node.serviceName : node.dependencyName; - } + const serviceName = 'synth-go'; async function callApi() { return await apmApiClient.readUser({ @@ -60,265 +49,91 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon }); describe('when specific data is loaded', () => { - let response: { - status: number; - body: APIReturnType<'GET /internal/apm/services/{serviceName}/dependencies'>; - }; - - const indices = { - metric: 'apm-dependencies-metric', - transaction: 'apm-dependencies-transaction', - span: 'apm-dependencies-span', - }; - - const startTime = new Date(start).getTime(); - const endTime = new Date(end).getTime(); - - after(async () => { - const allIndices = Object.values(indices).join(','); - const indexExists = await es.indices.exists({ index: allIndices }); - if (indexExists) { - await es.indices.delete({ - index: allIndices, - }); - } - }); + let dependencies: APIReturnType<'GET /internal/apm/services/{serviceName}/dependencies'>; + let apmSynthtraceEsClient: ApmSynthtraceEsClient; before(async () => { - await es.indices.create({ - index: indices.metric, - mappings: apmDependenciesMapping, - }); - - await es.indices.create({ - index: indices.transaction, - mappings: apmDependenciesMapping, - }); - - await es.indices.create({ - index: indices.span, - mappings: apmDependenciesMapping, - }); - - const docs = [ - ...createServiceDependencyDocs({ - service: { - name: 'opbeans-java', - environment: 'production', - }, - agentName: 'java', - span: { - type: 'external', - subtype: 'http', - }, - resource: 'opbeans-node:3000', - outcome: 'success', - responseTime: { - count: 2, - sum: 10, - }, - time: startTime, - to: { - service: { - name: 'opbeans-node', - }, - agentName: 'nodejs', - }, - }), - ...createServiceDependencyDocs({ - service: { - name: 'opbeans-java', - environment: 'production', - }, - agentName: 'java', - span: { - type: 'external', - subtype: 'http', - }, - resource: 'opbeans-node:3000', - outcome: 'failure', - responseTime: { - count: 1, - sum: 10, - }, - time: startTime, - }), - ...createServiceDependencyDocs({ - service: { - name: 'opbeans-java', - environment: 'production', - }, - agentName: 'java', - span: { - type: 'external', - subtype: 'http', - }, - resource: 'postgres', - outcome: 'success', - responseTime: { - count: 1, - sum: 3, - }, - time: startTime, - }), - ...createServiceDependencyDocs({ - service: { - name: 'opbeans-java', - environment: 'production', - }, - agentName: 'java', - span: { - type: 'external', - subtype: 'http', - }, - resource: 'opbeans-node-via-proxy', - outcome: 'success', - responseTime: { - count: 1, - sum: 1, - }, - time: endTime - 1, - to: { - service: { - name: 'opbeans-node', - }, - agentName: 'nodejs', - }, - }), - ]; - - const operations = docs.reduce( - (prev, doc) => { - return [...prev, { index: { _index: indices[doc.processor.event] } }, doc]; - }, - [] as Array< - | { - index: { - _index: string; - }; - } - | ValuesType - > - ); - - await es.bulk({ - operations, - refresh: 'wait_for', - }); - - response = await apmApiClient.readUser({ - endpoint: `GET /internal/apm/services/{serviceName}/dependencies`, - params: { - path: { serviceName: 'opbeans-java' }, - query: { - start: new Date(start).toISOString(), - end: new Date(end).toISOString(), - numBuckets: 20, - environment: ENVIRONMENT_ALL.value, - }, - }, - }); + apmSynthtraceEsClient = await synthtrace.createApmSynthtraceEsClient(); + await generateData({ apmSynthtraceEsClient, start, end }); + const response = await callApi(); + dependencies = response.body; }); - it('returns a 200', () => { - expect(response.status).to.be(200); - }); + after(() => apmSynthtraceEsClient.clean()); - it('returns two dependencies', () => { - expect(response.body.serviceDependencies.length).to.be(2); + it('returns one dependency', () => { + expect(dependencies.serviceDependencies.length).to.be(1); }); - it('returns opbeans-node as a dependency', () => { - const opbeansNode = response.body.serviceDependencies.find( - (item) => getName(item.location) === 'opbeans-node' - ); + it('returns correct dependency information', () => { + const location = dependencies.serviceDependencies[0].location as DependencyNode; + const { span } = dataConfig; - expect(opbeansNode !== undefined).to.be(true); - - const values = { - latency: roundNumber(opbeansNode?.currentStats.latency.value), - throughput: roundNumber(opbeansNode?.currentStats.throughput.value), - errorRate: roundNumber(opbeansNode?.currentStats.errorRate.value), - impact: opbeansNode?.currentStats.impact, - ...pick(opbeansNode?.location, 'serviceName', 'type', 'agentName', 'environment'), - }; - - const count = 4; - const sum = 21; - const errors = 1; - - expect(values).to.eql({ - agentName: 'nodejs', - environment: ENVIRONMENT_NOT_DEFINED.value, - serviceName: 'opbeans-node', - type: 'service', - errorRate: roundNumber(errors / count), - latency: roundNumber(sum / count), - throughput: roundNumber(count / ((endTime - startTime) / 1000 / 60)), - impact: 100, - }); - - const firstValue = roundNumber(opbeansNode?.currentStats.latency.timeseries[0].y); - const lastValue = roundNumber(last(opbeansNode?.currentStats.latency.timeseries)?.y); - - expect(firstValue).to.be(roundNumber(20 / 3)); - expect(lastValue).to.be(1); + expect(location.type).to.be(NodeType.dependency); + expect(location.dependencyName).to.be(span.destination); + expect(location.spanType).to.be(span.type); + expect(location.spanSubtype).to.be(span.subType); + expect(location).to.have.property('id'); }); - it('returns postgres as an external dependency', () => { - const postgres = response.body.serviceDependencies.find( - (item) => getName(item.location) === 'postgres' - ); + it("doesn't have previous stats", () => { + expect(dependencies.serviceDependencies[0].previousStats).to.be(null); + }); - expect(postgres !== undefined).to.be(true); + it('has an "impact" property', () => { + expect(dependencies.serviceDependencies[0].currentStats).to.have.property('impact'); + }); - const values = { - latency: roundNumber(postgres?.currentStats.latency.value), - throughput: roundNumber(postgres?.currentStats.throughput.value), - errorRate: roundNumber(postgres?.currentStats.errorRate.value), - impact: postgres?.currentStats.impact, - ...pick(postgres?.location, 'spanType', 'spanSubtype', 'dependencyName', 'type'), - }; + it('returns the correct latency', () => { + const { + currentStats: { latency }, + } = dependencies.serviceDependencies[0]; - const count = 1; - const sum = 3; - const errors = 0; + const { transaction } = dataConfig; - expect(values).to.eql({ - spanType: 'external', - spanSubtype: 'http', - dependencyName: 'postgres', - type: 'dependency', - errorRate: roundNumber(errors / count), - latency: roundNumber(sum / count), - throughput: roundNumber(count / ((endTime - startTime) / 1000 / 60)), - impact: 0, - }); + const expectedValue = transaction.duration * 1000; + expect(latency.value).to.be(expectedValue); + expect(latency.timeseries?.every(({ y }) => y === expectedValue)).to.be(true); }); - }); - describe('when data is loaded', () => { - let apmSynthtraceEsClient: ApmSynthtraceEsClient; + it('returns the correct throughput', () => { + const { + currentStats: { throughput }, + } = dependencies.serviceDependencies[0]; + const { rate, errorRate } = dataConfig; - before(async () => { - apmSynthtraceEsClient = await synthtrace.createApmSynthtraceEsClient(); - await generateDependencyData({ apmSynthtraceEsClient, start, end }); + const expectedThroughput = rate + errorRate; + expect(roundNumber(throughput.value)).to.be(roundNumber(expectedThroughput)); + expect( + throughput.timeseries?.every( + ({ y }) => roundNumber(y) === roundNumber(expectedThroughput / bucketSize) + ) + ).to.be(true); }); - after(() => apmSynthtraceEsClient.clean()); - it('returns a list of dependencies for a service', async () => { - const { status, body } = await callApi(); + it('returns the correct total time', () => { + const { + currentStats: { totalTime }, + } = dependencies.serviceDependencies[0]; + const { rate, transaction, errorRate } = dataConfig; - expect(status).to.be(200); + const expectedValuePerBucket = (rate + errorRate) * transaction.duration * 1000; + expect(totalTime.value).to.be(expectedValuePerBucket * bucketSize); expect( - body.serviceDependencies.map( - ({ location }) => (location as DependencyNode).dependencyName + totalTime.timeseries?.every( + ({ y }) => roundNumber(y) === roundNumber(expectedValuePerBucket) ) - ).to.eql([dependencyName]); + ).to.be(true); + }); - const currentStatsLatencyValues = - body.serviceDependencies[0].currentStats.latency.timeseries; - expect(currentStatsLatencyValues.every(({ y }) => y === 1000000)).to.be(true); + it('returns the correct error rate', () => { + const { + currentStats: { errorRate }, + } = dependencies.serviceDependencies[0]; + const { rate, errorRate: dataConfigErroRate } = dataConfig; + const expectedValue = dataConfigErroRate / (rate + dataConfigErroRate); + expect(errorRate.value).to.be(expectedValue); + expect(errorRate.timeseries?.every(({ y }) => y === expectedValue)).to.be(true); }); }); });