diff --git a/packages/kbn-apm-synthtrace/src/lib/apm/apm_fields.ts b/packages/kbn-apm-synthtrace/src/lib/apm/apm_fields.ts index 690e6c3563f2..7736aa8e8879 100644 --- a/packages/kbn-apm-synthtrace/src/lib/apm/apm_fields.ts +++ b/packages/kbn-apm-synthtrace/src/lib/apm/apm_fields.ts @@ -18,6 +18,10 @@ export type ApmApplicationMetricFields = Partial<{ 'jvm.memory.heap.used': number; 'jvm.memory.non_heap.used': number; 'jvm.thread.count': number; + 'faas.billed_duration': number; + 'faas.timeout': number; + 'faas.coldstart_duration': number; + 'faas.duration': number; }>; export type ApmUserAgentFields = Partial<{ @@ -104,6 +108,7 @@ export type ApmFields = Fields & 'cloud.region': string; 'host.os.platform': string; 'faas.id': string; + 'faas.name': string; 'faas.coldstart': boolean; 'faas.execution': string; 'faas.trigger.type': string; diff --git a/packages/kbn-apm-synthtrace/src/lib/apm/index.ts b/packages/kbn-apm-synthtrace/src/lib/apm/index.ts index a136daabee8f..84e6bfc9e812 100644 --- a/packages/kbn-apm-synthtrace/src/lib/apm/index.ts +++ b/packages/kbn-apm-synthtrace/src/lib/apm/index.ts @@ -7,6 +7,7 @@ */ import { service } from './service'; import { browser } from './browser'; +import { serverlessFunction } from './serverless_function'; import { getTransactionMetrics } from './processors/get_transaction_metrics'; import { getSpanDestinationMetrics } from './processors/get_span_destination_metrics'; import { getChromeUserAgentDefaults } from './defaults/get_chrome_user_agent_defaults'; @@ -27,6 +28,7 @@ export const apm = { getApmWriteTargets, ApmSynthtraceEsClient, ApmSynthtraceKibanaClient, + serverlessFunction, }; export type { ApmSynthtraceEsClient, ApmException }; diff --git a/packages/kbn-apm-synthtrace/src/lib/apm/serverless.ts b/packages/kbn-apm-synthtrace/src/lib/apm/serverless.ts new file mode 100644 index 000000000000..b67586c18a07 --- /dev/null +++ b/packages/kbn-apm-synthtrace/src/lib/apm/serverless.ts @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { generateLongId, generateShortId } from '../utils/generate_id'; +import { ApmFields } from './apm_fields'; +import { BaseSpan } from './base_span'; +import { Metricset } from './metricset'; + +export type FaasTriggerType = 'http' | 'pubsub' | 'datasource' | 'timer' | 'other'; + +export class Serverless extends BaseSpan { + private readonly metric: Metricset; + + constructor(fields: ApmFields) { + const faasExection = generateLongId(); + const triggerType = 'other'; + super({ + ...fields, + 'processor.event': 'transaction', + 'transaction.id': generateShortId(), + 'transaction.sampled': true, + 'faas.execution': faasExection, + 'faas.trigger.type': triggerType, + 'transaction.name': fields['transaction.name'] || fields['faas.name'], + 'transaction.type': 'request', + }); + this.metric = new Metricset({ + ...fields, + 'metricset.name': 'app', + 'faas.execution': faasExection, + 'faas.id': fields['service.name'], + }); + } + + duration(duration: number) { + this.fields['transaction.duration.us'] = duration * 1000; + return this; + } + + coldStart(coldstart: boolean) { + this.fields['faas.coldstart'] = coldstart; + this.metric.fields['faas.coldstart'] = coldstart; + return this; + } + + billedDuration(billedDuration: number) { + this.metric.fields['faas.billed_duration'] = billedDuration; + return this; + } + + faasTimeout(faasTimeout: number) { + this.metric.fields['faas.timeout'] = faasTimeout; + return this; + } + + memory({ total, free }: { total: number; free: number }) { + this.metric.fields['system.memory.total'] = total; + this.metric.fields['system.memory.actual.free'] = free; + return this; + } + + coldStartDuration(coldStartDuration: number) { + this.metric.fields['faas.coldstart_duration'] = coldStartDuration; + return this; + } + + faasDuration(faasDuration: number) { + this.metric.fields['faas.duration'] = faasDuration; + return this; + } + + timestamp(time: number): this { + super.timestamp(time); + this.metric.fields['@timestamp'] = time; + return this; + } + + serialize(): ApmFields[] { + const transaction = super.serialize(); + const metric = this.metric.serialize(); + return [...transaction, ...metric]; + } +} diff --git a/packages/kbn-apm-synthtrace/src/lib/apm/serverless_function.ts b/packages/kbn-apm-synthtrace/src/lib/apm/serverless_function.ts new file mode 100644 index 000000000000..e10bb23b1f93 --- /dev/null +++ b/packages/kbn-apm-synthtrace/src/lib/apm/serverless_function.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { Entity } from '../entity'; +import { generateShortId } from '../utils/generate_id'; +import { ApmFields } from './apm_fields'; +import { ServerlessInstance } from './serverless_instance'; + +export class ServerlessFunction extends Entity { + instance({ instanceName, ...apmFields }: { instanceName: string } & ApmFields) { + return new ServerlessInstance({ + ...this.fields, + ['service.node.name']: instanceName, + 'host.name': instanceName, + ...apmFields, + }); + } +} + +export function serverlessFunction({ + functionName, + serviceName, + environment, + agentName, +}: { + functionName: string; + environment: string; + agentName: string; + serviceName?: string; +}) { + const faasId = `arn:aws:lambda:us-west-2:${generateShortId()}:function:${functionName}`; + return new ServerlessFunction({ + 'service.name': serviceName || faasId, + 'faas.id': faasId, + 'faas.name': functionName, + 'service.environment': environment, + 'agent.name': agentName, + 'service.runtime.name': 'AWS_lambda', + }); +} diff --git a/packages/kbn-apm-synthtrace/src/lib/apm/serverless_instance.ts b/packages/kbn-apm-synthtrace/src/lib/apm/serverless_instance.ts new file mode 100644 index 000000000000..2d1626add771 --- /dev/null +++ b/packages/kbn-apm-synthtrace/src/lib/apm/serverless_instance.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { Entity } from '../entity'; +import { ApmFields } from './apm_fields'; +import { FaasTriggerType, Serverless } from './serverless'; + +export class ServerlessInstance extends Entity { + invocation(params: { transactionName?: string; faasTriggerType?: FaasTriggerType } = {}) { + const { transactionName, faasTriggerType = 'other' } = params; + return new Serverless({ + ...this.fields, + 'transaction.name': transactionName, + 'faas.trigger.type': faasTriggerType, + }); + } +} diff --git a/packages/kbn-apm-synthtrace/src/scenarios/aws_lambda.ts b/packages/kbn-apm-synthtrace/src/scenarios/aws_lambda.ts index 2dcce23ab2a2..b89b0b5ea5f1 100644 --- a/packages/kbn-apm-synthtrace/src/scenarios/aws_lambda.ts +++ b/packages/kbn-apm-synthtrace/src/scenarios/aws_lambda.ts @@ -7,55 +7,90 @@ */ import { apm, timerange } from '../..'; -import { ApmFields } from '../lib/apm/apm_fields'; import { Scenario } from '../cli/scenario'; -import { getLogger } from '../cli/utils/get_common_services'; import { RunOptions } from '../cli/utils/parse_run_cli_flags'; +import { ApmFields } from '../lib/apm/apm_fields'; import { getSynthtraceEnvironment } from '../lib/utils/get_synthtrace_environment'; const ENVIRONMENT = getSynthtraceEnvironment(__filename); const scenario: Scenario = async (runOptions: RunOptions) => { - const logger = getLogger(runOptions); - return { generate: ({ from, to }) => { const range = timerange(from, to); const timestamps = range.ratePerMinute(180); - const instance = apm - .service({ name: 'lambda-python', environment: ENVIRONMENT, agentName: 'python' }) - .instance('instance'); - - const traceEventsSetups = [ - { functionName: 'lambda-python-1', coldStart: true }, - { functionName: 'lambda-python-2', coldStart: false }, - ]; - - const traceEvents = ({ functionName, coldStart }: typeof traceEventsSetups[0]) => { - return timestamps.generator((timestamp) => - instance - .transaction({ transactionName: 'GET /order/{id}' }) - .defaults({ - 'service.runtime.name': 'AWS_Lambda_python3.8', - 'cloud.provider': 'aws', - 'cloud.service.name': 'lambda', - 'cloud.region': 'us-east-1', - 'faas.id': `arn:aws:lambda:us-west-2:123456789012:function:${functionName}`, - 'faas.coldstart': coldStart, - 'faas.trigger.type': 'other', - }) + const cloudFields: ApmFields = { + 'cloud.provider': 'aws', + 'cloud.service.name': 'lambda', + 'cloud.region': 'us-west-2', + }; + + const instanceALambdaPython = apm + .serverlessFunction({ + serviceName: 'aws-lambdas', + environment: ENVIRONMENT, + agentName: 'python', + functionName: 'fn-python-1', + }) + .instance({ instanceName: 'instance_A', ...cloudFields }); + + const instanceALambdaNode = apm + .serverlessFunction({ + serviceName: 'aws-lambdas', + environment: ENVIRONMENT, + agentName: 'nodejs', + functionName: 'fn-node-1', + }) + .instance({ instanceName: 'instance_A', ...cloudFields }); + + const instanceALambdaNode2 = apm + .serverlessFunction({ + environment: ENVIRONMENT, + agentName: 'nodejs', + functionName: 'fn-node-2', + }) + .instance({ instanceName: 'instance_A', ...cloudFields }); + + const memory = { + total: 536870912, // 0.5gb + free: 94371840, // ~0.08 gb + }; + + const awsLambdaEvents = timestamps.generator((timestamp) => { + return [ + instanceALambdaPython + .invocation() + .duration(1000) .timestamp(timestamp) + .coldStart(true) + .billedDuration(4000) + .faasTimeout(10000) + .memory(memory) + .coldStartDuration(4000) + .faasDuration(4000), + instanceALambdaNode + .invocation() .duration(1000) - .success() - ); - }; + .timestamp(timestamp) + .coldStart(false) + .billedDuration(4000) + .faasTimeout(10000) + .memory(memory) + .faasDuration(4000), + instanceALambdaNode2 + .invocation() + .duration(1000) + .timestamp(timestamp) + .coldStart(false) + .billedDuration(4000) + .faasTimeout(10000) + .memory(memory) + .faasDuration(4000), + ]; + }); - return traceEventsSetups - .map((traceEventsSetup) => - logger.perf('generating_apm_events', () => traceEvents(traceEventsSetup)) - ) - .reduce((p, c) => p.merge(c)); + return awsLambdaEvents; }, }; }; diff --git a/src/plugins/chart_expressions/expression_xy/public/helpers/data_layers.tsx b/src/plugins/chart_expressions/expression_xy/public/helpers/data_layers.tsx index 7476d43f773e..917c9afdb304 100644 --- a/src/plugins/chart_expressions/expression_xy/public/helpers/data_layers.tsx +++ b/src/plugins/chart_expressions/expression_xy/public/helpers/data_layers.tsx @@ -481,7 +481,7 @@ export const getSeriesProps: GetSeriesPropsFn = ({ return { splitSeriesAccessors: splitColumnIds.length ? splitColumnIds : [], - stackAccessors: isStacked && xColumnId ? [xColumnId] : [], + stackAccessors: isStacked ? [xColumnId || 'unifiedX'] : [], id: generateSeriesId( layer, splitColumnIds.length ? splitColumnIds : [EMPTY_ACCESSOR], diff --git a/src/plugins/charts/public/static/components/warnings.tsx b/src/plugins/charts/public/static/components/warnings.tsx index 9681cef1d6ac..f2ca61e3e1ed 100644 --- a/src/plugins/charts/public/static/components/warnings.tsx +++ b/src/plugins/charts/public/static/components/warnings.tsx @@ -21,7 +21,13 @@ export function Warnings({ warnings }: { warnings: React.ReactNode[] }) { panelPaddingSize="none" closePopover={() => setOpen(false)} button={ - setOpen(!open)} size="xs"> + setOpen(!open)} + size="xs" + data-test-subj="chart-inline-warning-button" + > {i18n.translate('charts.warning.warningLabel', { defaultMessage: '{numberWarnings, number} {numberWarnings, plural, one {warning} other {warnings}}', @@ -39,6 +45,7 @@ export function Warnings({ warnings }: { warnings: React.ReactNode[] }) { css={{ padding: euiThemeVars.euiSizeS, }} + data-test-subj="chart-inline-warning" > {w} diff --git a/x-pack/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap b/x-pack/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap index 0065245e507e..fddee59d9c6c 100644 --- a/x-pack/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap +++ b/x-pack/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap @@ -61,8 +61,14 @@ exports[`Error ERROR_PAGE_URL 1`] = `undefined`; exports[`Error EVENT_OUTCOME 1`] = `undefined`; +exports[`Error FAAS_BILLED_DURATION 1`] = `undefined`; + exports[`Error FAAS_COLDSTART 1`] = `undefined`; +exports[`Error FAAS_COLDSTART_DURATION 1`] = `undefined`; + +exports[`Error FAAS_DURATION 1`] = `undefined`; + exports[`Error FAAS_ID 1`] = `undefined`; exports[`Error FAAS_TRIGGER_TYPE 1`] = `undefined`; @@ -284,8 +290,14 @@ exports[`Span ERROR_PAGE_URL 1`] = `undefined`; exports[`Span EVENT_OUTCOME 1`] = `"unknown"`; +exports[`Span FAAS_BILLED_DURATION 1`] = `undefined`; + exports[`Span FAAS_COLDSTART 1`] = `undefined`; +exports[`Span FAAS_COLDSTART_DURATION 1`] = `undefined`; + +exports[`Span FAAS_DURATION 1`] = `undefined`; + exports[`Span FAAS_ID 1`] = `undefined`; exports[`Span FAAS_TRIGGER_TYPE 1`] = `undefined`; @@ -499,8 +511,14 @@ exports[`Transaction ERROR_PAGE_URL 1`] = `undefined`; exports[`Transaction EVENT_OUTCOME 1`] = `"unknown"`; +exports[`Transaction FAAS_BILLED_DURATION 1`] = `undefined`; + exports[`Transaction FAAS_COLDSTART 1`] = `undefined`; +exports[`Transaction FAAS_COLDSTART_DURATION 1`] = `undefined`; + +exports[`Transaction FAAS_DURATION 1`] = `undefined`; + exports[`Transaction FAAS_ID 1`] = `undefined`; exports[`Transaction FAAS_TRIGGER_TYPE 1`] = `undefined`; diff --git a/x-pack/plugins/apm/common/elasticsearch_fieldnames.ts b/x-pack/plugins/apm/common/elasticsearch_fieldnames.ts index d8460bcad9b7..1e227713f0db 100644 --- a/x-pack/plugins/apm/common/elasticsearch_fieldnames.ts +++ b/x-pack/plugins/apm/common/elasticsearch_fieldnames.ts @@ -134,6 +134,9 @@ export const USER_AGENT_OS = 'user_agent.os.name'; export const FAAS_ID = 'faas.id'; export const FAAS_COLDSTART = 'faas.coldstart'; export const FAAS_TRIGGER_TYPE = 'faas.trigger.type'; +export const FAAS_DURATION = 'faas.duration'; +export const FAAS_COLDSTART_DURATION = 'faas.coldstart_duration'; +export const FAAS_BILLED_DURATION = 'faas.billed_duration'; // Metadata export const TIER = '_tier'; diff --git a/x-pack/plugins/apm/public/components/app/dependency_operation_detail_view/dependency_operation_detail_trace_list.tsx b/x-pack/plugins/apm/public/components/app/dependency_operation_detail_view/dependency_operation_detail_trace_list.tsx index cf51fe6cf75e..da4c603ff283 100644 --- a/x-pack/plugins/apm/public/components/app/dependency_operation_detail_view/dependency_operation_detail_trace_list.tsx +++ b/x-pack/plugins/apm/public/components/app/dependency_operation_detail_view/dependency_operation_detail_trace_list.tsx @@ -127,6 +127,7 @@ export function DependencyOperationDetailTraceList() { { defaultMessage: 'Trace' } ), field: 'traceId', + truncateText: true, render: ( _, { @@ -158,6 +159,7 @@ export function DependencyOperationDetailTraceList() { { defaultMessage: 'Originating service' } ), field: 'serviceName', + truncateText: true, render: (_, { serviceName, agentName }) => { const serviceLinkQuery = { comparisonEnabled, @@ -187,6 +189,7 @@ export function DependencyOperationDetailTraceList() { { defaultMessage: 'Transaction name' } ), field: 'transactionName', + truncateText: true, render: ( _, { @@ -227,6 +230,7 @@ export function DependencyOperationDetailTraceList() { { defaultMessage: 'Timestamp' } ), field: '@timestamp', + truncateText: true, render: (_, { '@timestamp': timestamp }) => { return ; }, @@ -286,7 +290,6 @@ export function DependencyOperationDetailTraceList() { status === FETCH_STATUS.LOADING || status === FETCH_STATUS.NOT_INITIATED } - tableLayout="auto" /> diff --git a/x-pack/plugins/apm/server/routes/metrics/by_agent/serverless/active_instances.ts b/x-pack/plugins/apm/server/routes/metrics/by_agent/serverless/active_instances.ts new file mode 100644 index 000000000000..426132b39cc0 --- /dev/null +++ b/x-pack/plugins/apm/server/routes/metrics/by_agent/serverless/active_instances.ts @@ -0,0 +1,114 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; +import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server'; +import { euiLightVars as theme } from '@kbn/ui-theme'; +import { + SERVICE_NAME, + SERVICE_NODE_NAME, +} from '../../../../../common/elasticsearch_fieldnames'; +import { environmentQuery } from '../../../../../common/utils/environment_query'; +import { getVizColorForIndex } from '../../../../../common/viz_colors'; +import { getMetricsDateHistogramParams } from '../../../../lib/helpers/metrics'; +import { Setup } from '../../../../lib/helpers/setup_request'; +import { + getDocumentTypeFilterForTransactions, + getProcessorEventForTransactions, +} from '../../../../lib/helpers/transactions'; +import { GenericMetricsChart } from '../../fetch_and_transform_metrics'; + +export async function getActiveInstances({ + environment, + kuery, + setup, + serviceName, + start, + end, + searchAggregatedTransactions, +}: { + environment: string; + kuery: string; + setup: Setup; + serviceName: string; + start: number; + end: number; + searchAggregatedTransactions: boolean; +}): Promise { + const { apmEventClient, config } = setup; + + const aggs = { + activeInstances: { + cardinality: { + field: SERVICE_NODE_NAME, + }, + }, + }; + + const params = { + apm: { + events: [getProcessorEventForTransactions(searchAggregatedTransactions)], + }, + body: { + size: 0, + query: { + bool: { + filter: [ + { term: { [SERVICE_NAME]: serviceName } }, + ...rangeQuery(start, end), + ...environmentQuery(environment), + ...kqlQuery(kuery), + ...getDocumentTypeFilterForTransactions( + searchAggregatedTransactions + ), + ], + }, + }, + aggs: { + ...aggs, + timeseriesData: { + date_histogram: getMetricsDateHistogramParams({ + start, + end, + metricsInterval: config.metricsInterval, + }), + aggs, + }, + }, + }, + }; + + const { aggregations } = await apmEventClient.search( + 'get_active_instances', + params + ); + + return { + title: i18n.translate('xpack.apm.agentMetrics.serverless.activeInstances', { + defaultMessage: 'Active instances', + }), + key: 'active_instances', + yUnit: 'number', + series: [ + { + title: i18n.translate( + 'xpack.apm.agentMetrics.serverless.series.activeInstances', + { defaultMessage: 'Active instances' } + ), + key: 'active_instances', + type: 'linemark', + color: getVizColorForIndex(0, theme), + overallValue: aggregations?.activeInstances.value ?? 0, + data: + aggregations?.timeseriesData.buckets.map((timeseriesBucket) => ({ + x: timeseriesBucket.key, + y: timeseriesBucket.activeInstances.value, + })) || [], + }, + ], + }; +} diff --git a/x-pack/plugins/apm/server/routes/metrics/by_agent/serverless/cold_start_count.ts b/x-pack/plugins/apm/server/routes/metrics/by_agent/serverless/cold_start_count.ts new file mode 100644 index 000000000000..fc94a59da9a2 --- /dev/null +++ b/x-pack/plugins/apm/server/routes/metrics/by_agent/serverless/cold_start_count.ts @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; +import { termQuery } from '@kbn/observability-plugin/server'; +import { + FAAS_COLDSTART, + METRICSET_NAME, +} from '../../../../../common/elasticsearch_fieldnames'; +import { Setup } from '../../../../lib/helpers/setup_request'; +import { fetchAndTransformMetrics } from '../../fetch_and_transform_metrics'; +import { ChartBase } from '../../types'; + +const chartBase: ChartBase = { + title: i18n.translate('xpack.apm.agentMetrics.serverless.coldStart', { + defaultMessage: 'Cold start', + }), + key: 'cold_start_count', + type: 'linemark', + yUnit: 'number', + series: { + coldStart: { + title: i18n.translate('xpack.apm.agentMetrics.serverless.coldStart', { + defaultMessage: 'Cold start', + }), + }, + }, +}; + +export function getColdStartCount({ + environment, + kuery, + setup, + serviceName, + start, + end, +}: { + environment: string; + kuery: string; + setup: Setup; + serviceName: string; + start: number; + end: number; +}) { + return fetchAndTransformMetrics({ + environment, + kuery, + setup, + serviceName, + start, + end, + chartBase, + aggs: { coldStart: { sum: { field: FAAS_COLDSTART } } }, + additionalFilters: [ + ...termQuery(FAAS_COLDSTART, true), + ...termQuery(METRICSET_NAME, 'transaction'), + ], + operationName: 'get_cold_start_count', + }); +} diff --git a/x-pack/plugins/apm/server/routes/metrics/by_agent/serverless/cold_start_duration.ts b/x-pack/plugins/apm/server/routes/metrics/by_agent/serverless/cold_start_duration.ts new file mode 100644 index 000000000000..84fac7f99303 --- /dev/null +++ b/x-pack/plugins/apm/server/routes/metrics/by_agent/serverless/cold_start_duration.ts @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; +import { FAAS_COLDSTART_DURATION } from '../../../../../common/elasticsearch_fieldnames'; +import { Setup } from '../../../../lib/helpers/setup_request'; +import { fetchAndTransformMetrics } from '../../fetch_and_transform_metrics'; +import { ChartBase } from '../../types'; + +const chartBase: ChartBase = { + title: i18n.translate('xpack.apm.agentMetrics.serverless.coldStartDuration', { + defaultMessage: 'Cold start duration', + }), + key: 'cold_start_duration', + type: 'linemark', + yUnit: 'time', + series: { + coldStart: { + title: i18n.translate( + 'xpack.apm.agentMetrics.serverless.coldStartDuration', + { defaultMessage: 'Cold start duration' } + ), + }, + }, +}; + +export function getColdStartDuration({ + environment, + kuery, + setup, + serviceName, + start, + end, +}: { + environment: string; + kuery: string; + setup: Setup; + serviceName: string; + start: number; + end: number; +}) { + return fetchAndTransformMetrics({ + environment, + kuery, + setup, + serviceName, + start, + end, + chartBase, + aggs: { coldStart: { avg: { field: FAAS_COLDSTART_DURATION } } }, + additionalFilters: [{ exists: { field: FAAS_COLDSTART_DURATION } }], + operationName: 'get_cold_start_duration', + }); +} diff --git a/x-pack/plugins/apm/server/routes/metrics/by_agent/serverless/compute_usage.ts b/x-pack/plugins/apm/server/routes/metrics/by_agent/serverless/compute_usage.ts new file mode 100644 index 000000000000..da57498c8af0 --- /dev/null +++ b/x-pack/plugins/apm/server/routes/metrics/by_agent/serverless/compute_usage.ts @@ -0,0 +1,161 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; +import { ProcessorEvent } from '@kbn/observability-plugin/common'; +import { + kqlQuery, + rangeQuery, + termQuery, +} from '@kbn/observability-plugin/server'; +import { euiLightVars as theme } from '@kbn/ui-theme'; +import { + FAAS_BILLED_DURATION, + METRICSET_NAME, + METRIC_SYSTEM_TOTAL_MEMORY, + SERVICE_NAME, +} from '../../../../../common/elasticsearch_fieldnames'; +import { environmentQuery } from '../../../../../common/utils/environment_query'; +import { isFiniteNumber } from '../../../../../common/utils/is_finite_number'; +import { getVizColorForIndex } from '../../../../../common/viz_colors'; +import { getMetricsDateHistogramParams } from '../../../../lib/helpers/metrics'; +import { Setup } from '../../../../lib/helpers/setup_request'; +import { GenericMetricsChart } from '../../fetch_and_transform_metrics'; +import { ChartBase } from '../../types'; + +const chartBase: ChartBase = { + title: i18n.translate('xpack.apm.agentMetrics.serverless.computeUsage', { + defaultMessage: 'Compute usage', + }), + key: 'compute_usage', + type: 'linemark', + yUnit: 'number', + series: { + computeUsage: { + title: i18n.translate('xpack.apm.agentMetrics.serverless.computeUsage', { + defaultMessage: 'Compute usage', + }), + }, + }, +}; + +/** + * To calculate the compute usage we need to multiply the "system.memory.total" by "faas.billed_duration". + * But the result of this calculation is in Bytes-milliseconds, as the "system.memory.total" is stored in bytes and the "faas.billed_duration" is stored in milliseconds. + * But to calculate the overall cost AWS uses GB-second, so we need to convert the result to this unit. + */ +const GB = 1024 ** 3; +function calculateComputeUsageGBSeconds({ + faasBilledDuration, + totalMemory, +}: { + faasBilledDuration?: number | null; + totalMemory?: number | null; +}) { + if (!isFiniteNumber(faasBilledDuration) || !isFiniteNumber(totalMemory)) { + return 0; + } + + const totalMemoryGB = totalMemory / GB; + const faasBilledDurationSec = faasBilledDuration / 1000; + return totalMemoryGB * faasBilledDurationSec; +} + +export async function getComputeUsage({ + environment, + kuery, + setup, + serviceName, + start, + end, +}: { + environment: string; + kuery: string; + setup: Setup; + serviceName: string; + start: number; + end: number; +}): Promise { + const { apmEventClient, config } = setup; + + const aggs = { + avgFaasBilledDuration: { avg: { field: FAAS_BILLED_DURATION } }, + avgTotalMemory: { avg: { field: METRIC_SYSTEM_TOTAL_MEMORY } }, + }; + + const params = { + apm: { + events: [ProcessorEvent.metric], + }, + body: { + size: 0, + query: { + bool: { + filter: [ + { term: { [SERVICE_NAME]: serviceName } }, + ...rangeQuery(start, end), + ...environmentQuery(environment), + ...kqlQuery(kuery), + { exists: { field: FAAS_BILLED_DURATION } }, + ...termQuery(METRICSET_NAME, 'app'), + ], + }, + }, + aggs: { + timeseriesData: { + date_histogram: getMetricsDateHistogramParams({ + start, + end, + metricsInterval: config.metricsInterval, + }), + aggs, + }, + ...aggs, + }, + }, + }; + + const { aggregations } = await apmEventClient.search( + 'get_compute_usage', + params + ); + const timeseriesData = aggregations?.timeseriesData; + + return { + title: chartBase.title, + key: chartBase.key, + yUnit: chartBase.yUnit, + series: + !timeseriesData || timeseriesData.buckets.length === 0 + ? [] + : [ + { + title: i18n.translate( + 'xpack.apm.agentMetrics.serverless.computeUsage', + { defaultMessage: 'Compute usage' } + ), + key: 'compute_usage', + type: 'linemark', + overallValue: calculateComputeUsageGBSeconds({ + faasBilledDuration: aggregations?.avgFaasBilledDuration.value, + totalMemory: aggregations?.avgTotalMemory.value, + }), + color: getVizColorForIndex(0, theme), + data: timeseriesData.buckets.map((bucket) => { + const computeUsage = calculateComputeUsageGBSeconds({ + faasBilledDuration: bucket.avgFaasBilledDuration.value, + totalMemory: bucket.avgTotalMemory.value, + }); + return { + x: bucket.key, + y: computeUsage, + }; + }), + }, + ], + }; +} diff --git a/x-pack/plugins/apm/server/routes/metrics/by_agent/serverless/index.ts b/x-pack/plugins/apm/server/routes/metrics/by_agent/serverless/index.ts new file mode 100644 index 000000000000..a34165dc95f4 --- /dev/null +++ b/x-pack/plugins/apm/server/routes/metrics/by_agent/serverless/index.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { withApmSpan } from '../../../../utils/with_apm_span'; +import { Setup } from '../../../../lib/helpers/setup_request'; +import { getServerlessFunctionLatency } from './serverless_function_latency'; +import { getColdStartDuration } from './cold_start_duration'; +import { getMemoryChartData } from '../shared/memory'; +import { getComputeUsage } from './compute_usage'; +import { getActiveInstances } from './active_instances'; +import { getColdStartCount } from './cold_start_count'; +import { getSearchAggregatedTransactions } from '../../../../lib/helpers/transactions'; + +export function getServerlessAgentMetricCharts({ + environment, + kuery, + setup, + serviceName, + start, + end, +}: { + environment: string; + kuery: string; + setup: Setup; + serviceName: string; + start: number; + end: number; +}) { + return withApmSpan('get_serverless_agent_metric_charts', async () => { + const searchAggregatedTransactions = await getSearchAggregatedTransactions({ + ...setup, + kuery, + start, + end, + }); + + const options = { + environment, + kuery, + setup, + serviceName, + start, + end, + }; + return await Promise.all([ + getServerlessFunctionLatency({ + ...options, + searchAggregatedTransactions, + }), + getColdStartDuration(options), + getColdStartCount(options), + getMemoryChartData(options), + getComputeUsage(options), + getActiveInstances({ ...options, searchAggregatedTransactions }), + ]); + }); +} diff --git a/x-pack/plugins/apm/server/routes/metrics/by_agent/serverless/serverless_function_latency.ts b/x-pack/plugins/apm/server/routes/metrics/by_agent/serverless/serverless_function_latency.ts new file mode 100644 index 000000000000..99a09c86e954 --- /dev/null +++ b/x-pack/plugins/apm/server/routes/metrics/by_agent/serverless/serverless_function_latency.ts @@ -0,0 +1,123 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; +import { euiLightVars as theme } from '@kbn/ui-theme'; +import { FAAS_BILLED_DURATION } from '../../../../../common/elasticsearch_fieldnames'; +import { LatencyAggregationType } from '../../../../../common/latency_aggregation_types'; +import { getVizColorForIndex } from '../../../../../common/viz_colors'; +import { Setup } from '../../../../lib/helpers/setup_request'; +import { getLatencyTimeseries } from '../../../transactions/get_latency_charts'; +import { + fetchAndTransformMetrics, + GenericMetricsChart, +} from '../../fetch_and_transform_metrics'; +import { ChartBase } from '../../types'; + +const billedDurationAvg = { + title: i18n.translate('xpack.apm.agentMetrics.serverless.billedDurationAvg', { + defaultMessage: 'Billed Duration', + }), +}; + +const chartBase: ChartBase = { + title: i18n.translate('xpack.apm.agentMetrics.serverless.avgDuration', { + defaultMessage: 'Avg. Duration', + }), + key: 'avg_duration', + type: 'linemark', + yUnit: 'time', + series: {}, +}; + +async function getServerlessLantecySeries({ + environment, + kuery, + setup, + serviceName, + start, + end, + searchAggregatedTransactions, +}: { + environment: string; + kuery: string; + setup: Setup; + serviceName: string; + start: number; + end: number; + searchAggregatedTransactions: boolean; +}): Promise { + const transactionLatency = await getLatencyTimeseries({ + environment, + kuery, + serviceName, + setup, + searchAggregatedTransactions, + latencyAggregationType: LatencyAggregationType.avg, + start, + end, + }); + + return [ + { + title: i18n.translate( + 'xpack.apm.agentMetrics.serverless.transactionDuration', + { defaultMessage: 'Transaction Duration' } + ), + key: 'transaction_duration', + type: 'linemark', + color: getVizColorForIndex(1, theme), + overallValue: transactionLatency.overallAvgDuration ?? 0, + data: transactionLatency.latencyTimeseries, + }, + ]; +} + +export async function getServerlessFunctionLatency({ + environment, + kuery, + setup, + serviceName, + start, + end, + searchAggregatedTransactions, +}: { + environment: string; + kuery: string; + setup: Setup; + serviceName: string; + start: number; + end: number; + searchAggregatedTransactions: boolean; +}): Promise { + const options = { + environment, + kuery, + setup, + serviceName, + start, + end, + }; + + const [billedDurationMetrics, serverlessDurationSeries] = await Promise.all([ + fetchAndTransformMetrics({ + ...options, + chartBase: { ...chartBase, series: { billedDurationAvg } }, + aggs: { + billedDurationAvg: { avg: { field: FAAS_BILLED_DURATION } }, + }, + additionalFilters: [{ exists: { field: FAAS_BILLED_DURATION } }], + operationName: 'get_billed_duration', + }), + getServerlessLantecySeries({ ...options, searchAggregatedTransactions }), + ]); + + return { + ...billedDurationMetrics, + series: [...billedDurationMetrics.series, ...serverlessDurationSeries], + }; +} diff --git a/x-pack/plugins/apm/server/routes/metrics/by_agent/shared/memory/index.ts b/x-pack/plugins/apm/server/routes/metrics/by_agent/shared/memory/index.ts index 714ac5cdf38d..a7e41ea71b72 100644 --- a/x-pack/plugins/apm/server/routes/metrics/by_agent/shared/memory/index.ts +++ b/x-pack/plugins/apm/server/routes/metrics/by_agent/shared/memory/index.ts @@ -6,8 +6,10 @@ */ import { i18n } from '@kbn/i18n'; +import { termQuery } from '@kbn/observability-plugin/server'; import { withApmSpan } from '../../../../../utils/with_apm_span'; import { + FAAS_ID, METRIC_CGROUP_MEMORY_LIMIT_BYTES, METRIC_CGROUP_MEMORY_USAGE_BYTES, METRIC_SYSTEM_FREE_MEMORY, @@ -84,6 +86,7 @@ export async function getMemoryChartData({ setup, serviceName, serviceNodeName, + faasId, start, end, }: { @@ -92,6 +95,7 @@ export async function getMemoryChartData({ setup: Setup; serviceName: string; serviceNodeName?: string; + faasId?: string; start: number; end: number; }) { @@ -111,6 +115,7 @@ export async function getMemoryChartData({ }, additionalFilters: [ { exists: { field: METRIC_CGROUP_MEMORY_USAGE_BYTES } }, + ...termQuery(FAAS_ID, faasId), ], operationName: 'get_cgroup_memory_metrics_charts', }); @@ -132,6 +137,7 @@ export async function getMemoryChartData({ additionalFilters: [ { exists: { field: METRIC_SYSTEM_FREE_MEMORY } }, { exists: { field: METRIC_SYSTEM_TOTAL_MEMORY } }, + ...termQuery(FAAS_ID, faasId), ], operationName: 'get_system_memory_metrics_charts', }); diff --git a/x-pack/plugins/apm/server/routes/metrics/get_metrics_chart_data_by_agent.ts b/x-pack/plugins/apm/server/routes/metrics/get_metrics_chart_data_by_agent.ts index c6927417687d..2aa5312d66b9 100644 --- a/x-pack/plugins/apm/server/routes/metrics/get_metrics_chart_data_by_agent.ts +++ b/x-pack/plugins/apm/server/routes/metrics/get_metrics_chart_data_by_agent.ts @@ -8,8 +8,9 @@ import { Setup } from '../../lib/helpers/setup_request'; import { getJavaMetricsCharts } from './by_agent/java'; import { getDefaultMetricsCharts } from './by_agent/default'; -import { isJavaAgentName } from '../../../common/agent_name'; +import { isJavaAgentName, isServerlessAgent } from '../../../common/agent_name'; import { GenericMetricsChart } from './fetch_and_transform_metrics'; +import { getServerlessAgentMetricCharts } from './by_agent/serverless'; export async function getMetricsChartDataByAgent({ environment, @@ -20,6 +21,7 @@ export async function getMetricsChartDataByAgent({ agentName, start, end, + serviceRuntimeName, }: { environment: string; kuery: string; @@ -29,25 +31,26 @@ export async function getMetricsChartDataByAgent({ agentName: string; start: number; end: number; + serviceRuntimeName?: string; }): Promise { - if (isJavaAgentName(agentName)) { - return getJavaMetricsCharts({ - environment, - kuery, - setup, - serviceName, - serviceNodeName, - start, - end, - }); - } - - return getDefaultMetricsCharts({ + const options = { environment, kuery, setup, serviceName, start, end, - }); + }; + if (isJavaAgentName(agentName)) { + return getJavaMetricsCharts({ + ...options, + serviceNodeName, + }); + } + + if (isServerlessAgent(serviceRuntimeName)) { + return getServerlessAgentMetricCharts(options); + } + + return getDefaultMetricsCharts(options); } diff --git a/x-pack/plugins/apm/server/routes/metrics/route.ts b/x-pack/plugins/apm/server/routes/metrics/route.ts index 3ee0c3b0fac8..f8b20ae4e029 100644 --- a/x-pack/plugins/apm/server/routes/metrics/route.ts +++ b/x-pack/plugins/apm/server/routes/metrics/route.ts @@ -7,9 +7,9 @@ import * as t from 'io-ts'; import { setupRequest } from '../../lib/helpers/setup_request'; -import { getMetricsChartDataByAgent } from './get_metrics_chart_data_by_agent'; import { createApmServerRoute } from '../apm_routes/create_apm_server_route'; import { environmentRt, kueryRt, rangeRt } from '../default_api_types'; +import { getMetricsChartDataByAgent } from './get_metrics_chart_data_by_agent'; const metricsChartsRoute = createApmServerRoute({ endpoint: 'GET /internal/apm/services/{serviceName}/metrics/charts', @@ -23,6 +23,7 @@ const metricsChartsRoute = createApmServerRoute({ }), t.partial({ serviceNodeName: t.string, + serviceRuntimeName: t.string, }), environmentRt, kueryRt, @@ -50,8 +51,15 @@ const metricsChartsRoute = createApmServerRoute({ const { params } = resources; const setup = await setupRequest(resources); const { serviceName } = params.path; - const { agentName, environment, kuery, serviceNodeName, start, end } = - params.query; + const { + agentName, + environment, + kuery, + serviceNodeName, + start, + end, + serviceRuntimeName, + } = params.query; const charts = await getMetricsChartDataByAgent({ environment, @@ -62,6 +70,7 @@ const metricsChartsRoute = createApmServerRoute({ serviceNodeName, start, end, + serviceRuntimeName, }); return { charts }; diff --git a/x-pack/plugins/apm/server/routes/transactions/get_latency_charts/index.ts b/x-pack/plugins/apm/server/routes/transactions/get_latency_charts/index.ts index 82da75e56043..325642db16f9 100644 --- a/x-pack/plugins/apm/server/routes/transactions/get_latency_charts/index.ts +++ b/x-pack/plugins/apm/server/routes/transactions/get_latency_charts/index.ts @@ -136,8 +136,8 @@ export async function getLatencyTimeseries({ environment: string; kuery: string; serviceName: string; - transactionType: string | undefined; - transactionName: string | undefined; + transactionType?: string; + transactionName?: string; setup: Setup; searchAggregatedTransactions: boolean; latencyAggregationType: LatencyAggregationType; diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/warnings_popover.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/warnings_popover.tsx index 6f90bd340c1e..75d22b2feea8 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/warnings_popover.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/warnings_popover.tsx @@ -35,6 +35,7 @@ export const WarningsPopover = ({ onClick={onButtonClick} iconType="alert" className="lnsWorkspaceWarning__button" + data-test-subj="lens-editor-warning-button" > {warningsCount} @@ -53,7 +54,11 @@ export const WarningsPopover = ({ >
    {React.Children.map(children, (child, index) => ( -
  • +
  • {child}
    • ))} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index 2017ef9f6328..fbd9a5650013 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -341,7 +341,7 @@ export function DimensionEditor(props: DimensionEditorProps) { const partialIcon = compatibleWithCurrentField && referencedField?.partiallyApplicableFunctions?.[operationType] && ( - <> + {' '} - + ); let label: EuiListGroupItemProps['label'] = ( <> diff --git a/x-pack/test/apm_api_integration/tests/metrics_charts/serverless.spec.ts b/x-pack/test/apm_api_integration/tests/metrics_charts/serverless.spec.ts new file mode 100644 index 000000000000..f08e3c644c8c --- /dev/null +++ b/x-pack/test/apm_api_integration/tests/metrics_charts/serverless.spec.ts @@ -0,0 +1,431 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { APIReturnType } from '@kbn/apm-plugin/public/services/rest/create_call_apm_api'; +import { apm, timerange } from '@kbn/apm-synthtrace'; +import expect from '@kbn/expect'; +import { meanBy, sumBy } from 'lodash'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; + +export default function ApiTest({ getService }: FtrProviderContext) { + const registry = getService('registry'); + const apmApiClient = getService('apmApiClient'); + const synthtraceEsClient = getService('synthtraceEsClient'); + + const start = new Date('2021-01-01T00:00:00.000Z').getTime(); + const end = new Date('2021-01-01T00:15:00.000Z').getTime() - 1; + + async function callApi(serviceName: string, agentName: string) { + return await apmApiClient.readUser({ + endpoint: `GET /internal/apm/services/{serviceName}/metrics/charts`, + params: { + path: { serviceName }, + query: { + environment: 'test', + agentName, + kuery: '', + start: new Date(start).toISOString(), + end: new Date(end).toISOString(), + serviceRuntimeName: 'aws_lambda', + }, + }, + }); + } + + registry.when( + 'Serverless metrics charts when data is loaded', + { config: 'basic', archives: [] }, + () => { + const MEMORY_TOTAL = 536870912; // 0.5gb; + const MEMORY_FREE = 94371840; // ~0.08 gb; + const BILLED_DURATION_MS = 4000; + const FAAS_TIMEOUT_MS = 10000; + const COLD_START_DURATION_PYTHON = 4000; + const COLD_START_DURATION_NODE = 0; + const FAAS_DURATION = 4000; + const TRANSACTION_DURATION = 1000; + + const numberOfTransactionsCreated = 15; + const numberOfPythonInstances = 2; + + before(async () => { + const cloudFields = { + 'cloud.provider': 'aws', + 'cloud.service.name': 'lambda', + 'cloud.region': 'us-west-2', + }; + + const instanceLambdaPython = apm + .serverlessFunction({ + serviceName: 'lambda-python', + environment: 'test', + agentName: 'python', + functionName: 'fn-lambda-python', + }) + .instance({ instanceName: 'instance python', ...cloudFields }); + + const instanceLambdaPython2 = apm + .serverlessFunction({ + serviceName: 'lambda-python', + environment: 'test', + agentName: 'python', + functionName: 'fn-lambda-python-2', + }) + .instance({ instanceName: 'instance python 2', ...cloudFields }); + + const instanceLambdaNode = apm + .serverlessFunction({ + serviceName: 'lambda-node', + environment: 'test', + agentName: 'nodejs', + functionName: 'fn-lambda-node', + }) + .instance({ instanceName: 'instance node', ...cloudFields }); + + const systemMemory = { + free: MEMORY_FREE, + total: MEMORY_TOTAL, + }; + + const transactionsEvents = timerange(start, end) + .interval('1m') + .rate(1) + .generator((timestamp) => [ + instanceLambdaPython + .invocation() + .billedDuration(BILLED_DURATION_MS) + .coldStart(true) + .coldStartDuration(COLD_START_DURATION_PYTHON) + .faasDuration(FAAS_DURATION) + .faasTimeout(FAAS_TIMEOUT_MS) + .memory(systemMemory) + .timestamp(timestamp) + .duration(TRANSACTION_DURATION) + .success(), + instanceLambdaPython2 + .invocation() + .billedDuration(BILLED_DURATION_MS) + .coldStart(true) + .coldStartDuration(COLD_START_DURATION_PYTHON) + .faasDuration(FAAS_DURATION) + .faasTimeout(FAAS_TIMEOUT_MS) + .memory(systemMemory) + .timestamp(timestamp) + .duration(TRANSACTION_DURATION) + .success(), + instanceLambdaNode + .invocation() + .billedDuration(BILLED_DURATION_MS) + .coldStart(false) + .coldStartDuration(COLD_START_DURATION_NODE) + .faasDuration(FAAS_DURATION) + .faasTimeout(FAAS_TIMEOUT_MS) + .memory(systemMemory) + .timestamp(timestamp) + .duration(TRANSACTION_DURATION) + .success(), + ]); + + await synthtraceEsClient.index(transactionsEvents); + }); + + after(() => synthtraceEsClient.clean()); + + describe('python', () => { + let metrics: APIReturnType<'GET /internal/apm/services/{serviceName}/metrics/charts'>; + before(async () => { + const { status, body } = await callApi('lambda-python', 'python'); + + expect(status).to.be(200); + metrics = body; + }); + + it('returns all metrics chart', () => { + expect(metrics.charts.length).to.be.greaterThan(0); + expect(metrics.charts.map(({ title }) => title).sort()).to.eql([ + 'Active instances', + 'Avg. Duration', + 'Cold start', + 'Cold start duration', + 'Compute usage', + 'System memory usage', + ]); + }); + + describe('Avg. Duration', () => { + const transactionDurationInMicroSeconds = TRANSACTION_DURATION * 1000; + [ + { title: 'Billed Duration', expectedValue: BILLED_DURATION_MS }, + { title: 'Transaction Duration', expectedValue: transactionDurationInMicroSeconds }, + ].map(({ title, expectedValue }) => + it(`returns correct ${title} value`, () => { + const avgDurationMetric = metrics.charts.find((chart) => { + return chart.key === 'avg_duration'; + }); + const series = avgDurationMetric?.series.find((item) => item.title === title); + expect(series?.overallValue).to.eql(expectedValue); + const meanValue = meanBy( + series?.data.filter((item) => item.y !== null), + 'y' + ); + expect(meanValue).to.eql(expectedValue); + }) + ); + }); + + describe('Cold start duration', () => { + let coldStartDurationMetric: typeof metrics['charts'][0] | undefined; + before(() => { + coldStartDurationMetric = metrics.charts.find((chart) => { + return chart.key === 'cold_start_duration'; + }); + }); + it('returns correct overall value', () => { + expect(coldStartDurationMetric?.series[0].overallValue).to.equal( + COLD_START_DURATION_PYTHON + ); + }); + + it('returns correct mean value', () => { + const meanValue = meanBy( + coldStartDurationMetric?.series[0]?.data.filter((item) => item.y !== null), + 'y' + ); + expect(meanValue).to.equal(COLD_START_DURATION_PYTHON); + }); + }); + + describe('Cold start count', () => { + let coldStartCountMetric: typeof metrics['charts'][0] | undefined; + before(() => { + coldStartCountMetric = metrics.charts.find((chart) => { + return chart.key === 'cold_start_count'; + }); + }); + + it('returns correct overall value', () => { + expect(coldStartCountMetric?.series[0].overallValue).to.equal( + numberOfTransactionsCreated * numberOfPythonInstances + ); + }); + + it('returns correct sum value', () => { + const sumValue = sumBy( + coldStartCountMetric?.series[0]?.data.filter((item) => item.y !== null), + 'y' + ); + expect(sumValue).to.equal(numberOfTransactionsCreated * numberOfPythonInstances); + }); + }); + + describe('memory usage', () => { + const expectedFreeMemory = 1 - MEMORY_FREE / MEMORY_TOTAL; + [ + { title: 'Max', expectedValue: expectedFreeMemory }, + { title: 'Average', expectedValue: expectedFreeMemory }, + ].map(({ title, expectedValue }) => + it(`returns correct ${title} value`, () => { + const memoryUsageMetric = metrics.charts.find((chart) => { + return chart.key === 'memory_usage_chart'; + }); + const series = memoryUsageMetric?.series.find((item) => item.title === title); + expect(series?.overallValue).to.eql(expectedValue); + const meanValue = meanBy( + series?.data.filter((item) => item.y !== null), + 'y' + ); + expect(meanValue).to.eql(expectedValue); + }) + ); + }); + + describe('Compute usage', () => { + const GBSeconds = 1024 * 1024 * 1024 * 1000; + const expectedValue = (MEMORY_TOTAL * BILLED_DURATION_MS) / GBSeconds; + let computeUsageMetric: typeof metrics['charts'][0] | undefined; + before(() => { + computeUsageMetric = metrics.charts.find((chart) => { + return chart.key === 'compute_usage'; + }); + }); + it('returns correct overall value', () => { + expect(computeUsageMetric?.series[0].overallValue).to.equal(expectedValue); + }); + + it('returns correct mean value', () => { + const meanValue = meanBy( + computeUsageMetric?.series[0]?.data.filter((item) => item.y !== 0), + 'y' + ); + expect(meanValue).to.equal(expectedValue); + }); + }); + + describe('Active instances', () => { + let activeInstancesMetric: typeof metrics['charts'][0] | undefined; + before(() => { + activeInstancesMetric = metrics.charts.find((chart) => { + return chart.key === 'active_instances'; + }); + }); + it('returns correct overall value', () => { + expect(activeInstancesMetric?.series[0].overallValue).to.equal(numberOfPythonInstances); + }); + + it('returns correct sum value', () => { + const sumValue = sumBy( + activeInstancesMetric?.series[0]?.data.filter((item) => item.y !== 0), + 'y' + ); + expect(sumValue).to.equal(numberOfTransactionsCreated * numberOfPythonInstances); + }); + }); + }); + + describe('nodejs', () => { + let metrics: APIReturnType<'GET /internal/apm/services/{serviceName}/metrics/charts'>; + before(async () => { + const { status, body } = await callApi('lambda-node', 'nodejs'); + expect(status).to.be(200); + metrics = body; + }); + + it('returns all metrics chart', () => { + expect(metrics.charts.length).to.be.greaterThan(0); + expect(metrics.charts.map(({ title }) => title).sort()).to.eql([ + 'Active instances', + 'Avg. Duration', + 'Cold start', + 'Cold start duration', + 'Compute usage', + 'System memory usage', + ]); + }); + describe('Avg. Duration', () => { + const transactionDurationInMicroSeconds = TRANSACTION_DURATION * 1000; + [ + { title: 'Billed Duration', expectedValue: BILLED_DURATION_MS }, + { title: 'Transaction Duration', expectedValue: transactionDurationInMicroSeconds }, + ].map(({ title, expectedValue }) => + it(`returns correct ${title} value`, () => { + const avgDurationMetric = metrics.charts.find((chart) => { + return chart.key === 'avg_duration'; + }); + const series = avgDurationMetric?.series.find((item) => item.title === title); + expect(series?.overallValue).to.eql(expectedValue); + const meanValue = meanBy( + series?.data.filter((item) => item.y !== null), + 'y' + ); + expect(meanValue).to.eql(expectedValue); + }) + ); + }); + + describe('Cold start duration', () => { + let coldStartDurationMetric: typeof metrics['charts'][0] | undefined; + before(() => { + coldStartDurationMetric = metrics.charts.find((chart) => { + return chart.key === 'cold_start_duration'; + }); + }); + + it('returns 0 overall value', () => { + expect(coldStartDurationMetric?.series[0].overallValue).to.equal( + COLD_START_DURATION_NODE + ); + }); + + it('returns 0 mean value', () => { + const meanValue = meanBy( + coldStartDurationMetric?.series[0]?.data.filter((item) => item.y !== null), + 'y' + ); + expect(meanValue).to.equal(COLD_START_DURATION_NODE); + }); + }); + + describe('Cold start count', () => { + let coldStartCountMetric: typeof metrics['charts'][0] | undefined; + before(() => { + coldStartCountMetric = metrics.charts.find((chart) => { + return chart.key === 'cold_start_count'; + }); + }); + + it('does not return cold start count', () => { + expect(coldStartCountMetric?.series).to.be.empty(); + }); + }); + + describe('memory usage', () => { + const expectedFreeMemory = 1 - MEMORY_FREE / MEMORY_TOTAL; + [ + { title: 'Max', expectedValue: expectedFreeMemory }, + { title: 'Average', expectedValue: expectedFreeMemory }, + ].map(({ title, expectedValue }) => + it(`returns correct ${title} value`, () => { + const memoryUsageMetric = metrics.charts.find((chart) => { + return chart.key === 'memory_usage_chart'; + }); + const series = memoryUsageMetric?.series.find((item) => item.title === title); + expect(series?.overallValue).to.eql(expectedValue); + const meanValue = meanBy( + series?.data.filter((item) => item.y !== null), + 'y' + ); + expect(meanValue).to.eql(expectedValue); + }) + ); + }); + + describe('Compute usage', () => { + const GBSeconds = 1024 * 1024 * 1024 * 1000; + const expectedValue = (MEMORY_TOTAL * BILLED_DURATION_MS) / GBSeconds; + let computeUsageMetric: typeof metrics['charts'][0] | undefined; + before(() => { + computeUsageMetric = metrics.charts.find((chart) => { + return chart.key === 'compute_usage'; + }); + }); + it('returns correct overall value', () => { + expect(computeUsageMetric?.series[0].overallValue).to.equal(expectedValue); + }); + + it('returns correct mean value', () => { + const meanValue = meanBy( + computeUsageMetric?.series[0]?.data.filter((item) => item.y !== 0), + 'y' + ); + expect(meanValue).to.equal(expectedValue); + }); + }); + + describe('Active instances', () => { + let activeInstancesMetric: typeof metrics['charts'][0] | undefined; + before(() => { + activeInstancesMetric = metrics.charts.find((chart) => { + return chart.key === 'active_instances'; + }); + }); + it('returns correct overall value', () => { + // there's only one node instance + expect(activeInstancesMetric?.series[0].overallValue).to.equal(1); + }); + + it('returns correct sum value', () => { + const sumValue = sumBy( + activeInstancesMetric?.series[0]?.data.filter((item) => item.y !== 0), + 'y' + ); + expect(sumValue).to.equal(numberOfTransactionsCreated); + }); + }); + }); + } + ); +} diff --git a/x-pack/test/functional/apps/lens/group2/index.ts b/x-pack/test/functional/apps/lens/group2/index.ts index f63fc0ecebca..477ad2f8561d 100644 --- a/x-pack/test/functional/apps/lens/group2/index.ts +++ b/x-pack/test/functional/apps/lens/group2/index.ts @@ -76,5 +76,6 @@ export default ({ getService, loadTestFile, getPageObjects }: FtrProviderContext loadTestFile(require.resolve('./epoch_millis')); loadTestFile(require.resolve('./show_underlying_data')); loadTestFile(require.resolve('./show_underlying_data_dashboard')); + loadTestFile(require.resolve('./tsdb')); }); }; diff --git a/x-pack/test/functional/apps/lens/group2/tsdb.ts b/x-pack/test/functional/apps/lens/group2/tsdb.ts new file mode 100644 index 000000000000..7a43fc47471a --- /dev/null +++ b/x-pack/test/functional/apps/lens/group2/tsdb.ts @@ -0,0 +1,156 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../../ftr_provider_context'; + +export default function ({ getService, getPageObjects }: FtrProviderContext) { + const queryBar = getService('queryBar'); + const PageObjects = getPageObjects(['common', 'timePicker', 'lens', 'dashboard']); + const testSubjects = getService('testSubjects'); + const retry = getService('retry'); + const es = getService('es'); + const log = getService('log'); + const indexPatterns = getService('indexPatterns'); + const kibanaServer = getService('kibanaServer'); + const esArchiver = getService('esArchiver'); + + describe('lens tsdb', function () { + const dataViewTitle = 'sample-01'; + const rollupDataViewTitle = 'sample-01,sample-01-rollup'; + const fromTime = 'Jun 17, 2022 @ 00:00:00.000'; + const toTime = 'Jun 23, 2022 @ 00:00:00.000'; + const testArchive = 'test/functional/fixtures/es_archiver/search/downsampled'; + const testIndex = 'sample-01'; + const testRollupIndex = 'sample-01-rollup'; + + before(async () => { + // create rollup data + log.info(`loading ${testIndex} index...`); + await esArchiver.loadIfNeeded(testArchive); + log.info(`add write block to ${testIndex} index...`); + await es.indices.addBlock({ index: testIndex, block: 'write' }); + try { + log.info(`rolling up ${testIndex} index...`); + // es client currently does not have method for downsample + await es.transport.request({ + method: 'POST', + path: '/sample-01/_downsample/sample-01-rollup', + body: { fixed_interval: '1h' }, + }); + } catch (err) { + log.info(`ignoring resource_already_exists_exception...`); + if (!err.message.match(/resource_already_exists_exception/)) { + throw err; + } + } + + log.info(`creating ${rollupDataViewTitle} data view...`); + await indexPatterns.create( + { + title: rollupDataViewTitle, + timeFieldName: '@timestamp', + }, + { override: true } + ); + await indexPatterns.create( + { + title: dataViewTitle, + timeFieldName: '@timestamp', + }, + { override: true } + ); + await kibanaServer.uiSettings.update({ + 'dateFormat:tz': 'UTC', + defaultIndex: '0ae0bc7a-e4ca-405c-ab67-f2b5913f2a51', + 'timepicker:timeDefaults': '{ "from": "now-1y", "to": "now" }', + }); + }); + + after(async () => { + await kibanaServer.savedObjects.cleanStandardList(); + await kibanaServer.uiSettings.replace({}); + await es.indices.delete({ index: [testIndex, testRollupIndex] }); + }); + + describe('for regular metric', () => { + it('defaults to median for non-rolled up metric', async () => { + await PageObjects.common.navigateToApp('lens'); + await PageObjects.timePicker.setAbsoluteRange(fromTime, toTime); + await PageObjects.lens.switchDataPanelIndexPattern(dataViewTitle); + await PageObjects.lens.waitForField('kubernetes.container.memory.available.bytes'); + await PageObjects.lens.dragFieldToWorkspace( + 'kubernetes.container.memory.available.bytes', + 'xyVisChart' + ); + expect(await PageObjects.lens.getDimensionTriggerText('lnsXY_yDimensionPanel')).to.eql( + 'Median of kubernetes.container.memory.available.bytes' + ); + }); + + it('does not show a warning', async () => { + await PageObjects.lens.openDimensionEditor('lnsXY_yDimensionPanel'); + await testSubjects.missingOrFail('median-partial-warning'); + await PageObjects.lens.assertNoEditorWarning(); + await PageObjects.lens.closeDimensionEditor(); + }); + }); + + describe('for rolled up metric', () => { + it('defaults to average for rolled up metric', async () => { + await PageObjects.lens.switchDataPanelIndexPattern(rollupDataViewTitle); + await PageObjects.lens.removeLayer(); + await PageObjects.lens.waitForField('kubernetes.container.memory.available.bytes'); + await PageObjects.lens.dragFieldToWorkspace( + 'kubernetes.container.memory.available.bytes', + 'xyVisChart' + ); + expect(await PageObjects.lens.getDimensionTriggerText('lnsXY_yDimensionPanel')).to.eql( + 'Average of kubernetes.container.memory.available.bytes' + ); + }); + it('shows warnings in editor when using median', async () => { + await PageObjects.lens.openDimensionEditor('lnsXY_yDimensionPanel'); + await testSubjects.existOrFail('median-partial-warning'); + await testSubjects.click('lns-indexPatternDimension-median'); + await PageObjects.lens.waitForVisualization('xyVisChart'); + await PageObjects.lens.assertEditorWarning( + '"Median of kubernetes.container.memory.available.bytes" does not work for all indices in the selected data view because it\'s using a function which is not supported on rolled up data. Please edit the visualization to use another function or change the time range.' + ); + }); + it('shows warnings in dashboards as well', async () => { + await PageObjects.lens.save('New', false, false, false, 'new'); + + await PageObjects.dashboard.waitForRenderComplete(); + await PageObjects.lens.assertInlineWarning( + '"Median of kubernetes.container.memory.available.bytes" does not work for all indices in the selected data view because it\'s using a function which is not supported on rolled up data. Please edit the visualization to use another function or change the time range.' + ); + }); + it('still shows other warnings as toast', async () => { + await es.indices.delete({ index: [testRollupIndex] }); + // index a document which will produce a shard failure because a string field doesn't support median + await es.create({ + id: '1', + index: testRollupIndex, + document: { + 'kubernetes.container.memory.available.bytes': 'fsdfdsf', + '@timestamp': '2022-06-20', + }, + wait_for_active_shards: 1, + }); + await retry.try(async () => { + await queryBar.clickQuerySubmitButton(); + expect( + await (await testSubjects.find('euiToastHeader__title', 1000)).getVisibleText() + ).to.equal('1 of 3 shards failed'); + }); + // as the rollup index is gone, there is no inline warning left + await PageObjects.lens.assertNoInlineWarning(); + }); + }); + }); +} diff --git a/x-pack/test/functional/page_objects/lens_page.ts b/x-pack/test/functional/page_objects/lens_page.ts index f45b8ad9c22d..20412284aa92 100644 --- a/x-pack/test/functional/page_objects/lens_page.ts +++ b/x-pack/test/functional/page_objects/lens_page.ts @@ -507,8 +507,8 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont // closes the dimension editor flyout async closeDimensionEditor() { await retry.try(async () => { - await testSubjects.click('lns-indexPattern-dimensionContainerBack'); - await testSubjects.missingOrFail('lns-indexPattern-dimensionContainerBack'); + await testSubjects.click('lns-indexPattern-dimensionContainerClose'); + await testSubjects.missingOrFail('lns-indexPattern-dimensionContainerClose'); }); }, @@ -1462,5 +1462,49 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont await testSubjects.waitForEnabled(applyButtonSelector); await testSubjects.click(applyButtonSelector); }, + + async assertNoInlineWarning() { + await testSubjects.missingOrFail('chart-inline-warning'); + }, + + async assertNoEditorWarning() { + await testSubjects.missingOrFail('lens-editor-warning'); + }, + + async assertInlineWarning(warningText: string) { + await testSubjects.click('chart-inline-warning-button'); + await testSubjects.existOrFail('chart-inline-warning'); + const warnings = await testSubjects.findAll('chart-inline-warning'); + let found = false; + for (const warning of warnings) { + const text = await warning.getVisibleText(); + log.info(text); + if (text === warningText) { + found = true; + } + } + await testSubjects.click('chart-inline-warning-button'); + if (!found) { + throw new Error(`Warning with text "${warningText}" not found`); + } + }, + + async assertEditorWarning(warningText: string) { + await testSubjects.click('lens-editor-warning-button'); + await testSubjects.existOrFail('lens-editor-warning'); + const warnings = await testSubjects.findAll('lens-editor-warning'); + let found = false; + for (const warning of warnings) { + const text = await warning.getVisibleText(); + log.info(text); + if (text === warningText) { + found = true; + } + } + await testSubjects.click('lens-editor-warning-button'); + if (!found) { + throw new Error(`Warning with text "${warningText}" not found`); + } + }, }); }