diff --git a/packages/kbn-apm-synthtrace-client/src/lib/apm/apm_fields.ts b/packages/kbn-apm-synthtrace-client/src/lib/apm/apm_fields.ts index d5efb1dbc2c97..911271f584152 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/apm/apm_fields.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/apm/apm_fields.ts @@ -12,6 +12,8 @@ export type ApmApplicationMetricFields = Partial<{ 'system.process.memory.size': number; 'system.memory.actual.free': number; 'system.memory.total': number; + 'system.process.cgroup.memory.mem.limit.bytes': number; + 'system.process.cgroup.memory.mem.usage.bytes': number; 'system.cpu.total.norm.pct': number; 'system.process.memory.rss.bytes': number; 'system.process.cpu.total.norm.pct': number; diff --git a/x-pack/plugins/apm/server/routes/metrics/__snapshots__/queries.test.ts.snap b/x-pack/plugins/apm/server/routes/metrics/__snapshots__/queries.test.ts.snap index 9648e4f8d8ec5..44e4c6335ec7c 100644 --- a/x-pack/plugins/apm/server/routes/metrics/__snapshots__/queries.test.ts.snap +++ b/x-pack/plugins/apm/server/routes/metrics/__snapshots__/queries.test.ts.snap @@ -332,8 +332,27 @@ Object { }, }, Object { - "exists": Object { - "field": "system.process.cgroup.memory.mem.usage.bytes", + "bool": Object { + "filter": Array [ + Object { + "exists": Object { + "field": "system.process.cgroup.memory.mem.usage.bytes", + }, + }, + ], + "minimum_should_match": 1, + "should": Array [ + Object { + "exists": Object { + "field": "system.process.cgroup.memory.mem.limit.bytes", + }, + }, + Object { + "exists": Object { + "field": "system.memory.total", + }, + }, + ], }, }, ], @@ -869,8 +888,27 @@ Object { }, }, Object { - "exists": Object { - "field": "system.process.cgroup.memory.mem.usage.bytes", + "bool": Object { + "filter": Array [ + Object { + "exists": Object { + "field": "system.process.cgroup.memory.mem.usage.bytes", + }, + }, + ], + "minimum_should_match": 1, + "should": Array [ + Object { + "exists": Object { + "field": "system.process.cgroup.memory.mem.limit.bytes", + }, + }, + Object { + "exists": Object { + "field": "system.memory.total", + }, + }, + ], }, }, ], @@ -1385,8 +1423,27 @@ Object { }, }, Object { - "exists": Object { - "field": "system.process.cgroup.memory.mem.usage.bytes", + "bool": Object { + "filter": Array [ + Object { + "exists": Object { + "field": "system.process.cgroup.memory.mem.usage.bytes", + }, + }, + ], + "minimum_should_match": 1, + "should": Array [ + Object { + "exists": Object { + "field": "system.process.cgroup.memory.mem.limit.bytes", + }, + }, + Object { + "exists": Object { + "field": "system.memory.total", + }, + }, + ], }, }, ], diff --git a/x-pack/plugins/apm/server/routes/metrics/by_agent/shared/memory/index.ts b/x-pack/plugins/apm/server/routes/metrics/by_agent/shared/memory/index.ts index 5c7993c4c9f2d..2afc82181cbfb 100644 --- a/x-pack/plugins/apm/server/routes/metrics/by_agent/shared/memory/index.ts +++ b/x-pack/plugins/apm/server/routes/metrics/by_agent/shared/memory/index.ts @@ -47,6 +47,15 @@ const chartBase: ChartBase = { series, }; +export const systemMemoryFilter = { + bool: { + filter: [ + { exists: { field: METRIC_SYSTEM_FREE_MEMORY } }, + { exists: { field: METRIC_SYSTEM_TOTAL_MEMORY } }, + ], + }, +}; + export const percentSystemMemoryUsedScript = { lang: 'painless', source: ` @@ -60,6 +69,17 @@ export const percentSystemMemoryUsedScript = { `, } as const; +export const cgroupMemoryFilter = { + bool: { + filter: [{ exists: { field: METRIC_CGROUP_MEMORY_USAGE_BYTES } }], + should: [ + { exists: { field: METRIC_CGROUP_MEMORY_LIMIT_BYTES } }, + { exists: { field: METRIC_SYSTEM_TOTAL_MEMORY } }, + ], + minimum_should_match: 1, + }, +}; + export const percentCgroupMemoryUsedScript = { lang: 'painless', source: ` @@ -147,7 +167,7 @@ export async function getMemoryChartData({ memoryUsedMax: { max: { script: percentCgroupMemoryUsedScript } }, }, additionalFilters: [ - { exists: { field: METRIC_CGROUP_MEMORY_USAGE_BYTES } }, + cgroupMemoryFilter, ...termQuery(FAAS_ID, serverlessId), ], operationName: 'get_cgroup_memory_metrics_charts', @@ -169,8 +189,7 @@ export async function getMemoryChartData({ memoryUsedMax: { max: { script: percentSystemMemoryUsedScript } }, }, additionalFilters: [ - { exists: { field: METRIC_SYSTEM_FREE_MEMORY } }, - { exists: { field: METRIC_SYSTEM_TOTAL_MEMORY } }, + systemMemoryFilter, ...termQuery(FAAS_ID, serverlessId), ], operationName: 'get_system_memory_metrics_charts', diff --git a/x-pack/plugins/apm/server/routes/service_map/get_service_map_service_node_info.ts b/x-pack/plugins/apm/server/routes/service_map/get_service_map_service_node_info.ts index 87963c19e6e08..d17811c8aea39 100644 --- a/x-pack/plugins/apm/server/routes/service_map/get_service_map_service_node_info.ts +++ b/x-pack/plugins/apm/server/routes/service_map/get_service_map_service_node_info.ts @@ -9,10 +9,7 @@ import type { ESFilter } from '@kbn/es-types'; import { rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; import { - METRIC_CGROUP_MEMORY_USAGE_BYTES, METRIC_SYSTEM_CPU_PERCENT, - METRIC_SYSTEM_FREE_MEMORY, - METRIC_SYSTEM_TOTAL_MEMORY, SERVICE_NAME, TRANSACTION_TYPE, } from '../../../common/es_fields/apm'; @@ -31,6 +28,8 @@ import { withApmSpan } from '../../utils/with_apm_span'; import { percentCgroupMemoryUsedScript, percentSystemMemoryUsedScript, + systemMemoryFilter, + cgroupMemoryFilter, } from '../metrics/by_agent/shared/memory'; import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; import { ApmDocumentType } from '../../../common/document_type'; @@ -358,19 +357,14 @@ function getMemoryStats({ }; let memoryUsage = await getMemoryUsage({ - additionalFilters: [ - { exists: { field: METRIC_CGROUP_MEMORY_USAGE_BYTES } }, - ], script: percentCgroupMemoryUsedScript, + additionalFilters: [cgroupMemoryFilter], }); if (!memoryUsage) { memoryUsage = await getMemoryUsage({ - additionalFilters: [ - { exists: { field: METRIC_SYSTEM_FREE_MEMORY } }, - { exists: { field: METRIC_SYSTEM_TOTAL_MEMORY } }, - ], script: percentSystemMemoryUsedScript, + additionalFilters: [systemMemoryFilter], }); } diff --git a/x-pack/plugins/apm/server/routes/services/get_service_instances/get_service_instances_system_metric_statistics.ts b/x-pack/plugins/apm/server/routes/services/get_service_instances/get_service_instances_system_metric_statistics.ts index 7462b649a6019..6a5f57cfe9d2f 100644 --- a/x-pack/plugins/apm/server/routes/services/get_service_instances/get_service_instances_system_metric_statistics.ts +++ b/x-pack/plugins/apm/server/routes/services/get_service_instances/get_service_instances_system_metric_statistics.ts @@ -9,10 +9,7 @@ import type { AggregationOptionsByType } from '@kbn/es-types'; import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; import { - METRIC_CGROUP_MEMORY_USAGE_BYTES, METRIC_PROCESS_CPU_PERCENT, - METRIC_SYSTEM_FREE_MEMORY, - METRIC_SYSTEM_TOTAL_MEMORY, SERVICE_NAME, SERVICE_NODE_NAME, } from '../../../../common/es_fields/apm'; @@ -24,6 +21,8 @@ import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm import { percentCgroupMemoryUsedScript, percentSystemMemoryUsedScript, + systemMemoryFilter, + cgroupMemoryFilter, } from '../../metrics/by_agent/shared/memory'; import { getOffsetInMs } from '../../../../common/utils/get_offset_in_ms'; @@ -82,19 +81,6 @@ export async function getServiceInstancesSystemMetricStatistics< numBuckets, }); - const systemMemoryFilter = { - bool: { - filter: [ - { exists: { field: METRIC_SYSTEM_FREE_MEMORY } }, - { exists: { field: METRIC_SYSTEM_TOTAL_MEMORY } }, - ], - }, - }; - - const cgroupMemoryFilter = { - exists: { field: METRIC_CGROUP_MEMORY_USAGE_BYTES }, - }; - const cpuUsageFilter = { exists: { field: METRIC_PROCESS_CPU_PERCENT } }; function withTimeseries( diff --git a/x-pack/test/apm_api_integration/tests/metrics/memory/generate_data.ts b/x-pack/test/apm_api_integration/tests/metrics/memory/generate_data.ts new file mode 100644 index 0000000000000..22abd71989206 --- /dev/null +++ b/x-pack/test/apm_api_integration/tests/metrics/memory/generate_data.ts @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { apm, timerange } from '@kbn/apm-synthtrace-client'; +import type { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace'; + +const gigabytesToBytes = (value: number) => value * Math.pow(1024, 3); + +export const config = { + memoryTotal: gigabytesToBytes(64), + memoryFree: gigabytesToBytes(5.5), + cGroupMemoryLimit: gigabytesToBytes(8), + cGroupMemoryUsage: gigabytesToBytes(1.5), +}; + +export const expectedValues = { + expectedMemoryUsedRate: (config.memoryTotal - config.memoryFree) / config.memoryTotal, + expectedMemoryUsed: config.memoryTotal - config.memoryFree, +}; + +export async function generateData({ + synthtraceEsClient, + start, + end, +}: { + synthtraceEsClient: ApmSynthtraceEsClient; + start: number; + end: number; +}) { + const { memoryTotal, memoryFree, cGroupMemoryLimit, cGroupMemoryUsage } = config; + + const systemMetricOnlyInstance = apm + .service({ name: 'system-metric-only-service', environment: 'production', agentName: 'go' }) + .instance('system-metric-only-production'); + + const cGroupMemoryOnlyInstance = apm + .service({ name: 'cgroup-memory-only-service', environment: 'production', agentName: 'go' }) + .instance('cgroup-memory-only-production'); + + const cGroupMemoryWithLimitInstance = apm + .service({ + name: 'cgroup-memory-with-limit-production', + environment: 'production', + agentName: 'go', + }) + .instance('cgroup-memory-with-limit-production'); + + const transactionsEvents = timerange(start, end) + .ratePerMinute(1) + .generator((timestamp) => [ + systemMetricOnlyInstance + .appMetrics({ + 'system.memory.actual.free': memoryFree, + 'system.memory.total': memoryTotal, + }) + .timestamp(timestamp), + cGroupMemoryOnlyInstance + .appMetrics({ + 'system.process.cgroup.memory.mem.usage.bytes': cGroupMemoryUsage, + }) + .timestamp(timestamp), + + cGroupMemoryWithLimitInstance + .appMetrics({ + 'system.process.cgroup.memory.mem.usage.bytes': cGroupMemoryUsage, + 'system.process.cgroup.memory.mem.limit.bytes': cGroupMemoryLimit, + 'system.memory.total': memoryTotal, + 'system.memory.actual.free': memoryFree, + }) + .timestamp(timestamp), + ]); + + await synthtraceEsClient.index(transactionsEvents); +} diff --git a/x-pack/test/apm_api_integration/tests/metrics/memory/memory_metrics.spec.ts b/x-pack/test/apm_api_integration/tests/metrics/memory/memory_metrics.spec.ts new file mode 100644 index 0000000000000..d67451e4d1abc --- /dev/null +++ b/x-pack/test/apm_api_integration/tests/metrics/memory/memory_metrics.spec.ts @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../../common/ftr_provider_context'; +import { config, generateData } from './generate_data'; + +export default function ApiTest({ getService }: FtrProviderContext) { + const registry = getService('registry'); + const apmApiClient = getService('apmApiClient'); + const synthtraceEsClient = getService('synthtraceEsClient'); + + const start = new Date('2023-01-01T00:00:00.000Z').getTime(); + const end = new Date('2023-01-01T00:15:00.000Z').getTime() - 1; + + async function callMetricChartsAPI(serviceName: string) { + return await apmApiClient.readUser({ + endpoint: `GET /internal/apm/services/{serviceName}/metrics/charts`, + params: { + path: { serviceName }, + query: { + environment: 'production', + start: new Date(start).toISOString(), + end: new Date(end).toISOString(), + kuery: '', + agentName: 'go', + }, + }, + }); + } + + registry.when('Memory', { config: 'trial', archives: [] }, () => { + before(async () => { + await generateData({ start, end, synthtraceEsClient }); + }); + + after(() => synthtraceEsClient.clean()); + + it('returns system memory stats', async () => { + const expectedFreeMemory = 1 - config.memoryFree / config.memoryTotal; + + const { status, body } = await callMetricChartsAPI('system-metric-only-service'); + const memoryChart = body.charts.find(({ key }) => key === 'memory_usage_chart'); + + expect(status).to.be(200); + [ + { title: 'Max', expectedValue: expectedFreeMemory }, + { title: 'Average', expectedValue: expectedFreeMemory }, + ].map(({ title, expectedValue }) => { + const series = memoryChart?.series.find((item) => item.title === title); + expect(series?.overallValue).to.eql(expectedValue); + }); + }); + + it('returns cgroup memory with system.process.cgroup.memory.mem.limit.bytes', async () => { + const expectedFreeMemory = config.cGroupMemoryUsage / config.cGroupMemoryLimit; + + const { status, body } = await callMetricChartsAPI('cgroup-memory-with-limit-production'); + const memoryChart = body.charts.find(({ key }) => key === 'memory_usage_chart'); + + expect(status).to.be(200); + [ + { title: 'Max', expectedValue: expectedFreeMemory }, + { title: 'Average', expectedValue: expectedFreeMemory }, + ].map(({ title, expectedValue }) => { + const series = memoryChart?.series.find((item) => item.title === title); + expect(series?.overallValue).to.eql(expectedValue); + }); + }); + + it('handles cgroup memory stats when system.process.cgroup.memory.mem.limit.bytes and system.memory.total are not present', async () => { + const { status, body } = await callMetricChartsAPI('cgroup-memory-only-production'); + expect(status).to.be(200); + + const memoryChart = body.charts.find(({ key }) => key === 'memory_usage_chart'); + expect(memoryChart?.series).to.eql([]); + }); + }); +} diff --git a/x-pack/test/apm_api_integration/tests/service_overview/instances_main_statistics.spec.ts b/x-pack/test/apm_api_integration/tests/service_overview/instances_main_statistics.spec.ts index 7a5e2f2f40780..f215882c09bd2 100644 --- a/x-pack/test/apm_api_integration/tests/service_overview/instances_main_statistics.spec.ts +++ b/x-pack/test/apm_api_integration/tests/service_overview/instances_main_statistics.spec.ts @@ -418,6 +418,16 @@ export default function ApiTest({ getService }: FtrProviderContext) { expect(javaStats).to.be(undefined); }); + it('does not return metrics', () => { + const goAStats = body.currentPeriod.find( + (stat) => stat.serviceNodeName === 'go-instance-a' + ); + + expect(goAStats).to.not.be(undefined); + expect(goAStats?.memoryUsage).to.be(undefined); + expect(goAStats?.cpuUsage).to.be(undefined); + }); + it('does not return data for missing service node name', () => { const missingNameStats = body.currentPeriod.find( (stat) => stat.serviceNodeName === SERVICE_NODE_NAME_MISSING