[8.8] [APM] Check if metric fields exist (#145348) (#156006)

# Backport This will backport the following commits from `main` to `8.8`: - [[APM] Check if metric fields exist (#145348)](#145348)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)  Co-authored-by: Katerina Patticha <[email protected]>
elastic · May 3, 2023 · b64f2f2 · b64f2f2
1 parent e204707
commit b64f2f2
Show file tree

Hide file tree

Showing 8 changed files with 263 additions and 35 deletions.
diff --git a/packages/kbn-apm-synthtrace-client/src/lib/apm/apm_fields.ts b/packages/kbn-apm-synthtrace-client/src/lib/apm/apm_fields.ts
@@ -12,6 +12,8 @@ export type ApmApplicationMetricFields = Partial<{
   'system.process.memory.size': number;
   'system.memory.actual.free': number;
   'system.memory.total': number;
+  'system.process.cgroup.memory.mem.limit.bytes': number;
+  'system.process.cgroup.memory.mem.usage.bytes': number;
   'system.cpu.total.norm.pct': number;
   'system.process.memory.rss.bytes': number;
   'system.process.cpu.total.norm.pct': number;

diff --git a/x-pack/plugins/apm/server/routes/metrics/__snapshots__/queries.test.ts.snap b/x-pack/plugins/apm/server/routes/metrics/__snapshots__/queries.test.ts.snap
diff --git a/x-pack/plugins/apm/server/routes/metrics/by_agent/shared/memory/index.ts b/x-pack/plugins/apm/server/routes/metrics/by_agent/shared/memory/index.ts
@@ -47,6 +47,15 @@ const chartBase: ChartBase = {
   series,
 };
 
+export const systemMemoryFilter = {
+  bool: {
+    filter: [
+      { exists: { field: METRIC_SYSTEM_FREE_MEMORY } },
+      { exists: { field: METRIC_SYSTEM_TOTAL_MEMORY } },
+    ],
+  },
+};
+
 export const percentSystemMemoryUsedScript = {
   lang: 'painless',
   source: `
@@ -60,6 +69,17 @@ export const percentSystemMemoryUsedScript = {
   `,
 } as const;
 
+export const cgroupMemoryFilter = {
+  bool: {
+    filter: [{ exists: { field: METRIC_CGROUP_MEMORY_USAGE_BYTES } }],
+    should: [
+      { exists: { field: METRIC_CGROUP_MEMORY_LIMIT_BYTES } },
+      { exists: { field: METRIC_SYSTEM_TOTAL_MEMORY } },
+    ],
+    minimum_should_match: 1,
+  },
+};
+
 export const percentCgroupMemoryUsedScript = {
   lang: 'painless',
   source: `
@@ -147,7 +167,7 @@ export async function getMemoryChartData({
           memoryUsedMax: { max: { script: percentCgroupMemoryUsedScript } },
         },
         additionalFilters: [
-          { exists: { field: METRIC_CGROUP_MEMORY_USAGE_BYTES } },
+          cgroupMemoryFilter,
           ...termQuery(FAAS_ID, serverlessId),
         ],
         operationName: 'get_cgroup_memory_metrics_charts',
@@ -169,8 +189,7 @@ export async function getMemoryChartData({
             memoryUsedMax: { max: { script: percentSystemMemoryUsedScript } },
           },
           additionalFilters: [
-            { exists: { field: METRIC_SYSTEM_FREE_MEMORY } },
-            { exists: { field: METRIC_SYSTEM_TOTAL_MEMORY } },
+            systemMemoryFilter,
             ...termQuery(FAAS_ID, serverlessId),
           ],
           operationName: 'get_system_memory_metrics_charts',

diff --git a/x-pack/plugins/apm/server/routes/service_map/get_service_map_service_node_info.ts b/x-pack/plugins/apm/server/routes/service_map/get_service_map_service_node_info.ts
@@ -9,10 +9,7 @@ import type { ESFilter } from '@kbn/es-types';
 import { rangeQuery } from '@kbn/observability-plugin/server';
 import { ProcessorEvent } from '@kbn/observability-plugin/common';
 import {
-  METRIC_CGROUP_MEMORY_USAGE_BYTES,
   METRIC_SYSTEM_CPU_PERCENT,
-  METRIC_SYSTEM_FREE_MEMORY,
-  METRIC_SYSTEM_TOTAL_MEMORY,
   SERVICE_NAME,
   TRANSACTION_TYPE,
 } from '../../../common/es_fields/apm';
@@ -31,6 +28,8 @@ import { withApmSpan } from '../../utils/with_apm_span';
 import {
   percentCgroupMemoryUsedScript,
   percentSystemMemoryUsedScript,
+  systemMemoryFilter,
+  cgroupMemoryFilter,
 } from '../metrics/by_agent/shared/memory';
 import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client';
 import { ApmDocumentType } from '../../../common/document_type';
@@ -358,19 +357,14 @@ function getMemoryStats({
     };
 
     let memoryUsage = await getMemoryUsage({
-      additionalFilters: [
-        { exists: { field: METRIC_CGROUP_MEMORY_USAGE_BYTES } },
-      ],
       script: percentCgroupMemoryUsedScript,
+      additionalFilters: [cgroupMemoryFilter],
     });
 
     if (!memoryUsage) {
       memoryUsage = await getMemoryUsage({
-        additionalFilters: [
-          { exists: { field: METRIC_SYSTEM_FREE_MEMORY } },
-          { exists: { field: METRIC_SYSTEM_TOTAL_MEMORY } },
-        ],
         script: percentSystemMemoryUsedScript,
+        additionalFilters: [systemMemoryFilter],
       });
     }
 

diff --git a/...r/routes/services/get_service_instances/get_service_instances_system_metric_statistics.ts b/...r/routes/services/get_service_instances/get_service_instances_system_metric_statistics.ts
@@ -9,10 +9,7 @@ import type { AggregationOptionsByType } from '@kbn/es-types';
 import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server';
 import { ProcessorEvent } from '@kbn/observability-plugin/common';
 import {
-  METRIC_CGROUP_MEMORY_USAGE_BYTES,
   METRIC_PROCESS_CPU_PERCENT,
-  METRIC_SYSTEM_FREE_MEMORY,
-  METRIC_SYSTEM_TOTAL_MEMORY,
   SERVICE_NAME,
   SERVICE_NODE_NAME,
 } from '../../../../common/es_fields/apm';
@@ -24,6 +21,8 @@ import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm
 import {
   percentCgroupMemoryUsedScript,
   percentSystemMemoryUsedScript,
+  systemMemoryFilter,
+  cgroupMemoryFilter,
 } from '../../metrics/by_agent/shared/memory';
 import { getOffsetInMs } from '../../../../common/utils/get_offset_in_ms';
 
@@ -82,19 +81,6 @@ export async function getServiceInstancesSystemMetricStatistics<
     numBuckets,
   });
 
-  const systemMemoryFilter = {
-    bool: {
-      filter: [
-        { exists: { field: METRIC_SYSTEM_FREE_MEMORY } },
-        { exists: { field: METRIC_SYSTEM_TOTAL_MEMORY } },
-      ],
-    },
-  };
-
-  const cgroupMemoryFilter = {
-    exists: { field: METRIC_CGROUP_MEMORY_USAGE_BYTES },
-  };
-
   const cpuUsageFilter = { exists: { field: METRIC_PROCESS_CPU_PERCENT } };
 
   function withTimeseries<TParams extends AggregationOptionsByType['avg']>(

diff --git a/x-pack/test/apm_api_integration/tests/metrics/memory/generate_data.ts b/x-pack/test/apm_api_integration/tests/metrics/memory/generate_data.ts
@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { apm, timerange } from '@kbn/apm-synthtrace-client';
+import type { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace';
+
+const gigabytesToBytes = (value: number) => value * Math.pow(1024, 3);
+
+export const config = {
+  memoryTotal: gigabytesToBytes(64),
+  memoryFree: gigabytesToBytes(5.5),
+  cGroupMemoryLimit: gigabytesToBytes(8),
+  cGroupMemoryUsage: gigabytesToBytes(1.5),
+};
+
+export const expectedValues = {
+  expectedMemoryUsedRate: (config.memoryTotal - config.memoryFree) / config.memoryTotal,
+  expectedMemoryUsed: config.memoryTotal - config.memoryFree,
+};
+
+export async function generateData({
+  synthtraceEsClient,
+  start,
+  end,
+}: {
+  synthtraceEsClient: ApmSynthtraceEsClient;
+  start: number;
+  end: number;
+}) {
+  const { memoryTotal, memoryFree, cGroupMemoryLimit, cGroupMemoryUsage } = config;
+
+  const systemMetricOnlyInstance = apm
+    .service({ name: 'system-metric-only-service', environment: 'production', agentName: 'go' })
+    .instance('system-metric-only-production');
+
+  const cGroupMemoryOnlyInstance = apm
+    .service({ name: 'cgroup-memory-only-service', environment: 'production', agentName: 'go' })
+    .instance('cgroup-memory-only-production');
+
+  const cGroupMemoryWithLimitInstance = apm
+    .service({
+      name: 'cgroup-memory-with-limit-production',
+      environment: 'production',
+      agentName: 'go',
+    })
+    .instance('cgroup-memory-with-limit-production');
+
+  const transactionsEvents = timerange(start, end)
+    .ratePerMinute(1)
+    .generator((timestamp) => [
+      systemMetricOnlyInstance
+        .appMetrics({
+          'system.memory.actual.free': memoryFree,
+          'system.memory.total': memoryTotal,
+        })
+        .timestamp(timestamp),
+      cGroupMemoryOnlyInstance
+        .appMetrics({
+          'system.process.cgroup.memory.mem.usage.bytes': cGroupMemoryUsage,
+        })
+        .timestamp(timestamp),
+
+      cGroupMemoryWithLimitInstance
+        .appMetrics({
+          'system.process.cgroup.memory.mem.usage.bytes': cGroupMemoryUsage,
+          'system.process.cgroup.memory.mem.limit.bytes': cGroupMemoryLimit,
+          'system.memory.total': memoryTotal,
+          'system.memory.actual.free': memoryFree,
+        })
+        .timestamp(timestamp),
+    ]);
+
+  await synthtraceEsClient.index(transactionsEvents);
+}
diff --git a/x-pack/test/apm_api_integration/tests/metrics/memory/memory_metrics.spec.ts b/x-pack/test/apm_api_integration/tests/metrics/memory/memory_metrics.spec.ts
@@ -0,0 +1,83 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import { FtrProviderContext } from '../../../common/ftr_provider_context';
+import { config, generateData } from './generate_data';
+
+export default function ApiTest({ getService }: FtrProviderContext) {
+  const registry = getService('registry');
+  const apmApiClient = getService('apmApiClient');
+  const synthtraceEsClient = getService('synthtraceEsClient');
+
+  const start = new Date('2023-01-01T00:00:00.000Z').getTime();
+  const end = new Date('2023-01-01T00:15:00.000Z').getTime() - 1;
+
+  async function callMetricChartsAPI(serviceName: string) {
+    return await apmApiClient.readUser({
+      endpoint: `GET /internal/apm/services/{serviceName}/metrics/charts`,
+      params: {
+        path: { serviceName },
+        query: {
+          environment: 'production',
+          start: new Date(start).toISOString(),
+          end: new Date(end).toISOString(),
+          kuery: '',
+          agentName: 'go',
+        },
+      },
+    });
+  }
+
+  registry.when('Memory', { config: 'trial', archives: [] }, () => {
+    before(async () => {
+      await generateData({ start, end, synthtraceEsClient });
+    });
+
+    after(() => synthtraceEsClient.clean());
+
+    it('returns system memory stats', async () => {
+      const expectedFreeMemory = 1 - config.memoryFree / config.memoryTotal;
+
+      const { status, body } = await callMetricChartsAPI('system-metric-only-service');
+      const memoryChart = body.charts.find(({ key }) => key === 'memory_usage_chart');
+
+      expect(status).to.be(200);
+      [
+        { title: 'Max', expectedValue: expectedFreeMemory },
+        { title: 'Average', expectedValue: expectedFreeMemory },
+      ].map(({ title, expectedValue }) => {
+        const series = memoryChart?.series.find((item) => item.title === title);
+        expect(series?.overallValue).to.eql(expectedValue);
+      });
+    });
+
+    it('returns cgroup memory with system.process.cgroup.memory.mem.limit.bytes', async () => {
+      const expectedFreeMemory = config.cGroupMemoryUsage / config.cGroupMemoryLimit;
+
+      const { status, body } = await callMetricChartsAPI('cgroup-memory-with-limit-production');
+      const memoryChart = body.charts.find(({ key }) => key === 'memory_usage_chart');
+
+      expect(status).to.be(200);
+      [
+        { title: 'Max', expectedValue: expectedFreeMemory },
+        { title: 'Average', expectedValue: expectedFreeMemory },
+      ].map(({ title, expectedValue }) => {
+        const series = memoryChart?.series.find((item) => item.title === title);
+        expect(series?.overallValue).to.eql(expectedValue);
+      });
+    });
+
+    it('handles cgroup memory stats when system.process.cgroup.memory.mem.limit.bytes and system.memory.total are not present', async () => {
+      const { status, body } = await callMetricChartsAPI('cgroup-memory-only-production');
+      expect(status).to.be(200);
+
+      const memoryChart = body.charts.find(({ key }) => key === 'memory_usage_chart');
+      expect(memoryChart?.series).to.eql([]);
+    });
+  });
+}