From 03a86d90e9d73e550d976965f780fab8226e5c20 Mon Sep 17 00:00:00 2001 From: Milosz Marcinkowski Date: Wed, 18 Sep 2024 13:52:55 +0200 Subject: [PATCH 01/25] 'dependency metadata' mappings --- .../get_metadata_for_dependency.ts | 5 ++--- .../apm/server/utils/es_fields_mappings.ts | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) create mode 100644 x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts diff --git a/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_metadata_for_dependency.ts b/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_metadata_for_dependency.ts index 9d7e1d7a44e14..d54f422baa136 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_metadata_for_dependency.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_metadata_for_dependency.ts @@ -7,11 +7,10 @@ import { rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; -import type { Span } from '@kbn/apm-types/es_schemas_ui'; import { maybe } from '../../../common/utils/maybe'; import { SPAN_DESTINATION_SERVICE_RESOURCE } from '../../../common/es_fields/apm'; import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; -import { normalizeFields } from '../../utils/normalize_fields'; +import { metadataForDependencyMapping } from '../../utils/es_fields_mappings'; export interface MetadataForDependencyResponse { spanType: string | undefined; @@ -56,7 +55,7 @@ export async function getMetadataForDependency({ }); const sample = maybe(sampleResponse.hits.hits[0])?.fields; - const sampleNorm = sample ? (normalizeFields(sample) as unknown as Span) : null; + const sampleNorm = sample ? metadataForDependencyMapping(sample) : null; return { spanType: sampleNorm?.span.type, diff --git a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts new file mode 100644 index 0000000000000..809e6a852a13a --- /dev/null +++ b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const metadataForDependencyMapping = (fields: Partial>) => { + return { + span: { + type: normalizeValue(fields['span.type']), + subtype: normalizeValue(fields['span.subtype']), + }, + }; +}; + +const normalizeValue = (field: unknown[] | unknown): T => { + return (Array.isArray(field) && field.length > 0 ? field[0] : field) as T; +}; From 2e97e813c59c97734b5db9d0751cac4c5bb27df2 Mon Sep 17 00:00:00 2001 From: Milosz Marcinkowski Date: Wed, 18 Sep 2024 15:26:26 +0200 Subject: [PATCH 02/25] 'dependency spans' mapping --- .../dependencies/get_top_dependency_spans.ts | 11 ++--- .../apm/server/utils/es_fields_mappings.ts | 44 +++++++++++++++++++ 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_top_dependency_spans.ts b/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_top_dependency_spans.ts index 66c1eb015bd0a..143478a8a54b9 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_top_dependency_spans.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_top_dependency_spans.ts @@ -8,8 +8,6 @@ import { ProcessorEvent } from '@kbn/observability-plugin/common'; import { kqlQuery, rangeQuery, termQuery, termsQuery } from '@kbn/observability-plugin/server'; import { keyBy } from 'lodash'; -import { Span } from '@kbn/apm-types/es_schemas_ui'; -import type { TransactionRaw } from '@kbn/apm-types/es_schemas_raw'; import { AGENT_NAME, EVENT_OUTCOME, @@ -30,7 +28,10 @@ import { environmentQuery } from '../../../common/utils/environment_query'; import { maybe } from '../../../common/utils/maybe'; import { AgentName } from '../../../typings/es_schemas/ui/fields/agent'; import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; -import { normalizeFields } from '../../utils/normalize_fields'; +import { + topDependencySpansMapping, + transactionsForDependencySpansMapping, +} from '../../utils/es_fields_mappings'; const MAX_NUM_SPANS = 1000; @@ -115,7 +116,7 @@ export async function getTopDependencySpans({ ], }, }) - ).hits.hits.map((hit) => normalizeFields(hit?.fields) as unknown as Span); + ).hits.hits.map((hit) => topDependencySpansMapping(hit?.fields)); const transactionIds = spans.map((span) => span.transaction!.id); @@ -138,7 +139,7 @@ export async function getTopDependencySpans({ }, }, }) - ).hits.hits.map((hit) => normalizeFields(hit.fields) as unknown as TransactionRaw); + ).hits.hits.map((hit) => transactionsForDependencySpansMapping(hit.fields)); const transactionsById = keyBy(transactions, (transaction) => transaction.transaction.id); diff --git a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts index 809e6a852a13a..191f7e0186518 100644 --- a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts +++ b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts @@ -5,6 +5,9 @@ * 2.0. */ +import { AgentName } from '@kbn/elastic-agent-utils'; +import { EventOutcome } from '../../common/event_outcome'; + export const metadataForDependencyMapping = (fields: Partial>) => { return { span: { @@ -14,6 +17,47 @@ export const metadataForDependencyMapping = (fields: Partial> +) => { + return { + transaction: { + id: normalizeValue(fields['transaction.id']), + name: normalizeValue(fields['transaction.name']), + type: normalizeValue(fields['transaction.type']), + }, + }; +}; + +export const topDependencySpansMapping = (fields: Partial>) => { + return { + transaction: { + id: normalizeValue(fields['transaction.id']), + }, + span: { + id: normalizeValue(fields['span.id']), + name: normalizeValue(fields['span.name']), + duration: { + us: normalizeValue(fields['span.duration.us']), + }, + }, + trace: { + id: normalizeValue(fields['trace.id']), + }, + service: { + name: normalizeValue(fields['service.name']), + environment: normalizeValue(fields['service.environment']), + }, + agent: { + name: normalizeValue(fields['agent.name']), + }, + event: { + outcome: normalizeValue(fields['event.outcome']), + }, + '@timestamp': normalizeValue(fields['@timestamp']), + }; +}; + const normalizeValue = (field: unknown[] | unknown): T => { return (Array.isArray(field) && field.length > 0 ? field[0] : field) as T; }; From a6244f58c8fbc72450a319bdf6290df16d5c36b0 Mon Sep 17 00:00:00 2001 From: Milosz Marcinkowski Date: Thu, 19 Sep 2024 11:37:28 +0200 Subject: [PATCH 03/25] `span` mappings --- .../routes/transactions/get_span/index.ts | 5 +- .../apm/server/utils/es_fields_mappings.ts | 54 +++++++++++++++++++ 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_span/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_span/index.ts index 62a3462dde76a..145ffbf14f0ea 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_span/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_span/index.ts @@ -7,13 +7,13 @@ import { rangeQuery, termQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; +import { spanMapping } from '../../../utils/es_fields_mappings'; import { SPAN_ID, TRACE_ID } from '../../../../common/es_fields/apm'; import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client'; import { getTransaction } from '../get_transaction'; import { Span } from '../../../../typings/es_schemas/ui/span'; import { Transaction } from '../../../../typings/es_schemas/ui/transaction'; -import { normalizeFields } from '../../../utils/normalize_fields'; export async function getSpan({ spanId, @@ -62,8 +62,7 @@ export async function getSpan({ : undefined, ]); - const fields = spanResp.hits.hits[0]?.fields; - const fieldsNorm = normalizeFields(fields) as unknown as Span; + const fieldsNorm = spanMapping(spanResp.hits.hits[0]?.fields); return { span: fieldsNorm, parentTransaction }; } diff --git a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts index 191f7e0186518..3c862fe90e964 100644 --- a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts +++ b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts @@ -6,6 +6,7 @@ */ import { AgentName } from '@kbn/elastic-agent-utils'; +import { Span } from '@kbn/apm-types'; import { EventOutcome } from '../../common/event_outcome'; export const metadataForDependencyMapping = (fields: Partial>) => { @@ -58,6 +59,59 @@ export const topDependencySpansMapping = (fields: Partial>): Span => { + return { + parent: { + id: normalizeValue(fields['parent.id']), + }, + observer: { + type: normalizeValue(fields['observer.type']), + version: normalizeValue(fields['observer.version']), + version_major: normalizeValue(fields['observer.version_major']), + }, + agent: { + name: normalizeValue(fields['agent.name']), + version: normalizeValue(fields['agent.version']), + }, + trace: { + id: normalizeValue(fields['trace.id']), + }, + '@timestamp': normalizeValue(fields['@timestamp']), + service: { + name: normalizeValue(fields['service.name']), + environment: normalizeValue(fields['service.environment']), + }, + event: { + outcome: normalizeValue(fields['event.outcome']), + }, + processor: { + name: normalizeValue<'transaction'>(fields['processor.name']), + event: normalizeValue<'span'>(fields['processor.event']), + }, + transaction: { + id: normalizeValue(fields['transaction.id']), + }, + span: { + duration: { + us: normalizeValue(fields['span.duration.us']), + }, + subtype: normalizeValue(fields['span.subtype']), + name: normalizeValue(fields['span.name']), + destination: { + service: { + resource: normalizeValue(fields['span.destination.service.resource']), + }, + }, + id: normalizeValue(fields['span.id']), + type: normalizeValue(fields['span.type']), + }, + timestamp: { + us: normalizeValue(fields['timestamp.us']), + }, + }; +}; + const normalizeValue = (field: unknown[] | unknown): T => { return (Array.isArray(field) && field.length > 0 ? field[0] : field) as T; }; From cb9f1d4b21d5c1798ac95cf59739be3e258f933c Mon Sep 17 00:00:00 2001 From: Milosz Marcinkowski Date: Thu, 19 Sep 2024 17:39:41 +0200 Subject: [PATCH 04/25] `span links` mappings --- .../routes/span_links/get_linked_parents.ts | 7 +-- .../span_links/get_span_links_details.ts | 4 +- .../apm/server/utils/es_fields_mappings.ts | 53 +++++++++++++++++++ 3 files changed, 57 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_linked_parents.ts b/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_linked_parents.ts index 63cad913578a0..81e3aa845fd08 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_linked_parents.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_linked_parents.ts @@ -6,6 +6,7 @@ */ import { rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; +import { linkedParentsOfSpanMapping } from '../../utils/es_fields_mappings'; import { SPAN_ID, SPAN_LINKS, @@ -13,10 +14,7 @@ import { TRANSACTION_ID, PROCESSOR_EVENT, } from '../../../common/es_fields/apm'; -import { SpanRaw } from '../../../typings/es_schemas/raw/span_raw'; -import { TransactionRaw } from '../../../typings/es_schemas/raw/transaction_raw'; import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; -import { normalizeFields } from '../../utils/normalize_fields'; export async function getLinkedParentsOfSpan({ apmEventClient, @@ -58,8 +56,7 @@ export async function getLinkedParentsOfSpan({ }, }); - const fields = response.hits.hits[0]?.fields; - const fieldsNorm = normalizeFields(fields) as unknown as TransactionRaw | SpanRaw | undefined; + const fieldsNorm = linkedParentsOfSpanMapping(response.hits.hits[0]?.fields); return fieldsNorm?.span?.links || []; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_span_links_details.ts b/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_span_links_details.ts index 45efddc7f863f..5e35a12730d5f 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_span_links_details.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_span_links_details.ts @@ -7,6 +7,7 @@ import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; import { chunk, compact, isEmpty, keyBy } from 'lodash'; +import { spanLinksDetailsMapping } from '../../utils/es_fields_mappings'; import { SERVICE_NAME, SPAN_ID, @@ -29,7 +30,6 @@ import { SpanRaw } from '../../../typings/es_schemas/raw/span_raw'; import { TransactionRaw } from '../../../typings/es_schemas/raw/transaction_raw'; import { getBufferedTimerange } from './utils'; import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; -import { normalizeFields } from '../../utils/normalize_fields'; async function fetchSpanLinksDetails({ apmEventClient, @@ -126,7 +126,7 @@ async function fetchSpanLinksDetails({ // The above query might return other spans from the same transaction because siblings spans share the same transaction.id // so, if it is a span we need to guarantee that the span.id is the same as the span links ids if (fields['processor.event']?.[0] === ProcessorEvent.span) { - const span = normalizeFields(fields) as unknown as SpanRaw; + const span = spanLinksDetailsMapping(fields); const hasSpanId = spanIdsMap[span.span.id] || false; return hasSpanId; } diff --git a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts index 3c862fe90e964..ffe03111e0aa6 100644 --- a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts +++ b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts @@ -112,6 +112,59 @@ export const spanMapping = (fields: Partial>): Span => }; }; +export const spanLinksDetailsMapping = (fields: Partial>) => { + return { + trace: { + id: normalizeValue(fields['trace.id']), + }, + span: { + id: normalizeValue(fields['span.id']), + name: normalizeValue(fields['span.name']), + type: normalizeValue(fields['span.type']), + subtype: normalizeValue(fields['span.subtype']), + duration: { + us: normalizeValue(fields['span.duration.us']), + }, + }, + transaction: { + id: normalizeValue(fields['transaction.id']), + name: normalizeValue(fields['transaction.name']), + duration: { + us: normalizeValue(fields['transaction.duration.us']), + }, + }, + service: { + name: normalizeValue(fields['service.name']), + environment: normalizeValue(fields['service.environment']), + }, + processor: { + event: normalizeValue<'span'>(fields['processor.event']), + }, + agent: { + name: normalizeValue(fields['agent.name']), + }, + }; +}; + +export const linkedParentsOfSpanMapping = (fields: Partial>) => { + return { + span: { + links: [ + { + trace: { + // todo(milosz): confirm `span.links.trace.id` format + id: normalizeValue(fields['span.links.trace.id']), + }, + span: { + // todo(milosz): confirm `span.links.span.id` format + id: normalizeValue(fields['span.links.span.id']), + }, + }, + ], + }, + }; +}; + const normalizeValue = (field: unknown[] | unknown): T => { return (Array.isArray(field) && field.length > 0 ? field[0] : field) as T; }; From 0b86fc28e96a4360455536014390655460211238 Mon Sep 17 00:00:00 2001 From: Milosz Marcinkowski Date: Fri, 20 Sep 2024 10:20:28 +0200 Subject: [PATCH 05/25] replace field keys with TS constants --- packages/kbn-apm-types/src/es_fields/apm.ts | 5 + .../apm/server/utils/es_fields_mappings.ts | 130 +++++++++++------- 2 files changed, 84 insertions(+), 51 deletions(-) diff --git a/packages/kbn-apm-types/src/es_fields/apm.ts b/packages/kbn-apm-types/src/es_fields/apm.ts index 6b0a68379f5d4..88c1542f6869d 100644 --- a/packages/kbn-apm-types/src/es_fields/apm.ts +++ b/packages/kbn-apm-types/src/es_fields/apm.ts @@ -8,6 +8,7 @@ */ export const TIMESTAMP = 'timestamp.us'; +export const AT_TIMESTAMP = '@timestamp'; export const AGENT = 'agent'; export const AGENT_NAME = 'agent.name'; export const AGENT_VERSION = 'agent.version'; @@ -50,7 +51,11 @@ export const USER_AGENT_NAME = 'user_agent.name'; export const OBSERVER_HOSTNAME = 'observer.hostname'; export const OBSERVER_LISTENING = 'observer.listening'; +export const OBSERVER_TYPE = 'observer.type'; +export const OBSERVER_VERSION = 'observer.version'; +export const OBSERVER_VERSION_MAJOR = 'observer.version_major'; export const PROCESSOR_EVENT = 'processor.event'; +export const PROCESOR_NAME = 'processor.name'; export const TRANSACTION_DURATION = 'transaction.duration.us'; export const TRANSACTION_DURATION_HISTOGRAM = 'transaction.duration.histogram'; diff --git a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts index ffe03111e0aa6..157212fc01944 100644 --- a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts +++ b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts @@ -7,13 +7,41 @@ import { AgentName } from '@kbn/elastic-agent-utils'; import { Span } from '@kbn/apm-types'; +import { + AGENT_NAME, + AGENT_VERSION, + AT_TIMESTAMP, + EVENT_OUTCOME, + OBSERVER_TYPE, + OBSERVER_VERSION, + OBSERVER_VERSION_MAJOR, + PARENT_ID, + PROCESOR_NAME, + PROCESSOR_EVENT, + SERVICE_ENVIRONMENT, + SERVICE_NAME, + SPAN_DESTINATION_SERVICE_RESOURCE, + SPAN_DURATION, + SPAN_ID, + SPAN_LINKS_SPAN_ID, + SPAN_LINKS_TRACE_ID, + SPAN_NAME, + SPAN_SUBTYPE, + SPAN_TYPE, + TIMESTAMP, + TRACE_ID, + TRANSACTION_DURATION, + TRANSACTION_ID, + TRANSACTION_NAME, + TRANSACTION_TYPE, +} from '@kbn/apm-types/src/es_fields/apm'; import { EventOutcome } from '../../common/event_outcome'; export const metadataForDependencyMapping = (fields: Partial>) => { return { span: { - type: normalizeValue(fields['span.type']), - subtype: normalizeValue(fields['span.subtype']), + type: normalizeValue(fields[SPAN_TYPE]), + subtype: normalizeValue(fields[SPAN_SUBTYPE]), }, }; }; @@ -23,9 +51,9 @@ export const transactionsForDependencySpansMapping = ( ) => { return { transaction: { - id: normalizeValue(fields['transaction.id']), - name: normalizeValue(fields['transaction.name']), - type: normalizeValue(fields['transaction.type']), + id: normalizeValue(fields[TRANSACTION_ID]), + name: normalizeValue(fields[TRANSACTION_NAME]), + type: normalizeValue(fields[TRANSACTION_TYPE]), }, }; }; @@ -33,29 +61,29 @@ export const transactionsForDependencySpansMapping = ( export const topDependencySpansMapping = (fields: Partial>) => { return { transaction: { - id: normalizeValue(fields['transaction.id']), + id: normalizeValue(fields[TRANSACTION_ID]), }, span: { - id: normalizeValue(fields['span.id']), - name: normalizeValue(fields['span.name']), + id: normalizeValue(fields[SPAN_ID]), + name: normalizeValue(fields[SPAN_NAME]), duration: { - us: normalizeValue(fields['span.duration.us']), + us: normalizeValue(fields[SPAN_DURATION]), }, }, trace: { - id: normalizeValue(fields['trace.id']), + id: normalizeValue(fields[TRACE_ID]), }, service: { - name: normalizeValue(fields['service.name']), - environment: normalizeValue(fields['service.environment']), + name: normalizeValue(fields[SERVICE_NAME]), + environment: normalizeValue(fields[SERVICE_ENVIRONMENT]), }, agent: { - name: normalizeValue(fields['agent.name']), + name: normalizeValue(fields[AGENT_NAME]), }, event: { - outcome: normalizeValue(fields['event.outcome']), + outcome: normalizeValue(fields[EVENT_OUTCOME]), }, - '@timestamp': normalizeValue(fields['@timestamp']), + '@timestamp': normalizeValue(fields[AT_TIMESTAMP]), }; }; @@ -63,51 +91,51 @@ export const topDependencySpansMapping = (fields: Partial>): Span => { return { parent: { - id: normalizeValue(fields['parent.id']), + id: normalizeValue(fields[PARENT_ID]), }, observer: { - type: normalizeValue(fields['observer.type']), - version: normalizeValue(fields['observer.version']), - version_major: normalizeValue(fields['observer.version_major']), + type: normalizeValue(fields[OBSERVER_TYPE]), + version: normalizeValue(fields[OBSERVER_VERSION]), + version_major: normalizeValue(fields[OBSERVER_VERSION_MAJOR]), }, agent: { - name: normalizeValue(fields['agent.name']), - version: normalizeValue(fields['agent.version']), + name: normalizeValue(fields[AGENT_NAME]), + version: normalizeValue(fields[AGENT_VERSION]), }, trace: { - id: normalizeValue(fields['trace.id']), + id: normalizeValue(fields[TRACE_ID]), }, - '@timestamp': normalizeValue(fields['@timestamp']), + '@timestamp': normalizeValue(fields[AT_TIMESTAMP]), service: { - name: normalizeValue(fields['service.name']), - environment: normalizeValue(fields['service.environment']), + name: normalizeValue(fields[SERVICE_NAME]), + environment: normalizeValue(fields[SERVICE_ENVIRONMENT]), }, event: { - outcome: normalizeValue(fields['event.outcome']), + outcome: normalizeValue(fields[EVENT_OUTCOME]), }, processor: { - name: normalizeValue<'transaction'>(fields['processor.name']), - event: normalizeValue<'span'>(fields['processor.event']), + name: normalizeValue<'transaction'>(fields[PROCESOR_NAME]), + event: normalizeValue<'span'>(fields[PROCESSOR_EVENT]), }, transaction: { - id: normalizeValue(fields['transaction.id']), + id: normalizeValue(fields[TRANSACTION_ID]), }, span: { duration: { - us: normalizeValue(fields['span.duration.us']), + us: normalizeValue(fields[SPAN_DURATION]), }, - subtype: normalizeValue(fields['span.subtype']), - name: normalizeValue(fields['span.name']), + subtype: normalizeValue(fields[SPAN_SUBTYPE]), + name: normalizeValue(fields[SPAN_NAME]), destination: { service: { - resource: normalizeValue(fields['span.destination.service.resource']), + resource: normalizeValue(fields[SPAN_DESTINATION_SERVICE_RESOURCE]), }, }, - id: normalizeValue(fields['span.id']), - type: normalizeValue(fields['span.type']), + id: normalizeValue(fields[SPAN_ID]), + type: normalizeValue(fields[SPAN_TYPE]), }, timestamp: { - us: normalizeValue(fields['timestamp.us']), + us: normalizeValue(fields[TIMESTAMP]), }, }; }; @@ -115,33 +143,33 @@ export const spanMapping = (fields: Partial>): Span => export const spanLinksDetailsMapping = (fields: Partial>) => { return { trace: { - id: normalizeValue(fields['trace.id']), + id: normalizeValue(fields[TRACE_ID]), }, span: { - id: normalizeValue(fields['span.id']), - name: normalizeValue(fields['span.name']), - type: normalizeValue(fields['span.type']), - subtype: normalizeValue(fields['span.subtype']), + id: normalizeValue(fields[SPAN_ID]), + name: normalizeValue(fields[SPAN_NAME]), + type: normalizeValue(fields[SPAN_TYPE]), + subtype: normalizeValue(fields[SPAN_SUBTYPE]), duration: { - us: normalizeValue(fields['span.duration.us']), + us: normalizeValue(fields[SPAN_DURATION]), }, }, transaction: { - id: normalizeValue(fields['transaction.id']), - name: normalizeValue(fields['transaction.name']), + id: normalizeValue(fields[TRANSACTION_ID]), + name: normalizeValue(fields[TRANSACTION_NAME]), duration: { - us: normalizeValue(fields['transaction.duration.us']), + us: normalizeValue(fields[TRANSACTION_DURATION]), }, }, service: { - name: normalizeValue(fields['service.name']), - environment: normalizeValue(fields['service.environment']), + name: normalizeValue(fields[SERVICE_NAME]), + environment: normalizeValue(fields[SERVICE_ENVIRONMENT]), }, processor: { - event: normalizeValue<'span'>(fields['processor.event']), + event: normalizeValue<'span'>(fields[PROCESSOR_EVENT]), }, agent: { - name: normalizeValue(fields['agent.name']), + name: normalizeValue(fields[AGENT_NAME]), }, }; }; @@ -153,11 +181,11 @@ export const linkedParentsOfSpanMapping = (fields: Partial(fields['span.links.trace.id']), + id: normalizeValue(fields[SPAN_LINKS_TRACE_ID]), }, span: { // todo(milosz): confirm `span.links.span.id` format - id: normalizeValue(fields['span.links.span.id']), + id: normalizeValue(fields[SPAN_LINKS_SPAN_ID]), }, }, ], From f54e2b34d34f4e3c4c92881d256cc04bd3ed3999 Mon Sep 17 00:00:00 2001 From: Milosz Marcinkowski Date: Fri, 20 Sep 2024 13:52:43 +0200 Subject: [PATCH 06/25] 'transaction' mapping --- packages/kbn-apm-types/src/es_fields/apm.ts | 5 ++ .../transactions/get_transaction/index.ts | 8 +- .../get_transaction_by_name/index.ts | 8 +- .../get_transaction_by_trace/index.ts | 7 +- .../apm/server/utils/es_fields_mappings.ts | 73 +++++++++++++++++++ 5 files changed, 84 insertions(+), 17 deletions(-) diff --git a/packages/kbn-apm-types/src/es_fields/apm.ts b/packages/kbn-apm-types/src/es_fields/apm.ts index 88c1542f6869d..f00d1eeffe3c8 100644 --- a/packages/kbn-apm-types/src/es_fields/apm.ts +++ b/packages/kbn-apm-types/src/es_fields/apm.ts @@ -72,6 +72,8 @@ export const TRANSACTION_OVERFLOW_COUNT = 'transaction.aggregation.overflow_coun // for transaction metrics export const TRANSACTION_ROOT = 'transaction.root'; export const TRANSACTION_PROFILER_STACK_TRACE_IDS = 'transaction.profiler_stack_trace_ids'; +export const TRANSACTION_NAME_TEXT = 'transaction.name.text'; +export const TRANSACTION_REPRESENTATIVE_COUNT = 'transaction.representative_count'; export const EVENT_OUTCOME = 'event.outcome'; @@ -143,6 +145,7 @@ export const LABEL_GC = 'labels.gc'; export const LABEL_TYPE = 'labels.type'; export const LABEL_TELEMETRY_AUTO_VERSION = 'labels.telemetry_auto_version'; export const LABEL_LIFECYCLE_STATE = 'labels.lifecycle_state'; +export const LABEL_SOME_RESOURCE_ATTRIBUTE = 'labels.some_resource_attribute'; export const HOST = 'host'; export const HOST_HOSTNAME = 'host.hostname'; // Do not use. Please use `HOST_NAME` instead. @@ -185,6 +188,8 @@ export const VALUE_OTEL_JVM_PROCESS_MEMORY_NON_HEAP = 'non_heap'; export const TIER = '_tier'; export const INDEX = '_index'; export const DATA_STEAM_TYPE = 'data_stream.type'; +export const DATA_STREAM_NAMESPACE = 'data_stream.namespace'; +export const DATA_STREAM_DATASET = 'data_stream.dataset'; // Mobile export const NETWORK_CONNECTION_TYPE = 'network.connection.type'; diff --git a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts index b807058a4aa43..e5f08762f6d6b 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts @@ -6,8 +6,7 @@ */ import { rangeQuery, termQuery } from '@kbn/observability-plugin/server'; -import type { Transaction } from '@kbn/apm-types/es_schemas_ui'; -import { normalizeFields } from '../../../utils/normalize_fields'; +import { transactionMapping } from '../../../utils/es_fields_mappings'; import { TRACE_ID, TRANSACTION_ID } from '../../../../common/es_fields/apm'; import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client'; @@ -53,8 +52,5 @@ export async function getTransaction({ }, }); - const fields = resp.hits.hits[0]?.fields; - const fieldsNorm = normalizeFields(fields) as unknown as Transaction; - - return fieldsNorm; + return transactionMapping(resp.hits.hits[0]?.fields); } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_name/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_name/index.ts index f90cad704c049..2de0b2aae04d2 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_name/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_name/index.ts @@ -6,13 +6,12 @@ */ import { rangeQuery } from '@kbn/observability-plugin/server'; -import type { Transaction } from '../../../../typings/es_schemas/ui/transaction'; -import { normalizeFields } from '../../../utils/normalize_fields'; import { ApmDocumentType } from '../../../../common/document_type'; import { SERVICE_NAME, TRANSACTION_NAME } from '../../../../common/es_fields/apm'; import { RollupInterval } from '../../../../common/rollup'; import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client'; +import { transactionMapping } from '../../../utils/es_fields_mappings'; export async function getTransactionByName({ transactionName, @@ -53,8 +52,5 @@ export async function getTransactionByName({ }, }); - const fields = resp.hits.hits[0]?.fields; - const fieldsNorm = normalizeFields(fields) as unknown as Transaction; - - return fieldsNorm; + return transactionMapping(resp.hits.hits[0]?.fields); } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_trace/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_trace/index.ts index 68891d3d908e3..b9aca516a5470 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_trace/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_trace/index.ts @@ -7,10 +7,9 @@ import { ProcessorEvent } from '@kbn/observability-plugin/common'; import { rangeQuery } from '@kbn/observability-plugin/server'; -import type { Transaction } from '@kbn/apm-types/es_schemas_ui'; -import { normalizeFields } from '../../../utils/normalize_fields'; import { TRACE_ID, PARENT_ID } from '../../../../common/es_fields/apm'; import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client'; +import { transactionMapping } from '../../../utils/es_fields_mappings'; export async function getRootTransactionByTraceId({ traceId, @@ -52,8 +51,6 @@ export async function getRootTransactionByTraceId({ }; const resp = await apmEventClient.search('get_root_transaction_by_trace_id', params); - const fields = resp.hits.hits[0]?.fields; - const transaction = normalizeFields(fields) as unknown as Transaction; - return { transaction }; + return transactionMapping(resp.hits.hits[0]?.fields); } diff --git a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts index 157212fc01944..0688ab07295d0 100644 --- a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts +++ b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts @@ -11,7 +11,13 @@ import { AGENT_NAME, AGENT_VERSION, AT_TIMESTAMP, + DATA_STEAM_TYPE, + DATA_STREAM_DATASET, + DATA_STREAM_NAMESPACE, EVENT_OUTCOME, + EVENT_SUCCESS_COUNT, + LABEL_SOME_RESOURCE_ATTRIBUTE, + OBSERVER_HOSTNAME, OBSERVER_TYPE, OBSERVER_VERSION, OBSERVER_VERSION_MAJOR, @@ -19,7 +25,9 @@ import { PROCESOR_NAME, PROCESSOR_EVENT, SERVICE_ENVIRONMENT, + SERVICE_FRAMEWORK_NAME, SERVICE_NAME, + SERVICE_NODE_NAME, SPAN_DESTINATION_SERVICE_RESOURCE, SPAN_DURATION, SPAN_ID, @@ -33,6 +41,9 @@ import { TRANSACTION_DURATION, TRANSACTION_ID, TRANSACTION_NAME, + TRANSACTION_REPRESENTATIVE_COUNT, + TRANSACTION_RESULT, + TRANSACTION_SAMPLED, TRANSACTION_TYPE, } from '@kbn/apm-types/src/es_fields/apm'; import { EventOutcome } from '../../common/event_outcome'; @@ -193,6 +204,68 @@ export const linkedParentsOfSpanMapping = (fields: Partial>) => { + return { + transaction: { + representative_count: normalizeValue(fields[TRANSACTION_REPRESENTATIVE_COUNT]), + result: normalizeValue(fields[TRANSACTION_RESULT]), + sampled: normalizeValue(fields[TRANSACTION_SAMPLED]), + id: normalizeValue(fields[TRANSACTION_ID]), + duration: { + us: normalizeValue(fields[TRANSACTION_DURATION]), + }, + type: normalizeValue(fields[TRANSACTION_TYPE]), + name: normalizeValue(fields[TRANSACTION_NAME]), + }, + service: { + node: { + name: normalizeValue(fields[SERVICE_NODE_NAME]), + }, + language: { + name: normalizeValue(fields[SERVICE_NODE_NAME]), + }, + name: normalizeValue(fields[SERVICE_NAME]), + framework: { + name: normalizeValue(fields[SERVICE_FRAMEWORK_NAME]), + }, + }, + trace: { + id: normalizeValue(fields[TRACE_ID]), + }, + agent: { + name: normalizeValue(fields[AGENT_NAME]), + version: normalizeValue(fields[AGENT_VERSION]), + }, + event: { + success_count: normalizeValue(fields[EVENT_SUCCESS_COUNT]), + outcome: normalizeValue(fields[EVENT_OUTCOME]), + }, + processor: { + event: normalizeValue<'transaction'>(fields[PROCESSOR_EVENT]), + }, + data_stream: { + namespace: normalizeValue(fields[DATA_STREAM_NAMESPACE]), + type: normalizeValue(fields[DATA_STEAM_TYPE]), + dataset: normalizeValue(fields[DATA_STREAM_DATASET]), + }, + span: { + id: normalizeValue(fields[SPAN_ID]), + }, + observer: { + hostname: normalizeValue(fields[OBSERVER_HOSTNAME]), + type: normalizeValue(fields[OBSERVER_TYPE]), + version: normalizeValue(fields[OBSERVER_VERSION]), + }, + timestamp: { + us: normalizeValue(fields[TIMESTAMP]), + }, + '@timestamp': normalizeValue(fields[AT_TIMESTAMP]), + labels: { + some_resource_attribute: normalizeValue(fields[LABEL_SOME_RESOURCE_ATTRIBUTE]), + }, + }; +}; + const normalizeValue = (field: unknown[] | unknown): T => { return (Array.isArray(field) && field.length > 0 ? field[0] : field) as T; }; From 37dd831e4183d074dd41c87c02ac898f438a020a Mon Sep 17 00:00:00 2001 From: Milosz Marcinkowski Date: Fri, 20 Sep 2024 14:24:17 +0200 Subject: [PATCH 07/25] 'trace items' mappings --- .../server/routes/traces/get_trace_items.ts | 10 +- .../apm/server/utils/es_fields_mappings.ts | 112 ++++++++++++++++++ 2 files changed, 115 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts b/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts index da4ddfde693fb..7ee2c06eb4433 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts @@ -11,6 +11,7 @@ import { QueryDslQueryContainer, Sort } from '@elastic/elasticsearch/lib/api/typ import { ProcessorEvent } from '@kbn/observability-plugin/common'; import { rangeQuery } from '@kbn/observability-plugin/server'; import { last } from 'lodash'; +import { errorDocsMapping, traceDocMapping } from '../../utils/es_fields_mappings'; import { APMConfig } from '../..'; import { AGENT_NAME, @@ -54,7 +55,6 @@ import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_ev import { getSpanLinksCountById } from '../span_links/get_linked_children'; import { ApmDocumentType } from '../../../common/document_type'; import { RollupInterval } from '../../../common/rollup'; -import { normalizeFields } from '../../utils/normalize_fields'; export interface TraceItems { exceedsMax: boolean; @@ -135,12 +135,8 @@ export async function getTraceItems({ const traceDocsTotal = traceResponse.total; const exceedsMax = traceDocsTotal > maxTraceItems; - const traceDocs = traceResponse.hits.map( - (hit) => normalizeFields(hit.fields) as unknown as WaterfallTransaction | WaterfallSpan - ); - const errorDocs = errorResponse.hits.hits.map( - (hit) => normalizeFields(hit.fields) as unknown as WaterfallError - ); + const traceDocs = traceResponse.hits.map((hit) => traceDocMapping(hit.fields)); + const errorDocs = errorResponse.hits.hits.map((hit) => errorDocsMapping(hit.fields)); return { exceedsMax, diff --git a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts index 0688ab07295d0..03384fa4e963d 100644 --- a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts +++ b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts @@ -11,11 +11,13 @@ import { AGENT_NAME, AGENT_VERSION, AT_TIMESTAMP, + CHILD_ID, DATA_STEAM_TYPE, DATA_STREAM_DATASET, DATA_STREAM_NAMESPACE, EVENT_OUTCOME, EVENT_SUCCESS_COUNT, + FAAS_COLDSTART, LABEL_SOME_RESOURCE_ATTRIBUTE, OBSERVER_HOSTNAME, OBSERVER_TYPE, @@ -28,6 +30,9 @@ import { SERVICE_FRAMEWORK_NAME, SERVICE_NAME, SERVICE_NODE_NAME, + SPAN_COMPOSITE_COMPRESSION_STRATEGY, + SPAN_COMPOSITE_COUNT, + SPAN_COMPOSITE_SUM, SPAN_DESTINATION_SERVICE_RESOURCE, SPAN_DURATION, SPAN_ID, @@ -35,6 +40,7 @@ import { SPAN_LINKS_TRACE_ID, SPAN_NAME, SPAN_SUBTYPE, + SPAN_SYNC, SPAN_TYPE, TIMESTAMP, TRACE_ID, @@ -46,7 +52,22 @@ import { TRANSACTION_SAMPLED, TRANSACTION_TYPE, } from '@kbn/apm-types/src/es_fields/apm'; +import { SpanLink } from '@kbn/apm-types/src/es_schemas/raw/fields/span_links'; +import { + ERROR_EXCEPTION, + ERROR_GROUP_ID, + ERROR_ID, + ERROR_LOG_MESSAGE, +} from '../../common/es_fields/apm'; +import { + WaterfallError, + WaterfallSpan, + WaterfallTransaction, +} from '../../common/waterfall/typings'; +import { SPAN_LINKS } from '../../common/es_fields/apm'; +import { SPAN_ACTION } from '../../common/es_fields/apm'; import { EventOutcome } from '../../common/event_outcome'; +import { Exception } from '../../typings/es_schemas/raw/error_raw'; export const metadataForDependencyMapping = (fields: Partial>) => { return { @@ -266,6 +287,97 @@ export const transactionMapping = (fields: Partial>) = }; }; +export const traceDocMapping = ( + fields: Partial> +): WaterfallTransaction | WaterfallSpan => { + return { + timestamp: { + us: normalizeValue(fields[TIMESTAMP]), + }, + trace: { + id: normalizeValue(fields[TRACE_ID]), + }, + parent: { + id: normalizeValue(fields[PARENT_ID]), + }, + service: { + name: normalizeValue(fields[SERVICE_NAME]), + environment: normalizeValue(fields[SERVICE_ENVIRONMENT]), + }, + agent: { + name: normalizeValue(fields[AGENT_NAME]), + }, + event: { + outcome: normalizeValue(fields[EVENT_OUTCOME]), + }, + processor: { + event: normalizeValue<'span'>(fields[PROCESSOR_EVENT]), + }, + transaction: { + result: normalizeValue(fields[TRANSACTION_RESULT]), + id: normalizeValue(fields[TRANSACTION_ID]), + duration: { + us: normalizeValue(fields[TRANSACTION_DURATION]), + }, + type: normalizeValue(fields[TRANSACTION_TYPE]), + name: normalizeValue(fields[TRANSACTION_NAME]), + }, + faas: { + coldstart: normalizeValue(fields[FAAS_COLDSTART]), + }, + span: { + id: normalizeValue(fields[SPAN_ID]), + name: normalizeValue(fields[SPAN_NAME]), + type: normalizeValue(fields[SPAN_TYPE]), + subtype: normalizeValue(fields[SPAN_SUBTYPE]), + duration: { + us: normalizeValue(fields[SPAN_DURATION]), + }, + action: normalizeValue(fields[SPAN_ACTION]), + links: normalizeValue(fields[SPAN_LINKS]), + composite: { + count: normalizeValue(fields[SPAN_COMPOSITE_COUNT]), + sum: { + us: normalizeValue(fields[SPAN_COMPOSITE_SUM]), + }, + compression_strategy: normalizeValue(fields[SPAN_COMPOSITE_COMPRESSION_STRATEGY]), + }, + sync: normalizeValue(fields[SPAN_SYNC]), + }, + child: { + id: normalizeValue(fields[CHILD_ID]), + }, + }; +}; + +export const errorDocsMapping = (fields: Partial>): WaterfallError => { + return { + timestamp: { + us: normalizeValue(fields[TIMESTAMP]), + }, + trace: { + id: normalizeValue(fields[TRACE_ID]), + }, + transaction: { + id: normalizeValue(fields[TRANSACTION_ID]), + }, + parent: { + id: normalizeValue(fields[PARENT_ID]), + }, + service: { + name: normalizeValue(fields[SERVICE_NAME]), + }, + error: { + id: normalizeValue(fields[ERROR_ID]), + log: { + message: normalizeValue(fields[ERROR_LOG_MESSAGE]), + }, + exception: normalizeValue(fields[ERROR_EXCEPTION]), + grouping_key: normalizeValue(fields[ERROR_GROUP_ID]), + }, + }; +}; + const normalizeValue = (field: unknown[] | unknown): T => { return (Array.isArray(field) && field.length > 0 ? field[0] : field) as T; }; From 8e55406108643c8779cd018d635e107a1e33ad22 Mon Sep 17 00:00:00 2001 From: Milosz Marcinkowski Date: Fri, 20 Sep 2024 15:36:22 +0200 Subject: [PATCH 08/25] 'error group main statistics' mapping --- .../get_error_group_main_statistics.ts | 6 +-- .../apm/server/utils/es_fields_mappings.ts | 38 ++++++++++++++++--- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_main_statistics.ts b/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_main_statistics.ts index e0b9428295214..f691ffeb2d39b 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_main_statistics.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_main_statistics.ts @@ -7,8 +7,6 @@ import { AggregationsAggregateOrder } from '@elastic/elasticsearch/lib/api/types'; import { kqlQuery, rangeQuery, termQuery, wildcardQuery } from '@kbn/observability-plugin/server'; -import type { ErrorRaw } from '@kbn/apm-types/es_schemas_raw'; -import { normalizeFields } from '../../../utils/normalize_fields'; import { ERROR_CULPRIT, ERROR_EXC_HANDLED, @@ -27,6 +25,7 @@ import { getErrorName } from '../../../lib/helpers/get_error_name'; import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client'; import { ApmDocumentType } from '../../../../common/document_type'; import { RollupInterval } from '../../../../common/rollup'; +import { errorGroupMainStatisticsMapping } from '../../../utils/es_fields_mappings'; export interface ErrorGroupMainStatisticsResponse { errorGroups: Array<{ @@ -159,8 +158,7 @@ export async function getErrorGroupMainStatistics({ const errorGroups = response.aggregations?.error_groups.buckets.map((bucket) => { - const fields = bucket.sample.hits.hits[0]?.fields; - const fieldsNorm = normalizeFields(fields) as unknown as ErrorRaw; + const fieldsNorm = errorGroupMainStatisticsMapping(bucket.sample.hits.hits[0]?.fields); return { groupId: bucket.key as string, name: getErrorName(fieldsNorm), diff --git a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts index 03384fa4e963d..0731f292acb6f 100644 --- a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts +++ b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts @@ -15,6 +15,9 @@ import { DATA_STEAM_TYPE, DATA_STREAM_DATASET, DATA_STREAM_NAMESPACE, + ERROR_CULPRIT, + ERROR_EXC_HANDLED, + ERROR_EXC_TYPE, EVENT_OUTCOME, EVENT_SUCCESS_COUNT, FAAS_COLDSTART, @@ -30,6 +33,7 @@ import { SERVICE_FRAMEWORK_NAME, SERVICE_NAME, SERVICE_NODE_NAME, + SPAN_ACTION, SPAN_COMPOSITE_COMPRESSION_STRATEGY, SPAN_COMPOSITE_COUNT, SPAN_COMPOSITE_SUM, @@ -42,6 +46,7 @@ import { SPAN_SUBTYPE, SPAN_SYNC, SPAN_TYPE, + SPAN_LINKS, TIMESTAMP, TRACE_ID, TRANSACTION_DURATION, @@ -51,21 +56,18 @@ import { TRANSACTION_RESULT, TRANSACTION_SAMPLED, TRANSACTION_TYPE, -} from '@kbn/apm-types/src/es_fields/apm'; -import { SpanLink } from '@kbn/apm-types/src/es_schemas/raw/fields/span_links'; -import { + ERROR_EXC_MESSAGE, ERROR_EXCEPTION, ERROR_GROUP_ID, ERROR_ID, ERROR_LOG_MESSAGE, -} from '../../common/es_fields/apm'; +} from '@kbn/apm-types/src/es_fields/apm'; +import { SpanLink } from '@kbn/apm-types/src/es_schemas/raw/fields/span_links'; import { WaterfallError, WaterfallSpan, WaterfallTransaction, } from '../../common/waterfall/typings'; -import { SPAN_LINKS } from '../../common/es_fields/apm'; -import { SPAN_ACTION } from '../../common/es_fields/apm'; import { EventOutcome } from '../../common/event_outcome'; import { Exception } from '../../typings/es_schemas/raw/error_raw'; @@ -378,6 +380,30 @@ export const errorDocsMapping = (fields: Partial>): Wa }; }; +export const errorGroupMainStatisticsMapping = (fields: Partial>) => { + return { + trace: { + id: normalizeValue(fields[TRACE_ID]), + }, + error: { + id: normalizeValue(fields[ERROR_ID]), + log: { + message: normalizeValue(fields[ERROR_LOG_MESSAGE]), + }, + exception: [ + { + message: normalizeValue(fields[ERROR_EXC_MESSAGE]), + handled: normalizeValue(fields[ERROR_EXC_HANDLED]), + type: normalizeValue(fields[ERROR_EXC_TYPE]), + }, + ], + culprit: normalizeValue(fields[ERROR_CULPRIT]), + grouping_key: normalizeValue(fields[ERROR_GROUP_ID]), + }, + '@timestamp': normalizeValue(fields[AT_TIMESTAMP]), + }; +}; + const normalizeValue = (field: unknown[] | unknown): T => { return (Array.isArray(field) && field.length > 0 ? field[0] : field) as T; }; From 0cabafb879ad64fefb8147fa4bcc9a853206f88d Mon Sep 17 00:00:00 2001 From: Milosz Marcinkowski Date: Fri, 20 Sep 2024 16:27:36 +0200 Subject: [PATCH 09/25] 'error sample details' mapping --- .../get_error_sample_details.ts | 5 +- .../apm/server/utils/es_fields_mappings.ts | 58 ++++++++++++++++++- 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts b/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts index f7e4abf4eac74..d1e3a48349986 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts @@ -6,7 +6,6 @@ */ import { rangeQuery, kqlQuery } from '@kbn/observability-plugin/server'; -import { normalizeFields } from '../../../utils/normalize_fields'; import { ERROR_ID, SERVICE_NAME } from '../../../../common/es_fields/apm'; import { environmentQuery } from '../../../../common/utils/environment_query'; import { ApmDocumentType } from '../../../../common/document_type'; @@ -15,6 +14,7 @@ import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm import { getTransaction } from '../../transactions/get_transaction'; import { Transaction } from '../../../../typings/es_schemas/ui/transaction'; import { APMError } from '../../../../typings/es_schemas/ui/apm_error'; +import { errorSampleDetails } from '../../../utils/es_fields_mappings'; export interface ErrorSampleDetailsResponse { transaction: Transaction | undefined; @@ -67,8 +67,7 @@ export async function getErrorSampleDetails({ const resp = await apmEventClient.search('get_error_sample_details', params); - // const error = resp.hits.hits[0]?.fields; - const errorNorm = normalizeFields(resp.hits.hits[0]?.fields) as unknown as APMError; + const errorNorm = errorSampleDetails(resp.hits.hits[0]?.fields); const transactionId = errorNorm?.transaction?.id; const traceId = errorNorm?.trace?.id; diff --git a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts index 0731f292acb6f..b2be009c064aa 100644 --- a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts +++ b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts @@ -6,8 +6,10 @@ */ import { AgentName } from '@kbn/elastic-agent-utils'; -import { Span } from '@kbn/apm-types'; import { + APMError, + Span, + SpanLink, AGENT_NAME, AGENT_VERSION, AT_TIMESTAMP, @@ -61,8 +63,8 @@ import { ERROR_GROUP_ID, ERROR_ID, ERROR_LOG_MESSAGE, -} from '@kbn/apm-types/src/es_fields/apm'; -import { SpanLink } from '@kbn/apm-types/src/es_schemas/raw/fields/span_links'; + HOST_NAME, +} from '@kbn/apm-types'; import { WaterfallError, WaterfallSpan, @@ -404,6 +406,56 @@ export const errorGroupMainStatisticsMapping = (fields: Partial>): APMError => { + return { + observer: { + type: normalizeValue(fields[OBSERVER_TYPE]), + version: normalizeValue(fields[OBSERVER_VERSION]), + version_major: normalizeValue(fields[OBSERVER_VERSION_MAJOR]), + }, + agent: { + name: normalizeValue(fields[AGENT_NAME]), + version: '', + }, + trace: { + id: normalizeValue(fields[TRACE_ID]), + }, + '@timestamp': normalizeValue(fields[AT_TIMESTAMP]), + service: { + node: { + name: normalizeValue(fields[SERVICE_NODE_NAME]), + }, + name: normalizeValue(fields[SERVICE_NAME]), + environment: normalizeValue(fields[SERVICE_ENVIRONMENT]), + }, + host: { + name: normalizeValue(fields[HOST_NAME]), + }, + error: { + id: normalizeValue(fields[ERROR_ID]), + exception: [ + { + message: normalizeValue(fields[ERROR_EXC_MESSAGE]), + type: normalizeValue(fields[ERROR_EXC_TYPE]), + }, + ], + grouping_key: normalizeValue(fields[ERROR_GROUP_ID]), + }, + processor: { + name: normalizeValue<'error'>(fields[PROCESOR_NAME]), + event: normalizeValue<'error'>(fields[PROCESSOR_EVENT]), + }, + transaction: { + id: normalizeValue(fields[TRANSACTION_ID]), + type: normalizeValue(fields[TRANSACTION_TYPE]), + sampled: normalizeValue(fields[TRANSACTION_SAMPLED]), + }, + timestamp: { + us: normalizeValue(fields[TIMESTAMP]), + }, + }; +}; + const normalizeValue = (field: unknown[] | unknown): T => { return (Array.isArray(field) && field.length > 0 ? field[0] : field) as T; }; From f5e627c4073deea7df5b4072a459aa489ddd9669 Mon Sep 17 00:00:00 2001 From: Milosz Marcinkowski Date: Fri, 20 Sep 2024 17:18:00 +0200 Subject: [PATCH 10/25] 'service metadata' mapping --- .../services/get_service_metadata_icons.ts | 9 +++---- .../apm/server/utils/es_fields_mappings.ts | 27 +++++++++++++++++++ 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts index 3e44a313ffdfd..0f83e4de974b5 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts @@ -7,6 +7,7 @@ import { rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; +import { serviceMetadataIcons } from '../../utils/es_fields_mappings'; import { AGENT_NAME, CLOUD_PROVIDER, @@ -21,13 +22,9 @@ import { SERVICE_FRAMEWORK_NAME, } from '../../../common/es_fields/apm'; import { ContainerType } from '../../../common/service_metadata'; -import { TransactionRaw } from '../../../typings/es_schemas/raw/transaction_raw'; import { getProcessorEventForTransactions } from '../../lib/helpers/transactions'; import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; import { ServerlessType, getServerlessTypeFromCloudData } from '../../../common/serverless'; -import { normalizeFields } from '../../utils/normalize_fields'; - -type ServiceMetadataIconsRaw = Pick; export interface ServiceMetadataIcons { agentName?: string; @@ -89,9 +86,9 @@ export async function getServiceMetadataIcons({ }; } - const { kubernetes, cloud, container, agent } = normalizeFields( + const { kubernetes, cloud, container, agent } = serviceMetadataIcons( response.hits.hits[0].fields - ) as ServiceMetadataIconsRaw; + ); let containerType: ContainerType; if (!!kubernetes) { diff --git a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts index b2be009c064aa..8748067130cfa 100644 --- a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts +++ b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts @@ -64,7 +64,13 @@ import { ERROR_ID, ERROR_LOG_MESSAGE, HOST_NAME, + Container, + Kubernetes, + CLOUD_PROVIDER, + CONTAINER, + KUBERNETES, } from '@kbn/apm-types'; +import { TransactionRaw } from '../../typings/es_schemas/raw/transaction_raw'; import { WaterfallError, WaterfallSpan, @@ -73,6 +79,8 @@ import { import { EventOutcome } from '../../common/event_outcome'; import { Exception } from '../../typings/es_schemas/raw/error_raw'; +type ServiceMetadataIconsRaw = Pick; + export const metadataForDependencyMapping = (fields: Partial>) => { return { span: { @@ -456,6 +464,25 @@ export const errorSampleDetails = (fields: Partial>): }; }; +export const serviceMetadataIcons = ( + fields: Partial> +): ServiceMetadataIconsRaw => { + return { + agent: { + name: normalizeValue(fields[AGENT_NAME]), + version: '', + }, + cloud: { + provider: normalizeValue(fields[CLOUD_PROVIDER]), + service: { + name: normalizeValue(fields[CLOUD_PROVIDER]), + }, + }, + container: normalizeValue(fields[CONTAINER]), + kubernetes: normalizeValue(fields[KUBERNETES]), + }; +}; + const normalizeValue = (field: unknown[] | unknown): T => { return (Array.isArray(field) && field.length > 0 ? field[0] : field) as T; }; From c52ddfdff9b1c2ecc4c33fe128cbd4e110db766a Mon Sep 17 00:00:00 2001 From: Milosz Marcinkowski Date: Mon, 23 Sep 2024 14:55:05 +0200 Subject: [PATCH 11/25] 'metadata and service version' mappings --- packages/kbn-apm-types/src/es_fields/apm.ts | 54 ++ .../collect_data_telemetry/tasks.ts | 10 +- .../get_service_instance_metadata_details.ts | 17 +- .../apm/server/utils/es_fields_mappings.ts | 463 +++++++++++++++++- 4 files changed, 524 insertions(+), 20 deletions(-) diff --git a/packages/kbn-apm-types/src/es_fields/apm.ts b/packages/kbn-apm-types/src/es_fields/apm.ts index f00d1eeffe3c8..58320ff96ba9f 100644 --- a/packages/kbn-apm-types/src/es_fields/apm.ts +++ b/packages/kbn-apm-types/src/es_fields/apm.ts @@ -27,6 +27,8 @@ export const CLOUD_INSTANCE_NAME = 'cloud.instance.name'; export const CLOUD_SERVICE_NAME = 'cloud.service.name'; export const EVENT_SUCCESS_COUNT = 'event.success_count'; +export const EVENT_SUCCESS_COUNT_SUM = 'event.success_count.sum'; +export const EVENT_SUCCESS_COUNT_VALUE_COUNT = 'event.success_count.value_count'; export const SERVICE = 'service'; export const SERVICE_NAME = 'service.name'; @@ -43,11 +45,31 @@ export const SERVICE_TARGET_TYPE = 'service.target.type'; export const SERVICE_OVERFLOW_COUNT = 'service_transaction.aggregation.overflow_count'; export const URL_FULL = 'url.full'; +export const URL_PATH = 'url.path'; +export const URL_ORIGINAL = 'url.original'; +export const URL_SCHEME = 'url.scheme'; +export const URL_PORT = 'url.port'; +export const URL_DOMAIN = 'url.domain'; export const HTTP_REQUEST_METHOD = 'http.request.method'; export const HTTP_RESPONSE_STATUS_CODE = 'http.response.status_code'; +export const HTTP_RESPONSE_HEADERS_TRANSFER_ENCODING = 'http.response.headers.Transfer-Encoding'; +export const HTTP_RESPONSE_HEADERS_CONNECTION = 'http.response.headers.Connection'; +export const HTTP_RESPONSE_HEADERS_DATE = 'http.response.headers.Date'; +export const HTTP_RESPONSE_HEADERS_X_POWERED_BY = 'http.response.headers.X-Powered-By'; +export const HTTP_VERSION = 'http.version'; +export const HTTP_REQUEST_HEADERS_ACCEPT = 'http.request.headers.Accept'; +export const HTTP_REQUEST_HEADERS_CONNECTION = 'http.request.headers.Connection'; +export const HTTP_REQUEST_HEADERS_USER_AGENT = 'http.request.headers.User-Agent'; +export const HTTP_REQUEST_HEADERS_HOST = 'http.request.headers.Host'; +export const HTTP_REQUEST_HEADERS_ELASTIC_APM_TRACEPARENT = + 'http.request.headers.Elastic-Apm-Traceparent'; +export const HTTP_REQUEST_HEADERS_TRACESTATE = 'http.request.headers.Tracestate'; +export const HTTP_REQUEST_HEADERS_TRACEPARENT = 'http.request.headers.Traceparent'; export const USER_ID = 'user.id'; export const USER_AGENT_ORIGINAL = 'user_agent.original'; export const USER_AGENT_NAME = 'user_agent.name'; +export const USER_AGENT_DEVICE_NAME = 'user_agent.device.name'; +export const USER_AGENT_VERSION = 'user_agent.version'; export const OBSERVER_HOSTNAME = 'observer.hostname'; export const OBSERVER_LISTENING = 'observer.listening'; @@ -59,7 +81,10 @@ export const PROCESOR_NAME = 'processor.name'; export const TRANSACTION_DURATION = 'transaction.duration.us'; export const TRANSACTION_DURATION_HISTOGRAM = 'transaction.duration.histogram'; +export const TRANSACTION_DURATION_HISTOGRAM_VALUES = 'transaction.duration.histogram.values'; export const TRANSACTION_DURATION_SUMMARY = 'transaction.duration.summary'; +export const TRANSACTION_DURATION_SUMMARY_SUM = 'transaction.duration.summary.sum'; +export const TRANSACTION_DURATION_SUMMARY_VALUE_COUNT = 'transaction.duration.summary.count'; export const TRANSACTION_TYPE = 'transaction.type'; export const TRANSACTION_RESULT = 'transaction.result'; export const TRANSACTION_NAME = 'transaction.name'; @@ -74,6 +99,7 @@ export const TRANSACTION_ROOT = 'transaction.root'; export const TRANSACTION_PROFILER_STACK_TRACE_IDS = 'transaction.profiler_stack_trace_ids'; export const TRANSACTION_NAME_TEXT = 'transaction.name.text'; export const TRANSACTION_REPRESENTATIVE_COUNT = 'transaction.representative_count'; +export const TRANSACTION_SPAN_COUNT_STARTED = 'transaction.span_count.started'; export const EVENT_OUTCOME = 'event.outcome'; @@ -124,6 +150,11 @@ export const METRIC_SYSTEM_FREE_MEMORY = 'system.memory.actual.free'; export const METRIC_SYSTEM_TOTAL_MEMORY = 'system.memory.total'; export const METRIC_SYSTEM_CPU_PERCENT = 'system.cpu.total.norm.pct'; export const METRIC_PROCESS_CPU_PERCENT = 'system.process.cpu.total.norm.pct'; +export const METRIC_PROCESS_CPU_SYSTEM_NORM_PCT = 'system.process.cpu.system.norm.pct'; +export const METRIC_PROCESS_CPU_TOTAL_NORM_PCT = 'system.process.cpu.total.norm.pct'; +export const METRIC_PROCESS_CPU_TOTAL_USER_PCT = 'system.process.cpu.user.norm.pct'; +export const METRIC_PROCESS_MEMORY_RSS_BYTES = 'system.process.memory.rss.bytes'; +export const METRIC_PROCESS_MEMORY_SIZE = 'system.process.memory.size'; export const METRIC_CGROUP_MEMORY_LIMIT_BYTES = 'system.process.cgroup.memory.mem.limit.bytes'; export const METRIC_CGROUP_MEMORY_USAGE_BYTES = 'system.process.cgroup.memory.mem.usage.bytes'; @@ -146,6 +177,8 @@ export const LABEL_TYPE = 'labels.type'; export const LABEL_TELEMETRY_AUTO_VERSION = 'labels.telemetry_auto_version'; export const LABEL_LIFECYCLE_STATE = 'labels.lifecycle_state'; export const LABEL_SOME_RESOURCE_ATTRIBUTE = 'labels.some_resource_attribute'; +export const LABEL_ENV = 'labels.env'; +export const LABEL_HOSTNAME = 'labels.hostname'; export const HOST = 'host'; export const HOST_HOSTNAME = 'host.hostname'; // Do not use. Please use `HOST_NAME` instead. @@ -153,6 +186,7 @@ export const HOST_NAME = 'host.name'; export const HOST_OS_PLATFORM = 'host.os.platform'; export const HOST_ARCHITECTURE = 'host.architecture'; export const HOST_OS_VERSION = 'host.os.version'; +export const HOST_IP = 'host.ip'; export const CONTAINER_ID = 'container.id'; export const CONTAINER = 'container'; @@ -161,6 +195,8 @@ export const CONTAINER_IMAGE = 'container.image.name'; export const KUBERNETES = 'kubernetes'; export const KUBERNETES_POD_NAME = 'kubernetes.pod.name'; export const KUBERNETES_POD_UID = 'kubernetes.pod.uid'; +export const KUBERNETES_NAMESPACE = 'kubernetes.namespace'; +export const KUBERNETES_NODE_NAME = 'kubernetes.node.name'; export const FAAS_ID = 'faas.id'; export const FAAS_NAME = 'faas.name'; @@ -204,7 +240,25 @@ export const CLIENT_GEO_REGION_ISO_CODE = 'client.geo.region_iso_code'; export const CLIENT_GEO_COUNTRY_NAME = 'client.geo.country_name'; export const CLIENT_GEO_CITY_NAME = 'client.geo.city_name'; export const CLIENT_GEO_REGION_NAME = 'client.geo.region_name'; +export const CLIENT_IP = 'client.ip'; export const CHILD_ID = 'child.id'; export const LOG_LEVEL = 'log.level'; + +export const NODEJS_EVENTLOOP_DELAY_AVG_MS = 'nodejs.eventloop.delay.avg.ms'; +export const NODEJS_HANDLES_ACTIVE = 'nodejs.handles.active'; +export const NODEJS_MEMORY_ARRAYBUFFERS_BYTES = 'nodejs.memory.arrayBuffers.bytes'; +export const NODEJS_MEMORY_EXTERNAL_BYTES = 'nodejs.memory.external.bytes'; +export const NODEJS_MEMORY_HEAP_ALLOCATED_BYTES = 'nodejs.memory.heap.allocated.bytes'; +export const NODEJS_MEMORY_HEAP_USED_BYTES = 'nodejs.memory.heap.used.bytes'; +export const NODEJS_REQUESTS_ACTIVE = 'nodejs.requests.active'; + +export const PROCESS_ARGS = 'process.args'; +export const PROCESS_PARENT_PID = 'process.parent.pid'; +export const PROCESS_PID = 'process.pid'; +export const PROCESS_TITLE = 'process.title'; + +export const SOURCE_IP = 'source.ip'; + +export const DOC_COUNT = '_doc_count'; diff --git a/x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts b/x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts index abebac6512b50..6aafed8b4877d 100644 --- a/x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts +++ b/x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts @@ -11,7 +11,6 @@ import { createHash } from 'crypto'; import { flatten, merge, pickBy, sortBy, sum, uniq } from 'lodash'; import { SavedObjectsClient } from '@kbn/core/server'; import type { APMIndices } from '@kbn/apm-data-access-plugin/server'; -import { normalizeFields } from '../../../utils/normalize_fields'; import { AGENT_NAMES, RUM_AGENT_NAMES } from '../../../../common/agent_name'; import { AGENT_ACTIVATION_METHOD, @@ -55,10 +54,7 @@ import { SavedServiceGroup, } from '../../../../common/service_groups'; import { asMutableArray } from '../../../../common/utils/as_mutable_array'; -import { APMError } from '../../../../typings/es_schemas/ui/apm_error'; import { AgentName } from '../../../../typings/es_schemas/ui/fields/agent'; -import { Span } from '../../../../typings/es_schemas/ui/span'; -import { Transaction } from '../../../../typings/es_schemas/ui/transaction'; import { APMDataTelemetry, APMPerService, @@ -75,6 +71,7 @@ import { APM_CUSTOM_DASHBOARDS_SAVED_OBJECT_TYPE, SavedApmCustomDashboard, } from '../../../../common/custom_dashboards'; +import { serviceVersionMapping } from '../../../utils/es_fields_mappings'; type ISavedObjectsClient = Pick; const TIME_RANGES = ['1d', 'all'] as const; @@ -695,10 +692,7 @@ export const tasks: TelemetryTask[] = [ }, }); - const hit = normalizeFields(response.hits.hits[0]?.fields) as Pick< - Transaction | Span | APMError, - 'observer' - >; + const hit = serviceVersionMapping(response.hits.hits[0]?.fields); if (!hit || !hit.observer?.version) { return {}; diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_metadata_details.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_metadata_details.ts index 86fdcf1d8d95d..515f6ef5ea3a1 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_metadata_details.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_metadata_details.ts @@ -7,6 +7,11 @@ import { merge } from 'lodash'; import { rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; +import { + metadataAppMetricMapping, + metadataAppTransactionEventMapping, + metaDataAppTransactionMetric, +} from '../../utils/es_fields_mappings'; import { METRICSET_NAME, SERVICE_NAME, SERVICE_NODE_NAME } from '../../../common/es_fields/apm'; import { maybe } from '../../../common/utils/maybe'; import { @@ -20,7 +25,6 @@ import { Container } from '../../../typings/es_schemas/raw/fields/container'; import { Kubernetes } from '../../../typings/es_schemas/raw/fields/kubernetes'; import { Host } from '../../../typings/es_schemas/raw/fields/host'; import { Cloud } from '../../../typings/es_schemas/raw/fields/cloud'; -import { normalizeFields } from '../../utils/normalize_fields'; export interface ServiceInstanceMetadataDetailsResponse { '@timestamp': string; @@ -72,11 +76,7 @@ export async function getServiceInstanceMetadataDetails({ } ); - return maybe( - normalizeFields( - response.hits.hits[0]?.fields - ) as unknown as ServiceInstanceMetadataDetailsResponse - ); + return maybe(metadataAppMetricMapping(response.hits.hits[0]?.fields)); } async function getTransactionEventSample() { @@ -96,7 +96,7 @@ export async function getServiceInstanceMetadataDetails({ } ); - return maybe(normalizeFields(response.hits.hits[0]?.fields)); + return maybe(metadataAppTransactionEventMapping(response.hits.hits[0]?.fields)); } async function getTransactionMetricSample() { @@ -119,7 +119,7 @@ export async function getServiceInstanceMetadataDetails({ }, } ); - return maybe(normalizeFields(response.hits.hits[0]?.fields)); + return maybe(metaDataAppTransactionMetric(response.hits.hits[0]?.fields)); } // we can expect the most detail of application metrics, @@ -138,6 +138,7 @@ export async function getServiceInstanceMetadataDetails({ applicationMetricSample ); + // todo: find scenarios for container and cloud then add to mappings const { agent, service, container, kubernetes, host, cloud } = sample; return { diff --git a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts index 8748067130cfa..cd1b3ec84e9d6 100644 --- a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts +++ b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts @@ -69,7 +69,77 @@ import { CLOUD_PROVIDER, CONTAINER, KUBERNETES, + AGENT_ACTIVATION_METHOD, + CLIENT_IP, + DOC_COUNT, + EVENT_SUCCESS_COUNT_SUM, + EVENT_SUCCESS_COUNT_VALUE_COUNT, + HOST_ARCHITECTURE, + HOST_HOSTNAME, + HOST_IP, + HOST_OS_PLATFORM, + HTTP_REQUEST_HEADERS_ACCEPT, + HTTP_REQUEST_HEADERS_CONNECTION, + HTTP_REQUEST_HEADERS_ELASTIC_APM_TRACEPARENT, + HTTP_REQUEST_HEADERS_HOST, + HTTP_REQUEST_HEADERS_TRACEPARENT, + HTTP_REQUEST_HEADERS_TRACESTATE, + HTTP_REQUEST_HEADERS_USER_AGENT, + HTTP_REQUEST_METHOD, + HTTP_RESPONSE_HEADERS_CONNECTION, + HTTP_RESPONSE_HEADERS_DATE, + HTTP_RESPONSE_HEADERS_TRANSFER_ENCODING, + HTTP_RESPONSE_HEADERS_X_POWERED_BY, + HTTP_RESPONSE_STATUS_CODE, + HTTP_VERSION, + KUBERNETES_NAMESPACE, + KUBERNETES_NODE_NAME, + KUBERNETES_POD_NAME, + KUBERNETES_POD_UID, + LABEL_ENV, + LABEL_HOSTNAME, + METRIC_PROCESS_CPU_SYSTEM_NORM_PCT, + METRIC_PROCESS_CPU_TOTAL_NORM_PCT, + METRIC_PROCESS_CPU_TOTAL_USER_PCT, + METRIC_PROCESS_MEMORY_RSS_BYTES, + METRIC_PROCESS_MEMORY_SIZE, + METRIC_SYSTEM_CPU_PERCENT, + METRIC_SYSTEM_FREE_MEMORY, + METRIC_SYSTEM_TOTAL_MEMORY, + METRICSET_INTERVAL, + METRICSET_NAME, + NODEJS_EVENTLOOP_DELAY_AVG_MS, + NODEJS_HANDLES_ACTIVE, + NODEJS_MEMORY_ARRAYBUFFERS_BYTES, + NODEJS_MEMORY_EXTERNAL_BYTES, + NODEJS_MEMORY_HEAP_ALLOCATED_BYTES, + NODEJS_MEMORY_HEAP_USED_BYTES, + NODEJS_REQUESTS_ACTIVE, + PROCESS_ARGS, + PROCESS_PARENT_PID, + PROCESS_PID, + PROCESS_TITLE, + SERVICE_FRAMEWORK_VERSION, + SERVICE_LANGUAGE_NAME, + SERVICE_RUNTIME_NAME, + SERVICE_RUNTIME_VERSION, + SERVICE_VERSION, + SOURCE_IP, + TRANSACTION_DURATION_HISTOGRAM_VALUES, + TRANSACTION_DURATION_SUMMARY_SUM, + TRANSACTION_DURATION_SUMMARY_VALUE_COUNT, + TRANSACTION_SPAN_COUNT_STARTED, + URL_DOMAIN, + URL_ORIGINAL, + URL_PATH, + URL_PORT, + URL_SCHEME, + USER_AGENT_DEVICE_NAME, + USER_AGENT_NAME, + USER_AGENT_ORIGINAL, + USER_AGENT_VERSION, } from '@kbn/apm-types'; +import { Transaction } from '../../typings/es_schemas/ui/transaction'; import { TransactionRaw } from '../../typings/es_schemas/raw/transaction_raw'; import { WaterfallError, @@ -81,6 +151,10 @@ import { Exception } from '../../typings/es_schemas/raw/error_raw'; type ServiceMetadataIconsRaw = Pick; +const normalizeValue = (field: unknown[] | unknown): T => { + return (Array.isArray(field) && field.length > 0 ? field[0] : field) as T; +}; + export const metadataForDependencyMapping = (fields: Partial>) => { return { span: { @@ -210,7 +284,7 @@ export const spanLinksDetailsMapping = (fields: Partial(fields[SERVICE_ENVIRONMENT]), }, processor: { - event: normalizeValue<'span'>(fields[PROCESSOR_EVENT]), + event: normalizeValue(fields[PROCESSOR_EVENT]), }, agent: { name: normalizeValue(fields[AGENT_NAME]), @@ -274,7 +348,7 @@ export const transactionMapping = (fields: Partial>) = outcome: normalizeValue(fields[EVENT_OUTCOME]), }, processor: { - event: normalizeValue<'transaction'>(fields[PROCESSOR_EVENT]), + event: normalizeValue(fields[PROCESSOR_EVENT]), }, data_stream: { namespace: normalizeValue(fields[DATA_STREAM_NAMESPACE]), @@ -483,6 +557,387 @@ export const serviceMetadataIcons = ( }; }; -const normalizeValue = (field: unknown[] | unknown): T => { - return (Array.isArray(field) && field.length > 0 ? field[0] : field) as T; +// todo(milosz): test it +export const serviceVersionMapping = ( + fields: Partial> +): Pick => { + return { + observer: { + version: normalizeValue(fields[OBSERVER_VERSION]), + version_major: normalizeValue(fields[OBSERVER_VERSION_MAJOR]), + }, + }; +}; + +export const metadataAppMetricMapping = (fields: Partial>) => { + return { + '@timestamp': normalizeValue(fields[AT_TIMESTAMP]), + agent: { + name: normalizeValue(fields[AGENT_NAME]), + version: normalizeValue(fields[AGENT_VERSION]), + activation_method: normalizeValue(fields[AGENT_ACTIVATION_METHOD]), + }, + data_stream: { + namespace: normalizeValue(fields[DATA_STREAM_NAMESPACE]), + type: normalizeValue(fields[DATA_STEAM_TYPE]), + dataset: normalizeValue(fields[DATA_STREAM_DATASET]), + }, + host: { + architecture: normalizeValue(fields[HOST_ARCHITECTURE]), + hostname: normalizeValue(fields[HOST_HOSTNAME]), + ip: normalizeValue(fields[HOST_IP]), + name: normalizeValue(fields[HOST_NAME]), + os: { + platform: normalizeValue(fields[HOST_OS_PLATFORM]), + }, + }, + kubernetes: { + namespace: normalizeValue(fields[KUBERNETES_NAMESPACE]), + node: { + name: normalizeValue(fields[KUBERNETES_NODE_NAME]), + }, + pod: { + name: normalizeValue(fields[KUBERNETES_POD_NAME]), + uid: normalizeValue(fields[KUBERNETES_POD_UID]), + }, + }, + labels: { + env: normalizeValue(fields[LABEL_ENV]), + hostname: normalizeValue(fields[LABEL_HOSTNAME]), + }, + metricset: { + name: normalizeValue(fields[METRICSET_NAME]), + }, + nodejs: { + eventloop: { + delay: { + avg: { + ms: normalizeValue(fields[NODEJS_EVENTLOOP_DELAY_AVG_MS]), + }, + }, + }, + handles: { + active: normalizeValue(fields[NODEJS_HANDLES_ACTIVE]), + }, + memory: { + arrayBuffers: { + bytes: normalizeValue(fields[NODEJS_MEMORY_ARRAYBUFFERS_BYTES]), + }, + external: { + bytes: normalizeValue(fields[NODEJS_MEMORY_EXTERNAL_BYTES]), + }, + heap: { + allocated: { + bytes: normalizeValue(fields[NODEJS_MEMORY_HEAP_ALLOCATED_BYTES]), + }, + used: { + bytes: normalizeValue(fields[NODEJS_MEMORY_HEAP_USED_BYTES]), + }, + }, + }, + requests: { + active: normalizeValue(fields[NODEJS_REQUESTS_ACTIVE]), + }, + }, + observer: { + hostname: normalizeValue(fields[OBSERVER_HOSTNAME]), + type: normalizeValue(fields[OBSERVER_TYPE]), + version: normalizeValue(fields[OBSERVER_VERSION]), + }, + process: { + args: fields[PROCESS_ARGS] as string[] | undefined, + parent: { + pid: normalizeValue(fields[PROCESS_PARENT_PID]), + }, + pid: normalizeValue(fields[PROCESS_PID]), + title: normalizeValue(fields[PROCESS_TITLE]), + }, + processor: { + event: normalizeValue(fields[PROCESSOR_EVENT]), + }, + service: { + name: normalizeValue(fields[SERVICE_NAME]), + environment: normalizeValue(fields[SERVICE_ENVIRONMENT]), + framework: { + name: normalizeValue(fields[SERVICE_FRAMEWORK_NAME]), + versions: normalizeValue(fields[SERVICE_FRAMEWORK_VERSION]), + }, + language: { + name: normalizeValue(fields[SERVICE_LANGUAGE_NAME]), + }, + node: { + name: normalizeValue(fields[SERVICE_NODE_NAME]), + }, + runtime: { + name: normalizeValue(fields[SERVICE_RUNTIME_NAME]), + version: normalizeValue(fields[SERVICE_RUNTIME_VERSION]), + }, + version: normalizeValue(fields[SERVICE_VERSION]), + }, + system: { + cpu: { + total: { + norm: { + pct: normalizeValue(fields[METRIC_SYSTEM_CPU_PERCENT]), + }, + }, + }, + memory: { + actual: { + free: normalizeValue(fields[METRIC_SYSTEM_FREE_MEMORY]), + }, + total: normalizeValue(fields[METRIC_SYSTEM_TOTAL_MEMORY]), + }, + process: { + cpu: { + system: { + norm: { + pct: normalizeValue(fields[METRIC_PROCESS_CPU_SYSTEM_NORM_PCT]), + }, + }, + total: { + norm: { + pct: normalizeValue(fields[METRIC_PROCESS_CPU_TOTAL_NORM_PCT]), + }, + }, + user: { + norm: { + pct: normalizeValue(fields[METRIC_PROCESS_CPU_TOTAL_USER_PCT]), + }, + }, + }, + memory: { + rss: { + bytes: normalizeValue(fields[METRIC_PROCESS_MEMORY_RSS_BYTES]), + }, + size: normalizeValue(fields[METRIC_PROCESS_MEMORY_SIZE]), + }, + }, + }, + }; +}; + +export const metadataAppTransactionEventMapping = (fields: Partial>) => { + return { + kubernetes: { + namespace: normalizeValue(fields[KUBERNETES_NAMESPACE]), + node: { + name: normalizeValue(fields[KUBERNETES_NODE_NAME]), + }, + pod: { + name: normalizeValue(fields[KUBERNETES_POD_NAME]), + uid: normalizeValue(fields[KUBERNETES_POD_UID]), + }, + }, + parent: { + id: normalizeValue(fields[PARENT_ID]), + }, + agent: { + name: normalizeValue(fields[AGENT_NAME]), + version: normalizeValue(fields[AGENT_VERSION]), + activation_method: normalizeValue(fields[AGENT_ACTIVATION_METHOD]), + }, + process: { + args: fields[PROCESS_ARGS] as string[] | undefined, + parent: { + pid: normalizeValue(fields[PROCESS_PARENT_PID]), + }, + pid: normalizeValue(fields[PROCESS_PID]), + title: normalizeValue(fields[PROCESS_TITLE]), + }, + source: { + ip: normalizeValue(fields[SOURCE_IP]), + }, + processor: { + event: normalizeValue(fields[PROCESSOR_EVENT]), + }, + url: { + path: normalizeValue(fields[URL_PATH]), + original: normalizeValue(fields[URL_ORIGINAL]), + scheme: normalizeValue(fields[URL_SCHEME]), + port: normalizeValue(fields[URL_PORT]), + domain: normalizeValue(fields[URL_DOMAIN]), + full: normalizeValue(fields[SOURCE_IP]), + }, + observer: { + hostname: normalizeValue(fields[OBSERVER_HOSTNAME]), + type: normalizeValue(fields[OBSERVER_TYPE]), + version: normalizeValue(fields[OBSERVER_VERSION]), + }, + trace: { + id: normalizeValue(fields[TRACE_ID]), + }, + '@timestamp': normalizeValue(fields[AT_TIMESTAMP]), + data_stream: { + namespace: normalizeValue(fields[DATA_STREAM_NAMESPACE]), + type: normalizeValue(fields[DATA_STEAM_TYPE]), + dataset: normalizeValue(fields[DATA_STREAM_DATASET]), + }, + service: { + name: normalizeValue(fields[SERVICE_NAME]), + environment: normalizeValue(fields[SERVICE_ENVIRONMENT]), + framework: { + name: normalizeValue(fields[SERVICE_FRAMEWORK_NAME]), + versions: normalizeValue(fields[SERVICE_FRAMEWORK_VERSION]), + }, + language: { + name: normalizeValue(fields[SERVICE_LANGUAGE_NAME]), + }, + node: { + name: normalizeValue(fields[SERVICE_NODE_NAME]), + }, + runtime: { + name: normalizeValue(fields[SERVICE_RUNTIME_NAME]), + version: normalizeValue(fields[SERVICE_RUNTIME_VERSION]), + }, + version: normalizeValue(fields[SERVICE_VERSION]), + }, + host: { + architecture: normalizeValue(fields[HOST_ARCHITECTURE]), + hostname: normalizeValue(fields[HOST_HOSTNAME]), + ip: fields[HOST_IP] as string[] | undefined, + name: normalizeValue(fields[HOST_NAME]), + os: { + platform: normalizeValue(fields[HOST_OS_PLATFORM]), + }, + }, + client: { + ip: normalizeValue(fields[CLIENT_IP]), + }, + http: { + request: { + headers: { + Accept: fields[HTTP_REQUEST_HEADERS_ACCEPT] as string[] | undefined, + Connection: fields[HTTP_REQUEST_HEADERS_CONNECTION] as string[] | undefined, + 'User-Agent': fields[HTTP_REQUEST_HEADERS_USER_AGENT] as string[] | undefined, + Host: fields[HTTP_REQUEST_HEADERS_HOST] as string[] | undefined, + 'Elastic-Apm-Traceparent': fields[HTTP_REQUEST_HEADERS_ELASTIC_APM_TRACEPARENT] as + | string[] + | undefined, + Tracestate: fields[HTTP_REQUEST_HEADERS_TRACESTATE] as string[] | undefined, + Traceparent: fields[HTTP_REQUEST_HEADERS_TRACEPARENT] as string[] | undefined, + }, + method: normalizeValue(fields[HTTP_REQUEST_METHOD]), + }, + response: { + headers: { + 'Transfer-Encoding': fields[HTTP_RESPONSE_HEADERS_TRANSFER_ENCODING] as + | string + | undefined, + Connection: fields[HTTP_RESPONSE_HEADERS_CONNECTION] as string | undefined, + Date: fields[HTTP_RESPONSE_HEADERS_DATE] as string | undefined, + 'X-Powered-By': fields[HTTP_RESPONSE_HEADERS_X_POWERED_BY] as string | undefined, + }, + status_code: normalizeValue(fields[HTTP_RESPONSE_STATUS_CODE]), + }, + version: normalizeValue(fields[HTTP_VERSION]), + }, + event: { + success_count: normalizeValue(fields[EVENT_SUCCESS_COUNT]), + outcome: normalizeValue(fields[EVENT_OUTCOME]), + }, + transaction: { + result: normalizeValue(fields[TRANSACTION_RESULT]), + representative_count: normalizeValue(fields[TRANSACTION_REPRESENTATIVE_COUNT]), + sampled: normalizeValue(fields[TRANSACTION_SAMPLED]), + id: normalizeValue(fields[TRANSACTION_ID]), + duration: { + us: normalizeValue(fields[TRANSACTION_DURATION]), + }, + type: normalizeValue(fields[TRANSACTION_TYPE]), + name: normalizeValue(fields[TRANSACTION_NAME]), + span_count: { + started: normalizeValue(fields[TRANSACTION_SPAN_COUNT_STARTED]), + }, + }, + user_agent: { + original: normalizeValue(fields[USER_AGENT_ORIGINAL]), + name: normalizeValue(fields[USER_AGENT_NAME]), + device: { + name: normalizeValue(fields[USER_AGENT_DEVICE_NAME]), + }, + version: normalizeValue(fields[USER_AGENT_VERSION]), + }, + span: { + id: normalizeValue(fields[SPAN_ID]), + }, + timestamp: { + us: normalizeValue(fields[TIMESTAMP]), + }, + }; +}; + +export const metaDataAppTransactionMetric = (fields: Partial>) => { + return { + '@timestamp': normalizeValue(fields[AT_TIMESTAMP]), + _doc_count: normalizeValue(fields[DOC_COUNT]), + agent: { + name: normalizeValue(fields[AGENT_NAME]), + }, + data_stream: { + namespace: normalizeValue(fields[DATA_STREAM_NAMESPACE]), + type: normalizeValue(fields[DATA_STEAM_TYPE]), + dataset: normalizeValue(fields[DATA_STREAM_DATASET]), + }, + event: { + outcome: normalizeValue(fields[EVENT_OUTCOME]), + success_count: { + sum: normalizeValue(fields[EVENT_SUCCESS_COUNT_SUM]), + value_count: normalizeValue(fields[EVENT_SUCCESS_COUNT_VALUE_COUNT]), + }, + }, + host: { + hostname: normalizeValue(fields[HOST_HOSTNAME]), + name: normalizeValue(fields[HOST_NAME]), + os: { + platform: normalizeValue(fields[HOST_OS_PLATFORM]), + }, + }, + kubernetes: { + pod: { + name: normalizeValue(fields[KUBERNETES_POD_NAME]), + }, + }, + metricset: { + name: normalizeValue(fields[METRICSET_NAME]), + interval: normalizeValue(fields[METRICSET_INTERVAL]), + }, + observer: { + hostname: normalizeValue(fields[OBSERVER_HOSTNAME]), + type: normalizeValue(fields[OBSERVER_TYPE]), + version: normalizeValue(fields[OBSERVER_VERSION]), + }, + processor: { + event: normalizeValue(fields[PROCESSOR_EVENT]), + }, + service: { + name: normalizeValue(fields[SERVICE_NAME]), + environment: normalizeValue(fields[SERVICE_ENVIRONMENT]), + language: { + name: normalizeValue(fields[SERVICE_LANGUAGE_NAME]), + }, + node: { + name: normalizeValue(fields[SERVICE_NODE_NAME]), + }, + runtime: { + name: normalizeValue(fields[SERVICE_RUNTIME_NAME]), + version: normalizeValue(fields[SERVICE_RUNTIME_VERSION]), + }, + version: normalizeValue(fields[SERVICE_VERSION]), + }, + transaction: { + duration: { + histogram: { + values: fields[TRANSACTION_DURATION_HISTOGRAM_VALUES] as number[] | undefined, + }, + summary: { + sum: normalizeValue(fields[TRANSACTION_DURATION_SUMMARY_SUM]), + value_count: normalizeValue(fields[TRANSACTION_DURATION_SUMMARY_VALUE_COUNT]), + }, + }, + name: normalizeValue(fields[TRANSACTION_NAME]), + result: normalizeValue(fields[TRANSACTION_RESULT]), + type: normalizeValue(fields[TRANSACTION_TYPE]), + }, + }; }; From e77d9a4e0cb7fcc725070930103326a368a3a324 Mon Sep 17 00:00:00 2001 From: Milosz Marcinkowski Date: Mon, 23 Sep 2024 15:46:20 +0200 Subject: [PATCH 12/25] 'service agent name' mappings --- .../apm/server/routes/services/get_service_agent.ts | 5 ++--- .../routes/services/get_service_metadata_details.ts | 1 + .../apm/server/utils/es_fields_mappings.ts | 8 ++++++++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_agent.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_agent.ts index cbfddfe585e77..ed678a5c34748 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_agent.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_agent.ts @@ -7,6 +7,7 @@ import { rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; +import { serviceAgentName } from '../../utils/es_fields_mappings'; import { AGENT_NAME, SERVICE_NAME, @@ -16,7 +17,6 @@ import { } from '../../../common/es_fields/apm'; import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; import { getServerlessTypeFromCloudData, ServerlessType } from '../../../common/serverless'; -import { normalizeFields } from '../../utils/normalize_fields'; interface ServiceAgent { agent?: { @@ -102,8 +102,7 @@ export async function getServiceAgent({ return {}; } - // TODO fix in the mapper - const { agent, service, cloud } = normalizeFields(response.hits.hits[0].fields) as ServiceAgent; + const { agent, service, cloud } = serviceAgentName(response.hits.hits[0].fields) as ServiceAgent; const serverlessType = getServerlessTypeFromCloudData(cloud?.provider, cloud?.service?.name); return { diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts index a4f79109b0922..3609905e2d28f 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts @@ -173,6 +173,7 @@ export async function getServiceMetadataDetails({ const response = await apmEventClient.search('get_service_metadata_details', params); const fields = response.hits.hits[0]?.fields; + // todo: missing `fields` property? const fieldsNorm = (fields ? normalizeFields(fields) : undefined) as | ServiceMetadataDetailsRaw | undefined; diff --git a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts index cd1b3ec84e9d6..9970bcc6bb435 100644 --- a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts +++ b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts @@ -941,3 +941,11 @@ export const metaDataAppTransactionMetric = (fields: Partial>) => { + return { + agent: { + name: normalizeValue(fields[AGENT_NAME]), + }, + }; +}; From f8f9b7811eb3c827f1b30aa3588b0d80b901643f Mon Sep 17 00:00:00 2001 From: Milosz Marcinkowski Date: Wed, 25 Sep 2024 16:40:49 +0200 Subject: [PATCH 13/25] add fields to Span mapping --- packages/kbn-apm-types/src/es_fields/apm.ts | 45 +++++++- .../apm/server/utils/es_fields_mappings.ts | 105 +++++++++++++++++- 2 files changed, 147 insertions(+), 3 deletions(-) diff --git a/packages/kbn-apm-types/src/es_fields/apm.ts b/packages/kbn-apm-types/src/es_fields/apm.ts index 58320ff96ba9f..116686f503b97 100644 --- a/packages/kbn-apm-types/src/es_fields/apm.ts +++ b/packages/kbn-apm-types/src/es_fields/apm.ts @@ -259,6 +259,49 @@ export const PROCESS_PARENT_PID = 'process.parent.pid'; export const PROCESS_PID = 'process.pid'; export const PROCESS_TITLE = 'process.title'; +export const SCOPE_NAME = 'scope.name'; +export const SCOPE_ATTRIBUTES_SERVICE_FRAMEWORK_NAME = 'scope.attributes.service.framework.name'; +export const SCOPE_ATTRIBUTES_SERVICE_FRAMEWORK_VERSION = + 'scope.attributes.service.framework.version'; +export const SCOPE_DROPPED_ATTRIBUTES_COUNT = 'scope.dropped_attributes_count'; + +export const RESOURCE_ATTRIBUTES_AGENT_NAME = 'resource.attributes.agent.name'; +export const RESOURCE_ATTRIBUTES_AGENT_VERSION = 'resource.attributes.agent.version'; +export const RESOURCE_ATTRIBUTES_SERVICE_INSTANCE_ID = 'resource.attributes.service.instance.id'; +export const RESOURCE_ATTRIBUTES_SERVICE_NAME = 'resource.attributes.service.name'; +export const RESOURCE_ATTRIBUTES_SOME_RESOURCE_ATTRIBUTE = + 'resource.attributes.some.resource.attribute'; +export const RESOURCE_DROPPED_ATTRIBUTES_COUNT = 'resource.dropped_attributes_count'; +export const RESOURCE_SCHEMA_URL = 'resource.schema_url'; + +export const ATTRIBUTES_EVENT_OUTCOME = 'attributes.event.outcome'; +export const ATTRIBUTES_EVENT_SUCCESS_COUNT = 'attributes.event.success_count'; +export const ATTRIBUTES_HTTP_STATUS_CODE = 'attributes.http.status_code'; +export const ATTRIBUTES_PEER_SERVICE = 'attributes.peer.service'; +export const ATTRIBUTES_PROCESSOR_EVENT = 'attributes.processor.event'; +export const ATTRIBUTES_SERVICE_TARGET_NAME = 'attributes.service.target.name'; +export const ATTRIBUTES_SERVICE_TARGET_TYPE = 'attributes.service.target.type'; +export const ATTRIBUTES_SOME_SPAN_ATTRIBUTE = 'attributes.some.span.attribute'; +export const ATTRIBUTES_SPAN_DESTINATION_SERVICE_RESOURCE = + 'attributes.span.destination.service.resource'; +export const ATTRIBUTES_SPAN_DURATION_US = 'attributes.span.duration.us'; +export const ATTRIBUTES_SPAN_NAME = 'attributes.span.name'; +export const ATTRIBUTES_SPAN_RESPSENTATIVE_COUNT = 'attributes.span.representative_count'; +export const ATTRIBUTES_SPAN_SUBTYPE = 'attributes.span.subtype'; +export const ATTRIBUTES_SPAN_TYPE = 'attributes.span.type'; +export const ATTRIBUTES_TIMESTAMP_US = 'attributes.timestamp.us'; + +export const DROPPED_ATTRIBUTES_COUNT = 'dropped_attributes_count'; +export const DROPPED_EVENTS_COUNT = 'dropped_events_count'; +export const DROPPED_LINKS_COUNT = 'dropped_links_count'; + +// other otel collector fields export const SOURCE_IP = 'source.ip'; - export const DOC_COUNT = '_doc_count'; +export const DURATION = 'duration'; +export const KIND = 'kind'; +export const NAME = 'name'; +export const PARENT_SPAN_ID = 'parent_span_id'; +export const OTEL_SPAN_ID = 'span_id'; +export const OTEL_TRACE_ID = 'trace_id'; +export const STATUS_CODE = 'status.code'; diff --git a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts index 9970bcc6bb435..37476cc726f9b 100644 --- a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts +++ b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts @@ -138,6 +138,42 @@ import { USER_AGENT_NAME, USER_AGENT_ORIGINAL, USER_AGENT_VERSION, + ATTRIBUTES_EVENT_OUTCOME, + ATTRIBUTES_EVENT_SUCCESS_COUNT, + ATTRIBUTES_HTTP_STATUS_CODE, + ATTRIBUTES_PEER_SERVICE, + ATTRIBUTES_PROCESSOR_EVENT, + ATTRIBUTES_SERVICE_TARGET_NAME, + ATTRIBUTES_SERVICE_TARGET_TYPE, + ATTRIBUTES_SOME_SPAN_ATTRIBUTE, + ATTRIBUTES_SPAN_DESTINATION_SERVICE_RESOURCE, + ATTRIBUTES_SPAN_DURATION_US, + ATTRIBUTES_SPAN_NAME, + ATTRIBUTES_SPAN_RESPSENTATIVE_COUNT, + ATTRIBUTES_SPAN_SUBTYPE, + ATTRIBUTES_SPAN_TYPE, + ATTRIBUTES_TIMESTAMP_US, + DROPPED_ATTRIBUTES_COUNT, + DROPPED_EVENTS_COUNT, + DROPPED_LINKS_COUNT, + DURATION, + KIND, + NAME, + OTEL_SPAN_ID, + OTEL_TRACE_ID, + PARENT_SPAN_ID, + RESOURCE_ATTRIBUTES_AGENT_NAME, + RESOURCE_ATTRIBUTES_AGENT_VERSION, + RESOURCE_ATTRIBUTES_SERVICE_INSTANCE_ID, + RESOURCE_ATTRIBUTES_SERVICE_NAME, + RESOURCE_ATTRIBUTES_SOME_RESOURCE_ATTRIBUTE, + RESOURCE_DROPPED_ATTRIBUTES_COUNT, + RESOURCE_SCHEMA_URL, + SCOPE_ATTRIBUTES_SERVICE_FRAMEWORK_NAME, + SCOPE_ATTRIBUTES_SERVICE_FRAMEWORK_VERSION, + SCOPE_DROPPED_ATTRIBUTES_COUNT, + SCOPE_NAME, + STATUS_CODE, } from '@kbn/apm-types'; import { Transaction } from '../../typings/es_schemas/ui/transaction'; import { TransactionRaw } from '../../typings/es_schemas/raw/transaction_raw'; @@ -205,8 +241,7 @@ export const topDependencySpansMapping = (fields: Partial>): Span => { +export const spanMapping = (fields: Partial>) => { return { parent: { id: normalizeValue(fields[PARENT_ID]), @@ -255,6 +290,72 @@ export const spanMapping = (fields: Partial>): Span => timestamp: { us: normalizeValue(fields[TIMESTAMP]), }, + scope: { + attributes: { + 'service.framework.name': normalizeValue( + fields[SCOPE_ATTRIBUTES_SERVICE_FRAMEWORK_NAME] + ), + 'service.framework.version': normalizeValue( + fields[SCOPE_ATTRIBUTES_SERVICE_FRAMEWORK_VERSION] + ), + }, + dropped_attributes_count: normalizeValue(fields[SCOPE_DROPPED_ATTRIBUTES_COUNT]), + name: normalizeValue(fields[SCOPE_NAME]), + }, + resource: { + attributes: { + 'agent.name': normalizeValue(fields[RESOURCE_ATTRIBUTES_AGENT_NAME]), + 'agent.version': normalizeValue(fields[RESOURCE_ATTRIBUTES_AGENT_VERSION]), + 'service.instance.id': normalizeValue( + fields[RESOURCE_ATTRIBUTES_SERVICE_INSTANCE_ID] + ), + 'service.name': normalizeValue(fields[RESOURCE_ATTRIBUTES_SERVICE_NAME]), + 'some.resource.attribute': normalizeValue( + fields[RESOURCE_ATTRIBUTES_SOME_RESOURCE_ATTRIBUTE] + ), + }, + dropped_attributes_count: normalizeValue(fields[RESOURCE_DROPPED_ATTRIBUTES_COUNT]), + schema_url: normalizeValue(fields[RESOURCE_SCHEMA_URL]), + }, + attributes: { + 'event.outcome': normalizeValue(fields[ATTRIBUTES_EVENT_OUTCOME]), + 'event.success_count': normalizeValue(fields[ATTRIBUTES_EVENT_SUCCESS_COUNT]), + 'http.status_code': normalizeValue(fields[ATTRIBUTES_HTTP_STATUS_CODE]), + 'peer.service': normalizeValue(fields[ATTRIBUTES_PEER_SERVICE]), + 'processor.event': normalizeValue(fields[ATTRIBUTES_PROCESSOR_EVENT]), + 'service.target.name': normalizeValue(fields[ATTRIBUTES_SERVICE_TARGET_NAME]), + 'service.target.type': normalizeValue(fields[ATTRIBUTES_SERVICE_TARGET_TYPE]), + 'some.span.attribute': normalizeValue(fields[ATTRIBUTES_SOME_SPAN_ATTRIBUTE]), + 'span.destination.service.resource': normalizeValue( + fields[ATTRIBUTES_SPAN_DESTINATION_SERVICE_RESOURCE] + ), + 'span.duration.us': normalizeValue(fields[ATTRIBUTES_SPAN_DURATION_US]), + 'span.name': normalizeValue(fields[ATTRIBUTES_SPAN_NAME]), + 'span.representative_count': normalizeValue( + fields[ATTRIBUTES_SPAN_RESPSENTATIVE_COUNT] + ), + 'span.subtype': normalizeValue(fields[ATTRIBUTES_SPAN_SUBTYPE]), + 'span.type': normalizeValue(fields[ATTRIBUTES_SPAN_TYPE]), + 'timestamp.us': normalizeValue(fields[ATTRIBUTES_TIMESTAMP_US]), + }, + data_stream: { + namespace: normalizeValue(fields[DATA_STREAM_NAMESPACE]), + type: normalizeValue(fields[DATA_STEAM_TYPE]), + dataset: normalizeValue(fields[DATA_STREAM_DATASET]), + }, + dropped_attributes_count: normalizeValue(fields[DROPPED_ATTRIBUTES_COUNT]), + dropped_events_count: normalizeValue(fields[DROPPED_EVENTS_COUNT]), + dropped_links_count: normalizeValue(fields[DROPPED_LINKS_COUNT]), + duration: normalizeValue(fields[DURATION]), + kind: normalizeValue(fields[KIND]), + // links: // todo(milosz): handle arrays, what is the format in `fields` + name: normalizeValue(fields[NAME]), + parent_span_id: normalizeValue(fields[PARENT_SPAN_ID]), + span_id: normalizeValue(fields[OTEL_SPAN_ID]), + status: { + code: normalizeValue(fields[STATUS_CODE]), + }, + trace_id: normalizeValue(fields[OTEL_TRACE_ID]), }; }; From 3eac633c776eb930cffe0017d2c80979ba8adcfe Mon Sep 17 00:00:00 2001 From: Milosz Marcinkowski Date: Thu, 26 Sep 2024 16:54:38 +0200 Subject: [PATCH 14/25] remove unused properties in Span mapping --- packages/kbn-apm-types/src/es_fields/apm.ts | 47 +------- .../apm/server/utils/es_fields_mappings.ts | 114 +++--------------- 2 files changed, 20 insertions(+), 141 deletions(-) diff --git a/packages/kbn-apm-types/src/es_fields/apm.ts b/packages/kbn-apm-types/src/es_fields/apm.ts index 116686f503b97..74718530c2fc5 100644 --- a/packages/kbn-apm-types/src/es_fields/apm.ts +++ b/packages/kbn-apm-types/src/es_fields/apm.ts @@ -42,6 +42,7 @@ export const SERVICE_RUNTIME_VERSION = 'service.runtime.version'; export const SERVICE_NODE_NAME = 'service.node.name'; export const SERVICE_VERSION = 'service.version'; export const SERVICE_TARGET_TYPE = 'service.target.type'; +export const SERVICE_TARGET_NAME = 'service.target.name'; export const SERVICE_OVERFLOW_COUNT = 'service_transaction.aggregation.overflow_count'; export const URL_FULL = 'url.full'; @@ -128,6 +129,7 @@ export const SPAN_COMPOSITE_SUM = 'span.composite.sum.us'; export const SPAN_COMPOSITE_COMPRESSION_STRATEGY = 'span.composite.compression_strategy'; export const SPAN_SYNC = 'span.sync'; +export const SPAN_REPRESENTATIVE_COUNT = 'span.representative_count'; // Parent ID for a transaction or span export const PARENT_ID = 'parent.id'; @@ -259,49 +261,6 @@ export const PROCESS_PARENT_PID = 'process.parent.pid'; export const PROCESS_PID = 'process.pid'; export const PROCESS_TITLE = 'process.title'; -export const SCOPE_NAME = 'scope.name'; -export const SCOPE_ATTRIBUTES_SERVICE_FRAMEWORK_NAME = 'scope.attributes.service.framework.name'; -export const SCOPE_ATTRIBUTES_SERVICE_FRAMEWORK_VERSION = - 'scope.attributes.service.framework.version'; -export const SCOPE_DROPPED_ATTRIBUTES_COUNT = 'scope.dropped_attributes_count'; - -export const RESOURCE_ATTRIBUTES_AGENT_NAME = 'resource.attributes.agent.name'; -export const RESOURCE_ATTRIBUTES_AGENT_VERSION = 'resource.attributes.agent.version'; -export const RESOURCE_ATTRIBUTES_SERVICE_INSTANCE_ID = 'resource.attributes.service.instance.id'; -export const RESOURCE_ATTRIBUTES_SERVICE_NAME = 'resource.attributes.service.name'; -export const RESOURCE_ATTRIBUTES_SOME_RESOURCE_ATTRIBUTE = - 'resource.attributes.some.resource.attribute'; -export const RESOURCE_DROPPED_ATTRIBUTES_COUNT = 'resource.dropped_attributes_count'; -export const RESOURCE_SCHEMA_URL = 'resource.schema_url'; - -export const ATTRIBUTES_EVENT_OUTCOME = 'attributes.event.outcome'; -export const ATTRIBUTES_EVENT_SUCCESS_COUNT = 'attributes.event.success_count'; -export const ATTRIBUTES_HTTP_STATUS_CODE = 'attributes.http.status_code'; -export const ATTRIBUTES_PEER_SERVICE = 'attributes.peer.service'; -export const ATTRIBUTES_PROCESSOR_EVENT = 'attributes.processor.event'; -export const ATTRIBUTES_SERVICE_TARGET_NAME = 'attributes.service.target.name'; -export const ATTRIBUTES_SERVICE_TARGET_TYPE = 'attributes.service.target.type'; -export const ATTRIBUTES_SOME_SPAN_ATTRIBUTE = 'attributes.some.span.attribute'; -export const ATTRIBUTES_SPAN_DESTINATION_SERVICE_RESOURCE = - 'attributes.span.destination.service.resource'; -export const ATTRIBUTES_SPAN_DURATION_US = 'attributes.span.duration.us'; -export const ATTRIBUTES_SPAN_NAME = 'attributes.span.name'; -export const ATTRIBUTES_SPAN_RESPSENTATIVE_COUNT = 'attributes.span.representative_count'; -export const ATTRIBUTES_SPAN_SUBTYPE = 'attributes.span.subtype'; -export const ATTRIBUTES_SPAN_TYPE = 'attributes.span.type'; -export const ATTRIBUTES_TIMESTAMP_US = 'attributes.timestamp.us'; - -export const DROPPED_ATTRIBUTES_COUNT = 'dropped_attributes_count'; -export const DROPPED_EVENTS_COUNT = 'dropped_events_count'; -export const DROPPED_LINKS_COUNT = 'dropped_links_count'; - -// other otel collector fields export const SOURCE_IP = 'source.ip'; + export const DOC_COUNT = '_doc_count'; -export const DURATION = 'duration'; -export const KIND = 'kind'; -export const NAME = 'name'; -export const PARENT_SPAN_ID = 'parent_span_id'; -export const OTEL_SPAN_ID = 'span_id'; -export const OTEL_TRACE_ID = 'trace_id'; -export const STATUS_CODE = 'status.code'; diff --git a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts index 37476cc726f9b..3af39f90fe0a5 100644 --- a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts +++ b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts @@ -138,42 +138,9 @@ import { USER_AGENT_NAME, USER_AGENT_ORIGINAL, USER_AGENT_VERSION, - ATTRIBUTES_EVENT_OUTCOME, - ATTRIBUTES_EVENT_SUCCESS_COUNT, - ATTRIBUTES_HTTP_STATUS_CODE, - ATTRIBUTES_PEER_SERVICE, - ATTRIBUTES_PROCESSOR_EVENT, - ATTRIBUTES_SERVICE_TARGET_NAME, - ATTRIBUTES_SERVICE_TARGET_TYPE, - ATTRIBUTES_SOME_SPAN_ATTRIBUTE, - ATTRIBUTES_SPAN_DESTINATION_SERVICE_RESOURCE, - ATTRIBUTES_SPAN_DURATION_US, - ATTRIBUTES_SPAN_NAME, - ATTRIBUTES_SPAN_RESPSENTATIVE_COUNT, - ATTRIBUTES_SPAN_SUBTYPE, - ATTRIBUTES_SPAN_TYPE, - ATTRIBUTES_TIMESTAMP_US, - DROPPED_ATTRIBUTES_COUNT, - DROPPED_EVENTS_COUNT, - DROPPED_LINKS_COUNT, - DURATION, - KIND, - NAME, - OTEL_SPAN_ID, - OTEL_TRACE_ID, - PARENT_SPAN_ID, - RESOURCE_ATTRIBUTES_AGENT_NAME, - RESOURCE_ATTRIBUTES_AGENT_VERSION, - RESOURCE_ATTRIBUTES_SERVICE_INSTANCE_ID, - RESOURCE_ATTRIBUTES_SERVICE_NAME, - RESOURCE_ATTRIBUTES_SOME_RESOURCE_ATTRIBUTE, - RESOURCE_DROPPED_ATTRIBUTES_COUNT, - RESOURCE_SCHEMA_URL, - SCOPE_ATTRIBUTES_SERVICE_FRAMEWORK_NAME, - SCOPE_ATTRIBUTES_SERVICE_FRAMEWORK_VERSION, - SCOPE_DROPPED_ATTRIBUTES_COUNT, - SCOPE_NAME, - STATUS_CODE, + SERVICE_TARGET_NAME, + SERVICE_TARGET_TYPE, + SPAN_REPRESENTATIVE_COUNT, } from '@kbn/apm-types'; import { Transaction } from '../../typings/es_schemas/ui/transaction'; import { TransactionRaw } from '../../typings/es_schemas/raw/transaction_raw'; @@ -262,9 +229,18 @@ export const spanMapping = (fields: Partial>) => { service: { name: normalizeValue(fields[SERVICE_NAME]), environment: normalizeValue(fields[SERVICE_ENVIRONMENT]), + framework: { + name: normalizeValue(fields[SERVICE_FRAMEWORK_NAME]), + version: normalizeValue(fields[SERVICE_FRAMEWORK_VERSION]), + }, + target: { + name: normalizeValue(fields[SERVICE_TARGET_NAME]), + type: normalizeValue(fields[SERVICE_TARGET_TYPE]), + }, }, event: { outcome: normalizeValue(fields[EVENT_OUTCOME]), + success_count: normalizeValue(fields[EVENT_SUCCESS_COUNT]), }, processor: { name: normalizeValue<'transaction'>(fields[PROCESOR_NAME]), @@ -286,76 +262,21 @@ export const spanMapping = (fields: Partial>) => { }, id: normalizeValue(fields[SPAN_ID]), type: normalizeValue(fields[SPAN_TYPE]), + representative_count: normalizeValue(fields[SPAN_REPRESENTATIVE_COUNT]), }, timestamp: { us: normalizeValue(fields[TIMESTAMP]), }, - scope: { - attributes: { - 'service.framework.name': normalizeValue( - fields[SCOPE_ATTRIBUTES_SERVICE_FRAMEWORK_NAME] - ), - 'service.framework.version': normalizeValue( - fields[SCOPE_ATTRIBUTES_SERVICE_FRAMEWORK_VERSION] - ), - }, - dropped_attributes_count: normalizeValue(fields[SCOPE_DROPPED_ATTRIBUTES_COUNT]), - name: normalizeValue(fields[SCOPE_NAME]), - }, - resource: { - attributes: { - 'agent.name': normalizeValue(fields[RESOURCE_ATTRIBUTES_AGENT_NAME]), - 'agent.version': normalizeValue(fields[RESOURCE_ATTRIBUTES_AGENT_VERSION]), - 'service.instance.id': normalizeValue( - fields[RESOURCE_ATTRIBUTES_SERVICE_INSTANCE_ID] - ), - 'service.name': normalizeValue(fields[RESOURCE_ATTRIBUTES_SERVICE_NAME]), - 'some.resource.attribute': normalizeValue( - fields[RESOURCE_ATTRIBUTES_SOME_RESOURCE_ATTRIBUTE] - ), + http: { + response: { + status_code: normalizeValue(fields[HTTP_RESPONSE_STATUS_CODE]), }, - dropped_attributes_count: normalizeValue(fields[RESOURCE_DROPPED_ATTRIBUTES_COUNT]), - schema_url: normalizeValue(fields[RESOURCE_SCHEMA_URL]), - }, - attributes: { - 'event.outcome': normalizeValue(fields[ATTRIBUTES_EVENT_OUTCOME]), - 'event.success_count': normalizeValue(fields[ATTRIBUTES_EVENT_SUCCESS_COUNT]), - 'http.status_code': normalizeValue(fields[ATTRIBUTES_HTTP_STATUS_CODE]), - 'peer.service': normalizeValue(fields[ATTRIBUTES_PEER_SERVICE]), - 'processor.event': normalizeValue(fields[ATTRIBUTES_PROCESSOR_EVENT]), - 'service.target.name': normalizeValue(fields[ATTRIBUTES_SERVICE_TARGET_NAME]), - 'service.target.type': normalizeValue(fields[ATTRIBUTES_SERVICE_TARGET_TYPE]), - 'some.span.attribute': normalizeValue(fields[ATTRIBUTES_SOME_SPAN_ATTRIBUTE]), - 'span.destination.service.resource': normalizeValue( - fields[ATTRIBUTES_SPAN_DESTINATION_SERVICE_RESOURCE] - ), - 'span.duration.us': normalizeValue(fields[ATTRIBUTES_SPAN_DURATION_US]), - 'span.name': normalizeValue(fields[ATTRIBUTES_SPAN_NAME]), - 'span.representative_count': normalizeValue( - fields[ATTRIBUTES_SPAN_RESPSENTATIVE_COUNT] - ), - 'span.subtype': normalizeValue(fields[ATTRIBUTES_SPAN_SUBTYPE]), - 'span.type': normalizeValue(fields[ATTRIBUTES_SPAN_TYPE]), - 'timestamp.us': normalizeValue(fields[ATTRIBUTES_TIMESTAMP_US]), }, data_stream: { namespace: normalizeValue(fields[DATA_STREAM_NAMESPACE]), type: normalizeValue(fields[DATA_STEAM_TYPE]), dataset: normalizeValue(fields[DATA_STREAM_DATASET]), }, - dropped_attributes_count: normalizeValue(fields[DROPPED_ATTRIBUTES_COUNT]), - dropped_events_count: normalizeValue(fields[DROPPED_EVENTS_COUNT]), - dropped_links_count: normalizeValue(fields[DROPPED_LINKS_COUNT]), - duration: normalizeValue(fields[DURATION]), - kind: normalizeValue(fields[KIND]), - // links: // todo(milosz): handle arrays, what is the format in `fields` - name: normalizeValue(fields[NAME]), - parent_span_id: normalizeValue(fields[PARENT_SPAN_ID]), - span_id: normalizeValue(fields[OTEL_SPAN_ID]), - status: { - code: normalizeValue(fields[STATUS_CODE]), - }, - trace_id: normalizeValue(fields[OTEL_TRACE_ID]), }; }; @@ -393,17 +314,16 @@ export const spanLinksDetailsMapping = (fields: Partial>) => { return { span: { links: [ { trace: { - // todo(milosz): confirm `span.links.trace.id` format id: normalizeValue(fields[SPAN_LINKS_TRACE_ID]), }, span: { - // todo(milosz): confirm `span.links.span.id` format id: normalizeValue(fields[SPAN_LINKS_SPAN_ID]), }, }, From d467a38c2c7397ffcdf43e748a1328f711a584bd Mon Sep 17 00:00:00 2001 From: Milosz Marcinkowski Date: Thu, 26 Sep 2024 17:25:32 +0200 Subject: [PATCH 15/25] improve name consistency in mapping functions --- .../errors/get_error_groups/get_error_sample_details.ts | 4 ++-- .../apm/server/routes/services/get_service_agent.ts | 6 ++++-- .../services/get_service_instance_metadata_details.ts | 4 ++-- .../server/routes/services/get_service_metadata_icons.ts | 4 ++-- .../apm/server/utils/es_fields_mappings.ts | 8 ++++---- 5 files changed, 14 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts b/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts index d1e3a48349986..28f8a1d4573d1 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts @@ -14,7 +14,7 @@ import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm import { getTransaction } from '../../transactions/get_transaction'; import { Transaction } from '../../../../typings/es_schemas/ui/transaction'; import { APMError } from '../../../../typings/es_schemas/ui/apm_error'; -import { errorSampleDetails } from '../../../utils/es_fields_mappings'; +import { errorSampleDetailsMapping } from '../../../utils/es_fields_mappings'; export interface ErrorSampleDetailsResponse { transaction: Transaction | undefined; @@ -67,7 +67,7 @@ export async function getErrorSampleDetails({ const resp = await apmEventClient.search('get_error_sample_details', params); - const errorNorm = errorSampleDetails(resp.hits.hits[0]?.fields); + const errorNorm = errorSampleDetailsMapping(resp.hits.hits[0]?.fields); const transactionId = errorNorm?.transaction?.id; const traceId = errorNorm?.trace?.id; diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_agent.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_agent.ts index ed678a5c34748..5302e1dbb6481 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_agent.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_agent.ts @@ -7,7 +7,7 @@ import { rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; -import { serviceAgentName } from '../../utils/es_fields_mappings'; +import { serviceAgentNameMapping } from '../../utils/es_fields_mappings'; import { AGENT_NAME, SERVICE_NAME, @@ -102,7 +102,9 @@ export async function getServiceAgent({ return {}; } - const { agent, service, cloud } = serviceAgentName(response.hits.hits[0].fields) as ServiceAgent; + const { agent, service, cloud } = serviceAgentNameMapping( + response.hits.hits[0].fields + ) as ServiceAgent; const serverlessType = getServerlessTypeFromCloudData(cloud?.provider, cloud?.service?.name); return { diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_metadata_details.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_metadata_details.ts index 515f6ef5ea3a1..23491821aa9cf 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_metadata_details.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_metadata_details.ts @@ -10,7 +10,7 @@ import { ProcessorEvent } from '@kbn/observability-plugin/common'; import { metadataAppMetricMapping, metadataAppTransactionEventMapping, - metaDataAppTransactionMetric, + metaDataAppTransactionMetricMapping, } from '../../utils/es_fields_mappings'; import { METRICSET_NAME, SERVICE_NAME, SERVICE_NODE_NAME } from '../../../common/es_fields/apm'; import { maybe } from '../../../common/utils/maybe'; @@ -119,7 +119,7 @@ export async function getServiceInstanceMetadataDetails({ }, } ); - return maybe(metaDataAppTransactionMetric(response.hits.hits[0]?.fields)); + return maybe(metaDataAppTransactionMetricMapping(response.hits.hits[0]?.fields)); } // we can expect the most detail of application metrics, diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts index 0f83e4de974b5..8ba26cf5a55c5 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts @@ -7,7 +7,7 @@ import { rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; -import { serviceMetadataIcons } from '../../utils/es_fields_mappings'; +import { serviceMetadataIconsMapping } from '../../utils/es_fields_mappings'; import { AGENT_NAME, CLOUD_PROVIDER, @@ -86,7 +86,7 @@ export async function getServiceMetadataIcons({ }; } - const { kubernetes, cloud, container, agent } = serviceMetadataIcons( + const { kubernetes, cloud, container, agent } = serviceMetadataIconsMapping( response.hits.hits[0].fields ); diff --git a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts index 3af39f90fe0a5..71b3e9d9c20ed 100644 --- a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts +++ b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts @@ -509,7 +509,7 @@ export const errorGroupMainStatisticsMapping = (fields: Partial>): APMError => { +export const errorSampleDetailsMapping = (fields: Partial>): APMError => { return { observer: { type: normalizeValue(fields[OBSERVER_TYPE]), @@ -559,7 +559,7 @@ export const errorSampleDetails = (fields: Partial>): }; }; -export const serviceMetadataIcons = ( +export const serviceMetadataIconsMapping = ( fields: Partial> ): ServiceMetadataIconsRaw => { return { @@ -888,7 +888,7 @@ export const metadataAppTransactionEventMapping = (fields: Partial>) => { +export const metaDataAppTransactionMetricMapping = (fields: Partial>) => { return { '@timestamp': normalizeValue(fields[AT_TIMESTAMP]), _doc_count: normalizeValue(fields[DOC_COUNT]), @@ -963,7 +963,7 @@ export const metaDataAppTransactionMetric = (fields: Partial>) => { +export const serviceAgentNameMapping = (fields: Partial>) => { return { agent: { name: normalizeValue(fields[AGENT_NAME]), From ac07c730ad02c9c3e801e4325c5ad829e1b19766 Mon Sep 17 00:00:00 2001 From: Milosz Marcinkowski Date: Fri, 27 Sep 2024 13:20:19 +0200 Subject: [PATCH 16/25] remove unused fields from 'service instance metadata details' --- packages/kbn-apm-types/src/es_fields/apm.ts | 46 +-- .../apm/server/utils/es_fields_mappings.ts | 293 ------------------ 2 files changed, 1 insertion(+), 338 deletions(-) diff --git a/packages/kbn-apm-types/src/es_fields/apm.ts b/packages/kbn-apm-types/src/es_fields/apm.ts index 74718530c2fc5..eaf6e4e0282f8 100644 --- a/packages/kbn-apm-types/src/es_fields/apm.ts +++ b/packages/kbn-apm-types/src/es_fields/apm.ts @@ -46,26 +46,9 @@ export const SERVICE_TARGET_NAME = 'service.target.name'; export const SERVICE_OVERFLOW_COUNT = 'service_transaction.aggregation.overflow_count'; export const URL_FULL = 'url.full'; -export const URL_PATH = 'url.path'; -export const URL_ORIGINAL = 'url.original'; -export const URL_SCHEME = 'url.scheme'; -export const URL_PORT = 'url.port'; -export const URL_DOMAIN = 'url.domain'; export const HTTP_REQUEST_METHOD = 'http.request.method'; export const HTTP_RESPONSE_STATUS_CODE = 'http.response.status_code'; -export const HTTP_RESPONSE_HEADERS_TRANSFER_ENCODING = 'http.response.headers.Transfer-Encoding'; -export const HTTP_RESPONSE_HEADERS_CONNECTION = 'http.response.headers.Connection'; -export const HTTP_RESPONSE_HEADERS_DATE = 'http.response.headers.Date'; -export const HTTP_RESPONSE_HEADERS_X_POWERED_BY = 'http.response.headers.X-Powered-By'; -export const HTTP_VERSION = 'http.version'; -export const HTTP_REQUEST_HEADERS_ACCEPT = 'http.request.headers.Accept'; -export const HTTP_REQUEST_HEADERS_CONNECTION = 'http.request.headers.Connection'; -export const HTTP_REQUEST_HEADERS_USER_AGENT = 'http.request.headers.User-Agent'; -export const HTTP_REQUEST_HEADERS_HOST = 'http.request.headers.Host'; -export const HTTP_REQUEST_HEADERS_ELASTIC_APM_TRACEPARENT = - 'http.request.headers.Elastic-Apm-Traceparent'; -export const HTTP_REQUEST_HEADERS_TRACESTATE = 'http.request.headers.Tracestate'; -export const HTTP_REQUEST_HEADERS_TRACEPARENT = 'http.request.headers.Traceparent'; + export const USER_ID = 'user.id'; export const USER_AGENT_ORIGINAL = 'user_agent.original'; export const USER_AGENT_NAME = 'user_agent.name'; @@ -82,10 +65,7 @@ export const PROCESOR_NAME = 'processor.name'; export const TRANSACTION_DURATION = 'transaction.duration.us'; export const TRANSACTION_DURATION_HISTOGRAM = 'transaction.duration.histogram'; -export const TRANSACTION_DURATION_HISTOGRAM_VALUES = 'transaction.duration.histogram.values'; export const TRANSACTION_DURATION_SUMMARY = 'transaction.duration.summary'; -export const TRANSACTION_DURATION_SUMMARY_SUM = 'transaction.duration.summary.sum'; -export const TRANSACTION_DURATION_SUMMARY_VALUE_COUNT = 'transaction.duration.summary.count'; export const TRANSACTION_TYPE = 'transaction.type'; export const TRANSACTION_RESULT = 'transaction.result'; export const TRANSACTION_NAME = 'transaction.name'; @@ -152,11 +132,6 @@ export const METRIC_SYSTEM_FREE_MEMORY = 'system.memory.actual.free'; export const METRIC_SYSTEM_TOTAL_MEMORY = 'system.memory.total'; export const METRIC_SYSTEM_CPU_PERCENT = 'system.cpu.total.norm.pct'; export const METRIC_PROCESS_CPU_PERCENT = 'system.process.cpu.total.norm.pct'; -export const METRIC_PROCESS_CPU_SYSTEM_NORM_PCT = 'system.process.cpu.system.norm.pct'; -export const METRIC_PROCESS_CPU_TOTAL_NORM_PCT = 'system.process.cpu.total.norm.pct'; -export const METRIC_PROCESS_CPU_TOTAL_USER_PCT = 'system.process.cpu.user.norm.pct'; -export const METRIC_PROCESS_MEMORY_RSS_BYTES = 'system.process.memory.rss.bytes'; -export const METRIC_PROCESS_MEMORY_SIZE = 'system.process.memory.size'; export const METRIC_CGROUP_MEMORY_LIMIT_BYTES = 'system.process.cgroup.memory.mem.limit.bytes'; export const METRIC_CGROUP_MEMORY_USAGE_BYTES = 'system.process.cgroup.memory.mem.usage.bytes'; @@ -179,8 +154,6 @@ export const LABEL_TYPE = 'labels.type'; export const LABEL_TELEMETRY_AUTO_VERSION = 'labels.telemetry_auto_version'; export const LABEL_LIFECYCLE_STATE = 'labels.lifecycle_state'; export const LABEL_SOME_RESOURCE_ATTRIBUTE = 'labels.some_resource_attribute'; -export const LABEL_ENV = 'labels.env'; -export const LABEL_HOSTNAME = 'labels.hostname'; export const HOST = 'host'; export const HOST_HOSTNAME = 'host.hostname'; // Do not use. Please use `HOST_NAME` instead. @@ -247,20 +220,3 @@ export const CLIENT_IP = 'client.ip'; export const CHILD_ID = 'child.id'; export const LOG_LEVEL = 'log.level'; - -export const NODEJS_EVENTLOOP_DELAY_AVG_MS = 'nodejs.eventloop.delay.avg.ms'; -export const NODEJS_HANDLES_ACTIVE = 'nodejs.handles.active'; -export const NODEJS_MEMORY_ARRAYBUFFERS_BYTES = 'nodejs.memory.arrayBuffers.bytes'; -export const NODEJS_MEMORY_EXTERNAL_BYTES = 'nodejs.memory.external.bytes'; -export const NODEJS_MEMORY_HEAP_ALLOCATED_BYTES = 'nodejs.memory.heap.allocated.bytes'; -export const NODEJS_MEMORY_HEAP_USED_BYTES = 'nodejs.memory.heap.used.bytes'; -export const NODEJS_REQUESTS_ACTIVE = 'nodejs.requests.active'; - -export const PROCESS_ARGS = 'process.args'; -export const PROCESS_PARENT_PID = 'process.parent.pid'; -export const PROCESS_PID = 'process.pid'; -export const PROCESS_TITLE = 'process.title'; - -export const SOURCE_IP = 'source.ip'; - -export const DOC_COUNT = '_doc_count'; diff --git a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts index 71b3e9d9c20ed..5b8eda615a2b0 100644 --- a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts +++ b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts @@ -70,74 +70,20 @@ import { CONTAINER, KUBERNETES, AGENT_ACTIVATION_METHOD, - CLIENT_IP, - DOC_COUNT, - EVENT_SUCCESS_COUNT_SUM, - EVENT_SUCCESS_COUNT_VALUE_COUNT, HOST_ARCHITECTURE, HOST_HOSTNAME, HOST_IP, HOST_OS_PLATFORM, - HTTP_REQUEST_HEADERS_ACCEPT, - HTTP_REQUEST_HEADERS_CONNECTION, - HTTP_REQUEST_HEADERS_ELASTIC_APM_TRACEPARENT, - HTTP_REQUEST_HEADERS_HOST, - HTTP_REQUEST_HEADERS_TRACEPARENT, - HTTP_REQUEST_HEADERS_TRACESTATE, - HTTP_REQUEST_HEADERS_USER_AGENT, - HTTP_REQUEST_METHOD, - HTTP_RESPONSE_HEADERS_CONNECTION, - HTTP_RESPONSE_HEADERS_DATE, - HTTP_RESPONSE_HEADERS_TRANSFER_ENCODING, - HTTP_RESPONSE_HEADERS_X_POWERED_BY, HTTP_RESPONSE_STATUS_CODE, - HTTP_VERSION, KUBERNETES_NAMESPACE, KUBERNETES_NODE_NAME, KUBERNETES_POD_NAME, KUBERNETES_POD_UID, - LABEL_ENV, - LABEL_HOSTNAME, - METRIC_PROCESS_CPU_SYSTEM_NORM_PCT, - METRIC_PROCESS_CPU_TOTAL_NORM_PCT, - METRIC_PROCESS_CPU_TOTAL_USER_PCT, - METRIC_PROCESS_MEMORY_RSS_BYTES, - METRIC_PROCESS_MEMORY_SIZE, - METRIC_SYSTEM_CPU_PERCENT, - METRIC_SYSTEM_FREE_MEMORY, - METRIC_SYSTEM_TOTAL_MEMORY, - METRICSET_INTERVAL, - METRICSET_NAME, - NODEJS_EVENTLOOP_DELAY_AVG_MS, - NODEJS_HANDLES_ACTIVE, - NODEJS_MEMORY_ARRAYBUFFERS_BYTES, - NODEJS_MEMORY_EXTERNAL_BYTES, - NODEJS_MEMORY_HEAP_ALLOCATED_BYTES, - NODEJS_MEMORY_HEAP_USED_BYTES, - NODEJS_REQUESTS_ACTIVE, - PROCESS_ARGS, - PROCESS_PARENT_PID, - PROCESS_PID, - PROCESS_TITLE, SERVICE_FRAMEWORK_VERSION, SERVICE_LANGUAGE_NAME, SERVICE_RUNTIME_NAME, SERVICE_RUNTIME_VERSION, SERVICE_VERSION, - SOURCE_IP, - TRANSACTION_DURATION_HISTOGRAM_VALUES, - TRANSACTION_DURATION_SUMMARY_SUM, - TRANSACTION_DURATION_SUMMARY_VALUE_COUNT, - TRANSACTION_SPAN_COUNT_STARTED, - URL_DOMAIN, - URL_ORIGINAL, - URL_PATH, - URL_PORT, - URL_SCHEME, - USER_AGENT_DEVICE_NAME, - USER_AGENT_NAME, - USER_AGENT_ORIGINAL, - USER_AGENT_VERSION, SERVICE_TARGET_NAME, SERVICE_TARGET_TYPE, SPAN_REPRESENTATIVE_COUNT, @@ -598,11 +544,6 @@ export const metadataAppMetricMapping = (fields: Partial(fields[AGENT_VERSION]), activation_method: normalizeValue(fields[AGENT_ACTIVATION_METHOD]), }, - data_stream: { - namespace: normalizeValue(fields[DATA_STREAM_NAMESPACE]), - type: normalizeValue(fields[DATA_STEAM_TYPE]), - dataset: normalizeValue(fields[DATA_STREAM_DATASET]), - }, host: { architecture: normalizeValue(fields[HOST_ARCHITECTURE]), hostname: normalizeValue(fields[HOST_HOSTNAME]), @@ -622,60 +563,6 @@ export const metadataAppMetricMapping = (fields: Partial(fields[KUBERNETES_POD_UID]), }, }, - labels: { - env: normalizeValue(fields[LABEL_ENV]), - hostname: normalizeValue(fields[LABEL_HOSTNAME]), - }, - metricset: { - name: normalizeValue(fields[METRICSET_NAME]), - }, - nodejs: { - eventloop: { - delay: { - avg: { - ms: normalizeValue(fields[NODEJS_EVENTLOOP_DELAY_AVG_MS]), - }, - }, - }, - handles: { - active: normalizeValue(fields[NODEJS_HANDLES_ACTIVE]), - }, - memory: { - arrayBuffers: { - bytes: normalizeValue(fields[NODEJS_MEMORY_ARRAYBUFFERS_BYTES]), - }, - external: { - bytes: normalizeValue(fields[NODEJS_MEMORY_EXTERNAL_BYTES]), - }, - heap: { - allocated: { - bytes: normalizeValue(fields[NODEJS_MEMORY_HEAP_ALLOCATED_BYTES]), - }, - used: { - bytes: normalizeValue(fields[NODEJS_MEMORY_HEAP_USED_BYTES]), - }, - }, - }, - requests: { - active: normalizeValue(fields[NODEJS_REQUESTS_ACTIVE]), - }, - }, - observer: { - hostname: normalizeValue(fields[OBSERVER_HOSTNAME]), - type: normalizeValue(fields[OBSERVER_TYPE]), - version: normalizeValue(fields[OBSERVER_VERSION]), - }, - process: { - args: fields[PROCESS_ARGS] as string[] | undefined, - parent: { - pid: normalizeValue(fields[PROCESS_PARENT_PID]), - }, - pid: normalizeValue(fields[PROCESS_PID]), - title: normalizeValue(fields[PROCESS_TITLE]), - }, - processor: { - event: normalizeValue(fields[PROCESSOR_EVENT]), - }, service: { name: normalizeValue(fields[SERVICE_NAME]), environment: normalizeValue(fields[SERVICE_ENVIRONMENT]), @@ -695,46 +582,6 @@ export const metadataAppMetricMapping = (fields: Partial(fields[SERVICE_VERSION]), }, - system: { - cpu: { - total: { - norm: { - pct: normalizeValue(fields[METRIC_SYSTEM_CPU_PERCENT]), - }, - }, - }, - memory: { - actual: { - free: normalizeValue(fields[METRIC_SYSTEM_FREE_MEMORY]), - }, - total: normalizeValue(fields[METRIC_SYSTEM_TOTAL_MEMORY]), - }, - process: { - cpu: { - system: { - norm: { - pct: normalizeValue(fields[METRIC_PROCESS_CPU_SYSTEM_NORM_PCT]), - }, - }, - total: { - norm: { - pct: normalizeValue(fields[METRIC_PROCESS_CPU_TOTAL_NORM_PCT]), - }, - }, - user: { - norm: { - pct: normalizeValue(fields[METRIC_PROCESS_CPU_TOTAL_USER_PCT]), - }, - }, - }, - memory: { - rss: { - bytes: normalizeValue(fields[METRIC_PROCESS_MEMORY_RSS_BYTES]), - }, - size: normalizeValue(fields[METRIC_PROCESS_MEMORY_SIZE]), - }, - }, - }, }; }; @@ -750,50 +597,12 @@ export const metadataAppTransactionEventMapping = (fields: Partial(fields[KUBERNETES_POD_UID]), }, }, - parent: { - id: normalizeValue(fields[PARENT_ID]), - }, agent: { name: normalizeValue(fields[AGENT_NAME]), version: normalizeValue(fields[AGENT_VERSION]), activation_method: normalizeValue(fields[AGENT_ACTIVATION_METHOD]), }, - process: { - args: fields[PROCESS_ARGS] as string[] | undefined, - parent: { - pid: normalizeValue(fields[PROCESS_PARENT_PID]), - }, - pid: normalizeValue(fields[PROCESS_PID]), - title: normalizeValue(fields[PROCESS_TITLE]), - }, - source: { - ip: normalizeValue(fields[SOURCE_IP]), - }, - processor: { - event: normalizeValue(fields[PROCESSOR_EVENT]), - }, - url: { - path: normalizeValue(fields[URL_PATH]), - original: normalizeValue(fields[URL_ORIGINAL]), - scheme: normalizeValue(fields[URL_SCHEME]), - port: normalizeValue(fields[URL_PORT]), - domain: normalizeValue(fields[URL_DOMAIN]), - full: normalizeValue(fields[SOURCE_IP]), - }, - observer: { - hostname: normalizeValue(fields[OBSERVER_HOSTNAME]), - type: normalizeValue(fields[OBSERVER_TYPE]), - version: normalizeValue(fields[OBSERVER_VERSION]), - }, - trace: { - id: normalizeValue(fields[TRACE_ID]), - }, '@timestamp': normalizeValue(fields[AT_TIMESTAMP]), - data_stream: { - namespace: normalizeValue(fields[DATA_STREAM_NAMESPACE]), - type: normalizeValue(fields[DATA_STEAM_TYPE]), - dataset: normalizeValue(fields[DATA_STREAM_DATASET]), - }, service: { name: normalizeValue(fields[SERVICE_NAME]), environment: normalizeValue(fields[SERVICE_ENVIRONMENT]), @@ -822,91 +631,15 @@ export const metadataAppTransactionEventMapping = (fields: Partial(fields[HOST_OS_PLATFORM]), }, }, - client: { - ip: normalizeValue(fields[CLIENT_IP]), - }, - http: { - request: { - headers: { - Accept: fields[HTTP_REQUEST_HEADERS_ACCEPT] as string[] | undefined, - Connection: fields[HTTP_REQUEST_HEADERS_CONNECTION] as string[] | undefined, - 'User-Agent': fields[HTTP_REQUEST_HEADERS_USER_AGENT] as string[] | undefined, - Host: fields[HTTP_REQUEST_HEADERS_HOST] as string[] | undefined, - 'Elastic-Apm-Traceparent': fields[HTTP_REQUEST_HEADERS_ELASTIC_APM_TRACEPARENT] as - | string[] - | undefined, - Tracestate: fields[HTTP_REQUEST_HEADERS_TRACESTATE] as string[] | undefined, - Traceparent: fields[HTTP_REQUEST_HEADERS_TRACEPARENT] as string[] | undefined, - }, - method: normalizeValue(fields[HTTP_REQUEST_METHOD]), - }, - response: { - headers: { - 'Transfer-Encoding': fields[HTTP_RESPONSE_HEADERS_TRANSFER_ENCODING] as - | string - | undefined, - Connection: fields[HTTP_RESPONSE_HEADERS_CONNECTION] as string | undefined, - Date: fields[HTTP_RESPONSE_HEADERS_DATE] as string | undefined, - 'X-Powered-By': fields[HTTP_RESPONSE_HEADERS_X_POWERED_BY] as string | undefined, - }, - status_code: normalizeValue(fields[HTTP_RESPONSE_STATUS_CODE]), - }, - version: normalizeValue(fields[HTTP_VERSION]), - }, - event: { - success_count: normalizeValue(fields[EVENT_SUCCESS_COUNT]), - outcome: normalizeValue(fields[EVENT_OUTCOME]), - }, - transaction: { - result: normalizeValue(fields[TRANSACTION_RESULT]), - representative_count: normalizeValue(fields[TRANSACTION_REPRESENTATIVE_COUNT]), - sampled: normalizeValue(fields[TRANSACTION_SAMPLED]), - id: normalizeValue(fields[TRANSACTION_ID]), - duration: { - us: normalizeValue(fields[TRANSACTION_DURATION]), - }, - type: normalizeValue(fields[TRANSACTION_TYPE]), - name: normalizeValue(fields[TRANSACTION_NAME]), - span_count: { - started: normalizeValue(fields[TRANSACTION_SPAN_COUNT_STARTED]), - }, - }, - user_agent: { - original: normalizeValue(fields[USER_AGENT_ORIGINAL]), - name: normalizeValue(fields[USER_AGENT_NAME]), - device: { - name: normalizeValue(fields[USER_AGENT_DEVICE_NAME]), - }, - version: normalizeValue(fields[USER_AGENT_VERSION]), - }, - span: { - id: normalizeValue(fields[SPAN_ID]), - }, - timestamp: { - us: normalizeValue(fields[TIMESTAMP]), - }, }; }; export const metaDataAppTransactionMetricMapping = (fields: Partial>) => { return { '@timestamp': normalizeValue(fields[AT_TIMESTAMP]), - _doc_count: normalizeValue(fields[DOC_COUNT]), agent: { name: normalizeValue(fields[AGENT_NAME]), }, - data_stream: { - namespace: normalizeValue(fields[DATA_STREAM_NAMESPACE]), - type: normalizeValue(fields[DATA_STEAM_TYPE]), - dataset: normalizeValue(fields[DATA_STREAM_DATASET]), - }, - event: { - outcome: normalizeValue(fields[EVENT_OUTCOME]), - success_count: { - sum: normalizeValue(fields[EVENT_SUCCESS_COUNT_SUM]), - value_count: normalizeValue(fields[EVENT_SUCCESS_COUNT_VALUE_COUNT]), - }, - }, host: { hostname: normalizeValue(fields[HOST_HOSTNAME]), name: normalizeValue(fields[HOST_NAME]), @@ -919,18 +652,6 @@ export const metaDataAppTransactionMetricMapping = (fields: Partial(fields[KUBERNETES_POD_NAME]), }, }, - metricset: { - name: normalizeValue(fields[METRICSET_NAME]), - interval: normalizeValue(fields[METRICSET_INTERVAL]), - }, - observer: { - hostname: normalizeValue(fields[OBSERVER_HOSTNAME]), - type: normalizeValue(fields[OBSERVER_TYPE]), - version: normalizeValue(fields[OBSERVER_VERSION]), - }, - processor: { - event: normalizeValue(fields[PROCESSOR_EVENT]), - }, service: { name: normalizeValue(fields[SERVICE_NAME]), environment: normalizeValue(fields[SERVICE_ENVIRONMENT]), @@ -946,20 +667,6 @@ export const metaDataAppTransactionMetricMapping = (fields: Partial(fields[SERVICE_VERSION]), }, - transaction: { - duration: { - histogram: { - values: fields[TRANSACTION_DURATION_HISTOGRAM_VALUES] as number[] | undefined, - }, - summary: { - sum: normalizeValue(fields[TRANSACTION_DURATION_SUMMARY_SUM]), - value_count: normalizeValue(fields[TRANSACTION_DURATION_SUMMARY_VALUE_COUNT]), - }, - }, - name: normalizeValue(fields[TRANSACTION_NAME]), - result: normalizeValue(fields[TRANSACTION_RESULT]), - type: normalizeValue(fields[TRANSACTION_TYPE]), - }, }; }; From 8666c006dce19e2cd195d69d30989f237b7b9190 Mon Sep 17 00:00:00 2001 From: Milosz Marcinkowski Date: Fri, 27 Sep 2024 17:34:24 +0200 Subject: [PATCH 17/25] add check against undefined --- .../apm/server/utils/es_fields_mappings.ts | 49 +++++++++++++++++-- 1 file changed, 44 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts index 5b8eda615a2b0..b8b57b5f36b02 100644 --- a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts +++ b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts @@ -105,6 +105,8 @@ const normalizeValue = (field: unknown[] | unknown): T => { }; export const metadataForDependencyMapping = (fields: Partial>) => { + if (!fields) return undefined; + return { span: { type: normalizeValue(fields[SPAN_TYPE]), @@ -116,6 +118,8 @@ export const metadataForDependencyMapping = (fields: Partial> ) => { + if (!fields) return undefined; + return { transaction: { id: normalizeValue(fields[TRANSACTION_ID]), @@ -126,6 +130,8 @@ export const transactionsForDependencySpansMapping = ( }; export const topDependencySpansMapping = (fields: Partial>) => { + if (!fields) return undefined; + return { transaction: { id: normalizeValue(fields[TRANSACTION_ID]), @@ -155,6 +161,8 @@ export const topDependencySpansMapping = (fields: Partial>) => { + if (!fields) return undefined; + return { parent: { id: normalizeValue(fields[PARENT_ID]), @@ -227,6 +235,8 @@ export const spanMapping = (fields: Partial>) => { }; export const spanLinksDetailsMapping = (fields: Partial>) => { + if (!fields) return undefined; + return { trace: { id: normalizeValue(fields[TRACE_ID]), @@ -262,6 +272,8 @@ export const spanLinksDetailsMapping = (fields: Partial>) => { + if (!fields) return undefined; + return { span: { links: [ @@ -279,6 +291,8 @@ export const linkedParentsOfSpanMapping = (fields: Partial>) => { + if (!fields) return undefined; + return { transaction: { representative_count: normalizeValue(fields[TRANSACTION_REPRESENTATIVE_COUNT]), @@ -342,7 +356,9 @@ export const transactionMapping = (fields: Partial>) = export const traceDocMapping = ( fields: Partial> -): WaterfallTransaction | WaterfallSpan => { +): WaterfallTransaction | WaterfallSpan | undefined => { + if (!fields) return undefined; + return { timestamp: { us: normalizeValue(fields[TIMESTAMP]), @@ -403,7 +419,11 @@ export const traceDocMapping = ( }; }; -export const errorDocsMapping = (fields: Partial>): WaterfallError => { +export const errorDocsMapping = ( + fields: Partial> +): WaterfallError | undefined => { + if (!fields) return undefined; + return { timestamp: { us: normalizeValue(fields[TIMESTAMP]), @@ -432,6 +452,8 @@ export const errorDocsMapping = (fields: Partial>): Wa }; export const errorGroupMainStatisticsMapping = (fields: Partial>) => { + if (!fields) return undefined; + return { trace: { id: normalizeValue(fields[TRACE_ID]), @@ -455,7 +477,11 @@ export const errorGroupMainStatisticsMapping = (fields: Partial>): APMError => { +export const errorSampleDetailsMapping = ( + fields: Partial> +): APMError | undefined => { + if (!fields) return undefined; + return { observer: { type: normalizeValue(fields[OBSERVER_TYPE]), @@ -507,7 +533,9 @@ export const errorSampleDetailsMapping = (fields: Partial> -): ServiceMetadataIconsRaw => { +): ServiceMetadataIconsRaw | undefined => { + if (!fields) return undefined; + return { agent: { name: normalizeValue(fields[AGENT_NAME]), @@ -527,7 +555,9 @@ export const serviceMetadataIconsMapping = ( // todo(milosz): test it export const serviceVersionMapping = ( fields: Partial> -): Pick => { +): Pick | undefined => { + if (!fields) return undefined; + return { observer: { version: normalizeValue(fields[OBSERVER_VERSION]), @@ -537,6 +567,8 @@ export const serviceVersionMapping = ( }; export const metadataAppMetricMapping = (fields: Partial>) => { + if (!fields) return undefined; + return { '@timestamp': normalizeValue(fields[AT_TIMESTAMP]), agent: { @@ -586,6 +618,8 @@ export const metadataAppMetricMapping = (fields: Partial>) => { + if (!fields) return undefined; + return { kubernetes: { namespace: normalizeValue(fields[KUBERNETES_NAMESPACE]), @@ -635,6 +669,8 @@ export const metadataAppTransactionEventMapping = (fields: Partial>) => { + if (!fields) return undefined; + return { '@timestamp': normalizeValue(fields[AT_TIMESTAMP]), agent: { @@ -670,7 +706,10 @@ export const metaDataAppTransactionMetricMapping = (fields: Partial>) => { + if (!fields) return undefined; + return { agent: { name: normalizeValue(fields[AGENT_NAME]), From 06545b5766fec14b12b4f445127f88168e149cd9 Mon Sep 17 00:00:00 2001 From: Bryce Buchanan Date: Wed, 25 Sep 2024 12:47:21 -0700 Subject: [PATCH 18/25] added span-link normalization and fixed errors in getTransaction --- packages/kbn-apm-types/src/es_fields/apm.ts | 6 +- .../apm/common/correlations/types.ts | 1 - .../apm/common/es_fields/infra_metrics.ts | 1 + .../get_error_sample_details.ts | 131 +++++++++++++++++- .../transactions/get_transaction/index.ts | 58 +++++++- .../apm/server/utils/es_fields_mappings.ts | 38 +++-- 6 files changed, 208 insertions(+), 27 deletions(-) diff --git a/packages/kbn-apm-types/src/es_fields/apm.ts b/packages/kbn-apm-types/src/es_fields/apm.ts index eaf6e4e0282f8..ea5e89f91d71f 100644 --- a/packages/kbn-apm-types/src/es_fields/apm.ts +++ b/packages/kbn-apm-types/src/es_fields/apm.ts @@ -56,12 +56,14 @@ export const USER_AGENT_DEVICE_NAME = 'user_agent.device.name'; export const USER_AGENT_VERSION = 'user_agent.version'; export const OBSERVER_HOSTNAME = 'observer.hostname'; -export const OBSERVER_LISTENING = 'observer.listening'; +export const OBSERVER_ID = 'observer.id'; export const OBSERVER_TYPE = 'observer.type'; +export const OBSERVER_NAME = 'observer.name'; export const OBSERVER_VERSION = 'observer.version'; +export const OBSERVER_LISTENING = 'observer.listening'; export const OBSERVER_VERSION_MAJOR = 'observer.version_major'; export const PROCESSOR_EVENT = 'processor.event'; -export const PROCESOR_NAME = 'processor.name'; +export const PROCESSOR_NAME = 'processor.name'; export const TRANSACTION_DURATION = 'transaction.duration.us'; export const TRANSACTION_DURATION_HISTOGRAM = 'transaction.duration.histogram'; diff --git a/x-pack/plugins/observability_solution/apm/common/correlations/types.ts b/x-pack/plugins/observability_solution/apm/common/correlations/types.ts index 6555e8a5d953d..08b345b2461d5 100644 --- a/x-pack/plugins/observability_solution/apm/common/correlations/types.ts +++ b/x-pack/plugins/observability_solution/apm/common/correlations/types.ts @@ -27,7 +27,6 @@ export interface ResponseHitSource { } export interface ResponseHit { - _source: ResponseHitSource; fields: ResponseHitSource; } diff --git a/x-pack/plugins/observability_solution/apm/common/es_fields/infra_metrics.ts b/x-pack/plugins/observability_solution/apm/common/es_fields/infra_metrics.ts index fbf09e1887dfb..bfd8d6aa91f9b 100644 --- a/x-pack/plugins/observability_solution/apm/common/es_fields/infra_metrics.ts +++ b/x-pack/plugins/observability_solution/apm/common/es_fields/infra_metrics.ts @@ -7,6 +7,7 @@ // Kubernetes export const KUBERNETES_CONTAINER_NAME = 'kubernetes.container.name'; +export const KUBERNETES_CONTAINER_ID = 'kubernetes.container.id'; export const KUBERNETES_DEPLOYMENT = 'kubernetes.deployment'; export const KUBERNETES_DEPLOYMENT_NAME = 'kubernetes.deployment.name'; export const KUBERNETES_NAMESPACE_NAME = 'kubernetes.namespace.name'; diff --git a/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts b/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts index 28f8a1d4573d1..f008f8d6e44fb 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts @@ -6,7 +6,61 @@ */ import { rangeQuery, kqlQuery } from '@kbn/observability-plugin/server'; -import { ERROR_ID, SERVICE_NAME } from '../../../../common/es_fields/apm'; +import { OBSERVER_VERSION_MAJOR } from '@kbn/observability-shared-plugin/common'; +import { + ERROR_EXCEPTION_STACKTRACE, + ERROR_LOG_STACKTRACE, + ERROR_STACK_TRACE, +} from '@kbn/discover-utils/src/field_constants'; +import { + AGENT_NAME, + AGENT_VERSION, + CONTAINER_ID, + CONTAINER_IMAGE, + ERROR_ID, + HOST_ARCHITECTURE, + HOST_HOSTNAME, + HOST_NAME, + HTTP_RESPONSE_STATUS_CODE, + KUBERNETES_POD_UID, + OBSERVER_HOSTNAME, + OBSERVER_ID, + OBSERVER_TYPE, + OBSERVER_NAME, + OBSERVER_VERSION, + PARENT_ID, + PROCESSOR_EVENT, + PROCESSOR_NAME, + SERVICE_NAME, + TRACE_ID, + SERVICE_ENVIRONMENT, + SERVICE_FRAMEWORK_NAME, + SERVICE_FRAMEWORK_VERSION, + SERVICE_NODE_NAME, + SERVICE_RUNTIME_NAME, + SERVICE_RUNTIME_VERSION, + SERVICE_LANGUAGE_NAME, + SERVICE_LANGUAGE_VERSION, + SERVICE_VERSION, + HOST_OS_PLATFORM, + TRANSACTION_ID, + TRANSACTION_SAMPLED, + TRANSACTION_TYPE, + ERROR_CULPRIT, + ERROR_PAGE_URL, + ERROR_LOG_MESSAGE, + URL_FULL, + USER_ID, + PROCESS_PID, + PROCESS_ARGS, + PROCESS_TITLE, + AT_TIMESTAMP, + ERROR_EXC_HANDLED, + ERROR_EXC_TYPE, + ERROR_EXC_MESSAGE, + URL_DOMAIN, + URL_ORIGINAL, +} from '../../../../common/es_fields/apm'; import { environmentQuery } from '../../../../common/utils/environment_query'; import { ApmDocumentType } from '../../../../common/document_type'; import { RollupInterval } from '../../../../common/rollup'; @@ -15,6 +69,13 @@ import { getTransaction } from '../../transactions/get_transaction'; import { Transaction } from '../../../../typings/es_schemas/ui/transaction'; import { APMError } from '../../../../typings/es_schemas/ui/apm_error'; import { errorSampleDetailsMapping } from '../../../utils/es_fields_mappings'; +import { + KUBERNETES_CONTAINER_NAME, + KUBERNETES_DEPLOYMENT_NAME, + KUBERNETES_NAMESPACE, + KUBERNETES_REPLICASET_NAME, + KUBERNETES_CONTAINER_ID, +} from '../../../../common/es_fields/infra_metrics'; export interface ErrorSampleDetailsResponse { transaction: Transaction | undefined; @@ -61,7 +122,73 @@ export async function getErrorSampleDetails({ ], }, }, - fields: ['*'], + fields: [ + AGENT_VERSION, + AGENT_NAME, + PARENT_ID, + TRACE_ID, + // AGENT_EPHEMERAL_ID, + // OBSERVER_EPHEMERAL_ID, + OBSERVER_HOSTNAME, + OBSERVER_ID, + OBSERVER_TYPE, + OBSERVER_NAME, + OBSERVER_VERSION, + OBSERVER_VERSION_MAJOR, + CONTAINER_ID, + CONTAINER_IMAGE, + PROCESSOR_NAME, + PROCESSOR_EVENT, + HOST_ARCHITECTURE, + HOST_HOSTNAME, + HOST_NAME, + 'host.ip', + 'http.request.method', + HTTP_RESPONSE_STATUS_CODE, + 'http.version', + KUBERNETES_POD_UID, + KUBERNETES_NAMESPACE, + KUBERNETES_REPLICASET_NAME, + KUBERNETES_DEPLOYMENT_NAME, + KUBERNETES_CONTAINER_ID, + KUBERNETES_CONTAINER_NAME, + SERVICE_NAME, + SERVICE_ENVIRONMENT, + SERVICE_FRAMEWORK_NAME, + SERVICE_FRAMEWORK_VERSION, + SERVICE_NODE_NAME, + SERVICE_RUNTIME_NAME, + SERVICE_RUNTIME_VERSION, + SERVICE_LANGUAGE_NAME, + SERVICE_LANGUAGE_VERSION, + SERVICE_VERSION, + PROCESS_ARGS, + PROCESS_PID, + PROCESS_TITLE, + HOST_OS_PLATFORM, + AT_TIMESTAMP, + TRANSACTION_ID, + TRANSACTION_SAMPLED, + TRANSACTION_TYPE, + ERROR_ID, + ERROR_CULPRIT, + 'error.exception.attributes.response', + // ERROR_EXC_ATTRIBUTES_RESPONSE, + // ERROR_EXC_CODE, + ERROR_EXC_MESSAGE, + ERROR_EXC_TYPE, + // ERROR_EXCEPTION_MODULE, + ERROR_EXC_HANDLED, + ERROR_EXCEPTION_STACKTRACE, // todo: fix me + ERROR_PAGE_URL, + ERROR_LOG_MESSAGE, + ERROR_LOG_STACKTRACE, // todo: fixme + ERROR_STACK_TRACE, + URL_DOMAIN, + URL_FULL, + URL_ORIGINAL, + USER_ID, + ], }, }; diff --git a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts index e5f08762f6d6b..b3ce38f1df4a3 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts @@ -7,7 +7,34 @@ import { rangeQuery, termQuery } from '@kbn/observability-plugin/server'; import { transactionMapping } from '../../../utils/es_fields_mappings'; -import { TRACE_ID, TRANSACTION_ID } from '../../../../common/es_fields/apm'; +import { + AGENT_NAME, + AGENT_VERSION, + DATA_STEAM_TYPE, + DATA_STREAM_DATASET, + DATA_STREAM_NAMESPACE, + EVENT_OUTCOME, + EVENT_SUCCESS_COUNT, + LABEL_SOME_RESOURCE_ATTRIBUTE, + OBSERVER_HOSTNAME, + OBSERVER_TYPE, + OBSERVER_VERSION, + PROCESSOR_EVENT, + SERVICE_FRAMEWORK_NAME, + SERVICE_NAME, + SERVICE_NODE_NAME, + TIMESTAMP, + TRACE_ID, + TRANSACTION_DURATION, + TRANSACTION_ID, + TRANSACTION_NAME, + TRANSACTION_RESULT, + TRANSACTION_SAMPLED, + TRANSACTION_TYPE, + SPAN_ID, + AT_TIMESTAMP, + TRANSACTION_REPRESENTATIVE_COUNT, +} from '../../../../common/es_fields/apm'; import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client'; import { ApmDocumentType } from '../../../../common/document_type'; @@ -48,7 +75,34 @@ export async function getTransaction({ ]), }, }, - fields: ['*'], + fields: [ + TRANSACTION_REPRESENTATIVE_COUNT, + TRANSACTION_RESULT, + TRANSACTION_SAMPLED, + TRANSACTION_ID, + TRANSACTION_DURATION, + TRANSACTION_TYPE, + TRANSACTION_NAME, + SERVICE_NODE_NAME, + SERVICE_NAME, + SERVICE_FRAMEWORK_NAME, + TRACE_ID, + AGENT_NAME, + AGENT_VERSION, + EVENT_SUCCESS_COUNT, + EVENT_OUTCOME, + PROCESSOR_EVENT, + DATA_STREAM_NAMESPACE, + DATA_STEAM_TYPE, + DATA_STREAM_DATASET, + SPAN_ID, + OBSERVER_HOSTNAME, + OBSERVER_TYPE, + OBSERVER_VERSION, + TIMESTAMP, + AT_TIMESTAMP, + LABEL_SOME_RESOURCE_ATTRIBUTE, + ], }, }); diff --git a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts index b8b57b5f36b02..8b500b11b5838 100644 --- a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts +++ b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts @@ -29,7 +29,7 @@ import { OBSERVER_VERSION, OBSERVER_VERSION_MAJOR, PARENT_ID, - PROCESOR_NAME, + PROCESSOR_NAME, PROCESSOR_EVENT, SERVICE_ENVIRONMENT, SERVICE_FRAMEWORK_NAME, @@ -48,7 +48,6 @@ import { SPAN_SUBTYPE, SPAN_SYNC, SPAN_TYPE, - SPAN_LINKS, TIMESTAMP, TRACE_ID, TRANSACTION_DURATION, @@ -197,7 +196,7 @@ export const spanMapping = (fields: Partial>) => { success_count: normalizeValue(fields[EVENT_SUCCESS_COUNT]), }, processor: { - name: normalizeValue<'transaction'>(fields[PROCESOR_NAME]), + name: normalizeValue<'transaction'>(fields[PROCESSOR_NAME]), event: normalizeValue<'span'>(fields[PROCESSOR_EVENT]), }, transaction: { @@ -274,20 +273,16 @@ export const spanLinksDetailsMapping = (fields: Partial>) => { if (!fields) return undefined; - return { - span: { - links: [ - { - trace: { - id: normalizeValue(fields[SPAN_LINKS_TRACE_ID]), - }, - span: { - id: normalizeValue(fields[SPAN_LINKS_SPAN_ID]), - }, - }, - ], - }, - }; + return (fields[SPAN_LINKS_TRACE_ID] as string[]).map((v, index) => { + return { + trace: { + id: v, + }, + span: { + id: fields[SPAN_LINKS_SPAN_ID]?.[index] ?? '', + }, + }; + }) as SpanLink[]; }; export const transactionMapping = (fields: Partial>) => { @@ -329,7 +324,8 @@ export const transactionMapping = (fields: Partial>) = outcome: normalizeValue(fields[EVENT_OUTCOME]), }, processor: { - event: normalizeValue(fields[PROCESSOR_EVENT]), + event: normalizeValue<'transaction'>(fields[PROCESSOR_EVENT]), + name: normalizeValue<'transaction'>(fields[PROCESSOR_EVENT]), }, data_stream: { namespace: normalizeValue(fields[DATA_STREAM_NAMESPACE]), @@ -338,11 +334,13 @@ export const transactionMapping = (fields: Partial>) = }, span: { id: normalizeValue(fields[SPAN_ID]), + links: linkedParentsOfSpanMapping(fields), }, observer: { hostname: normalizeValue(fields[OBSERVER_HOSTNAME]), type: normalizeValue(fields[OBSERVER_TYPE]), version: normalizeValue(fields[OBSERVER_VERSION]), + version_major: normalizeValue(fields[OBSERVER_VERSION_MAJOR]), }, timestamp: { us: normalizeValue(fields[TIMESTAMP]), @@ -403,7 +401,7 @@ export const traceDocMapping = ( us: normalizeValue(fields[SPAN_DURATION]), }, action: normalizeValue(fields[SPAN_ACTION]), - links: normalizeValue(fields[SPAN_LINKS]), + links: linkedParentsOfSpanMapping(fields) as SpanLink[], composite: { count: normalizeValue(fields[SPAN_COMPOSITE_COUNT]), sum: { @@ -517,7 +515,7 @@ export const errorSampleDetailsMapping = ( grouping_key: normalizeValue(fields[ERROR_GROUP_ID]), }, processor: { - name: normalizeValue<'error'>(fields[PROCESOR_NAME]), + name: normalizeValue<'error'>(fields[PROCESSOR_NAME]), event: normalizeValue<'error'>(fields[PROCESSOR_EVENT]), }, transaction: { From a08ef938215db02e87e1dedb0979d4766eeadcf6 Mon Sep 17 00:00:00 2001 From: Bryce Buchanan Date: Thu, 26 Sep 2024 13:15:23 -0700 Subject: [PATCH 19/25] fix service metadata endpoints --- packages/kbn-apm-types/src/es_fields/apm.ts | 6 ++ .../apm/common/es_fields/infra_metrics.ts | 2 + .../services/get_service_metadata_details.ts | 87 +++++++++++++++++-- .../services/get_service_metadata_icons.ts | 45 +++++++++- .../apm/server/utils/es_fields_mappings.ts | 35 ++++++-- 5 files changed, 158 insertions(+), 17 deletions(-) diff --git a/packages/kbn-apm-types/src/es_fields/apm.ts b/packages/kbn-apm-types/src/es_fields/apm.ts index ea5e89f91d71f..6c7276317894c 100644 --- a/packages/kbn-apm-types/src/es_fields/apm.ts +++ b/packages/kbn-apm-types/src/es_fields/apm.ts @@ -21,7 +21,11 @@ export const CLOUD_AVAILABILITY_ZONE = 'cloud.availability_zone'; export const CLOUD_PROVIDER = 'cloud.provider'; export const CLOUD_REGION = 'cloud.region'; export const CLOUD_MACHINE_TYPE = 'cloud.machine.type'; +export const CLOUD_PROJECT_ID = 'cloud.project.id'; +export const CLOUD_PROJECT_NAME = 'cloud.project.name'; +export const CLOUD_IMAGE_ID = 'cloud.image.id'; export const CLOUD_ACCOUNT_ID = 'cloud.account.id'; +export const CLOUD_ACCOUNT_NAME = 'cloud.account.name'; export const CLOUD_INSTANCE_ID = 'cloud.instance.id'; export const CLOUD_INSTANCE_NAME = 'cloud.instance.name'; export const CLOUD_SERVICE_NAME = 'cloud.service.name'; @@ -168,6 +172,8 @@ export const HOST_IP = 'host.ip'; export const CONTAINER_ID = 'container.id'; export const CONTAINER = 'container'; export const CONTAINER_IMAGE = 'container.image.name'; +export const CONTAINER_OS = 'container.os'; +export const CONTAINER_TTL_INSTANCES = 'container.totalNumberInstances'; export const KUBERNETES = 'kubernetes'; export const KUBERNETES_POD_NAME = 'kubernetes.pod.name'; diff --git a/x-pack/plugins/observability_solution/apm/common/es_fields/infra_metrics.ts b/x-pack/plugins/observability_solution/apm/common/es_fields/infra_metrics.ts index bfd8d6aa91f9b..7d7081a0e1900 100644 --- a/x-pack/plugins/observability_solution/apm/common/es_fields/infra_metrics.ts +++ b/x-pack/plugins/observability_solution/apm/common/es_fields/infra_metrics.ts @@ -7,6 +7,7 @@ // Kubernetes export const KUBERNETES_CONTAINER_NAME = 'kubernetes.container.name'; +export const KUBERNETES_NODE_NAME = 'kubernetes.node.name'; export const KUBERNETES_CONTAINER_ID = 'kubernetes.container.id'; export const KUBERNETES_DEPLOYMENT = 'kubernetes.deployment'; export const KUBERNETES_DEPLOYMENT_NAME = 'kubernetes.deployment.name'; @@ -14,3 +15,4 @@ export const KUBERNETES_NAMESPACE_NAME = 'kubernetes.namespace.name'; export const KUBERNETES_NAMESPACE = 'kubernetes.namespace'; export const KUBERNETES_REPLICASET = 'kubernetes.replicaset'; export const KUBERNETES_REPLICASET_NAME = 'kubernetes.replicaset.name'; +export const KUBERNETES_POD_NAME = 'kubernetes.pod.name'; diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts index 3609905e2d28f..7ce433bd6f1ad 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts @@ -9,23 +9,50 @@ import { rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; import { environmentQuery } from '../../../common/utils/environment_query'; import { - AGENT, - CONTAINER, - CLOUD, + KUBERNETES_CONTAINER_NAME, + KUBERNETES_DEPLOYMENT_NAME, + KUBERNETES_NAMESPACE, + KUBERNETES_REPLICASET_NAME, + KUBERNETES_CONTAINER_ID, + KUBERNETES_NODE_NAME, + KUBERNETES_POD_NAME, +} from '../../../common/es_fields/infra_metrics'; +import { + AGENT_NAME, + AGENT_VERSION, + CONTAINER_ID, + CONTAINER_IMAGE, CLOUD_AVAILABILITY_ZONE, + CLOUD_INSTANCE_ID, CLOUD_REGION, + CLOUD_PROJECT_NAME, CLOUD_MACHINE_TYPE, CLOUD_SERVICE_NAME, - CONTAINER_ID, - HOST, - KUBERNETES, - SERVICE, - SERVICE_NAME, + CLOUD_INSTANCE_NAME, + CLOUD_PROVIDER, + CLOUD_ACCOUNT_ID, + CLOUD_ACCOUNT_NAME, + CLOUD_IMAGE_ID, + CLOUD_PROJECT_ID, + HOST_ARCHITECTURE, + HOST_HOSTNAME, + HOST_NAME, + HOST_IP, + SERVICE_ENVIRONMENT, + SERVICE_FRAMEWORK_NAME, + SERVICE_FRAMEWORK_VERSION, SERVICE_NODE_NAME, + SERVICE_RUNTIME_NAME, + SERVICE_RUNTIME_VERSION, + SERVICE_LANGUAGE_NAME, + SERVICE_LANGUAGE_VERSION, SERVICE_VERSION, + SERVICE_NAME, FAAS_ID, FAAS_TRIGGER_TYPE, LABEL_TELEMETRY_AUTO_VERSION, + CONTAINER_TTL_INSTANCES, + HOST_OS_PLATFORM, } from '../../../common/es_fields/apm'; import { ContainerType } from '../../../common/service_metadata'; @@ -113,6 +140,7 @@ export async function getServiceMetadataDetails({ body: { track_total_hits: 1, size: 1, + _source: false, query: { bool: { filter, should } }, aggs: { serviceVersions: { @@ -166,7 +194,48 @@ export async function getServiceMetadataDetails({ }, totalNumberInstances: { cardinality: { field: SERVICE_NODE_NAME } }, }, - fields: [SERVICE, AGENT, HOST, CONTAINER, KUBERNETES, CLOUD, LABEL_TELEMETRY_AUTO_VERSION], + fields: [ + SERVICE_ENVIRONMENT, + SERVICE_FRAMEWORK_NAME, + SERVICE_FRAMEWORK_VERSION, + SERVICE_NODE_NAME, + SERVICE_RUNTIME_NAME, + SERVICE_RUNTIME_VERSION, + SERVICE_LANGUAGE_NAME, + SERVICE_LANGUAGE_VERSION, + SERVICE_VERSION, + SERVICE_NAME, + AGENT_NAME, + AGENT_VERSION, + CONTAINER_ID, + CONTAINER_IMAGE, + HOST_ARCHITECTURE, + HOST_HOSTNAME, + HOST_NAME, + HOST_IP, + HOST_OS_PLATFORM, + CONTAINER_TTL_INSTANCES, + KUBERNETES_NAMESPACE, + KUBERNETES_NODE_NAME, + KUBERNETES_POD_NAME, + KUBERNETES_REPLICASET_NAME, + KUBERNETES_DEPLOYMENT_NAME, + KUBERNETES_CONTAINER_ID, + KUBERNETES_CONTAINER_NAME, + CLOUD_AVAILABILITY_ZONE, + CLOUD_INSTANCE_NAME, + CLOUD_INSTANCE_ID, + CLOUD_MACHINE_TYPE, + CLOUD_PROJECT_ID, + CLOUD_PROJECT_NAME, + CLOUD_PROVIDER, + CLOUD_REGION, + CLOUD_ACCOUNT_ID, + CLOUD_ACCOUNT_NAME, + CLOUD_IMAGE_ID, + CLOUD_SERVICE_NAME, + LABEL_TELEMETRY_AUTO_VERSION, + ], }, }; diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts index 8ba26cf5a55c5..42a0e8f40d3f4 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts @@ -13,18 +13,35 @@ import { CLOUD_PROVIDER, CLOUD_SERVICE_NAME, CONTAINER_ID, - KUBERNETES, SERVICE_NAME, KUBERNETES_POD_NAME, HOST_OS_PLATFORM, LABEL_TELEMETRY_AUTO_VERSION, AGENT_VERSION, SERVICE_FRAMEWORK_NAME, + CLOUD_AVAILABILITY_ZONE, + CLOUD_INSTANCE_NAME, + CLOUD_INSTANCE_ID, + CLOUD_MACHINE_TYPE, + CLOUD_PROJECT_ID, + CLOUD_PROJECT_NAME, + CLOUD_REGION, + CLOUD_ACCOUNT_ID, + CLOUD_ACCOUNT_NAME, + CLOUD_IMAGE_ID, } from '../../../common/es_fields/apm'; import { ContainerType } from '../../../common/service_metadata'; import { getProcessorEventForTransactions } from '../../lib/helpers/transactions'; import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; import { ServerlessType, getServerlessTypeFromCloudData } from '../../../common/serverless'; +import { + KUBERNETES_CONTAINER_ID, + KUBERNETES_CONTAINER_NAME, + KUBERNETES_DEPLOYMENT_NAME, + KUBERNETES_NAMESPACE, + KUBERNETES_NODE_NAME, + KUBERNETES_REPLICASET_NAME, +} from '../../../common/es_fields/infra_metrics'; export interface ServiceMetadataIcons { agentName?: string; @@ -70,8 +87,32 @@ export async function getServiceMetadataIcons({ body: { track_total_hits: 1, size: 1, + _source: false, query: { bool: { filter, should } }, - fields: [KUBERNETES, CLOUD_PROVIDER, CONTAINER_ID, AGENT_NAME, CLOUD_SERVICE_NAME], + fields: [ + KUBERNETES_NAMESPACE, + KUBERNETES_NODE_NAME, + KUBERNETES_POD_NAME, + KUBERNETES_REPLICASET_NAME, + KUBERNETES_DEPLOYMENT_NAME, + KUBERNETES_CONTAINER_ID, + KUBERNETES_CONTAINER_NAME, + CLOUD_AVAILABILITY_ZONE, + CLOUD_INSTANCE_NAME, + CLOUD_INSTANCE_ID, + CLOUD_MACHINE_TYPE, + CLOUD_PROJECT_ID, + CLOUD_PROJECT_NAME, + CLOUD_PROVIDER, + CLOUD_REGION, + CLOUD_ACCOUNT_ID, + CLOUD_ACCOUNT_NAME, + CLOUD_IMAGE_ID, + CLOUD_SERVICE_NAME, + CONTAINER_ID, + AGENT_NAME, + CLOUD_SERVICE_NAME, + ], }, }; diff --git a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts index 8b500b11b5838..bf511a0864a51 100644 --- a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts +++ b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts @@ -63,11 +63,9 @@ import { ERROR_ID, ERROR_LOG_MESSAGE, HOST_NAME, - Container, - Kubernetes, + CONTAINER_ID, + CONTAINER_IMAGE, CLOUD_PROVIDER, - CONTAINER, - KUBERNETES, AGENT_ACTIVATION_METHOD, HOST_ARCHITECTURE, HOST_HOSTNAME, @@ -87,6 +85,12 @@ import { SERVICE_TARGET_TYPE, SPAN_REPRESENTATIVE_COUNT, } from '@kbn/apm-types'; +import { + KUBERNETES_CONTAINER_NAME, + KUBERNETES_DEPLOYMENT_NAME, + KUBERNETES_REPLICASET_NAME, + KUBERNETES_CONTAINER_ID, +} from '../../common/es_fields/infra_metrics'; import { Transaction } from '../../typings/es_schemas/ui/transaction'; import { TransactionRaw } from '../../typings/es_schemas/raw/transaction_raw'; import { @@ -545,8 +549,27 @@ export const serviceMetadataIconsMapping = ( name: normalizeValue(fields[CLOUD_PROVIDER]), }, }, - container: normalizeValue(fields[CONTAINER]), - kubernetes: normalizeValue(fields[KUBERNETES]), + container: { + id: normalizeValue(fields[CONTAINER_ID]), + image: normalizeValue(fields[CONTAINER_IMAGE]), + }, + kubernetes: { + pod: { + name: normalizeValue(fields[KUBERNETES_POD_NAME]), + id: normalizeValue(fields[KUBERNETES_POD_UID]), + }, + namespace: normalizeValue(fields[KUBERNETES_NAMESPACE]), + replicaset: { + name: normalizeValue(fields[KUBERNETES_REPLICASET_NAME]), + }, + deployment: { + name: normalizeValue(fields[KUBERNETES_DEPLOYMENT_NAME]), + }, + container: { + id: normalizeValue(fields[KUBERNETES_CONTAINER_ID]), + name: normalizeValue(fields[KUBERNETES_CONTAINER_NAME]), + }, + }, }; }; From f15655962dc056250360e9eea3c1a1aa359a1d4b Mon Sep 17 00:00:00 2001 From: Bryce Buchanan Date: Fri, 27 Sep 2024 10:49:55 -0700 Subject: [PATCH 20/25] PR feedback fix --- .../apm/server/utils/es_fields_mappings.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts index bf511a0864a51..773d2ae9c2147 100644 --- a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts +++ b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts @@ -329,7 +329,7 @@ export const transactionMapping = (fields: Partial>) = }, processor: { event: normalizeValue<'transaction'>(fields[PROCESSOR_EVENT]), - name: normalizeValue<'transaction'>(fields[PROCESSOR_EVENT]), + name: normalizeValue<'transaction'>(fields[PROCESSOR_NAME]), }, data_stream: { namespace: normalizeValue(fields[DATA_STREAM_NAMESPACE]), From 2ef8527280340aad562b8f7e9465d2f6bf807bc4 Mon Sep 17 00:00:00 2001 From: Bryce Buchanan Date: Fri, 27 Sep 2024 13:43:43 -0700 Subject: [PATCH 21/25] added mapping for metadata_details --- .../services/get_service_metadata_details.ts | 4 +- .../services/get_service_metadata_icons.ts | 5 +- .../apm/server/utils/es_fields_mappings.ts | 189 +++++++++++++++--- 3 files changed, 166 insertions(+), 32 deletions(-) diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts index 7ce433bd6f1ad..2c0064e0782c3 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts @@ -7,6 +7,7 @@ import { rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; +import { serviceMetadataDetailsMapping } from '../../utils/es_fields_mappings'; import { environmentQuery } from '../../../common/utils/environment_query'; import { KUBERNETES_CONTAINER_NAME, @@ -59,7 +60,6 @@ import { ContainerType } from '../../../common/service_metadata'; import { TransactionRaw } from '../../../typings/es_schemas/raw/transaction_raw'; import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; import { should } from './get_service_metadata_icons'; -import { normalizeFields } from '../../utils/normalize_fields'; import { isOpenTelemetryAgentName, hasOpenTelemetryPrefix } from '../../../common/agent_name'; type ServiceMetadataDetailsRaw = Pick< @@ -243,7 +243,7 @@ export async function getServiceMetadataDetails({ const fields = response.hits.hits[0]?.fields; // todo: missing `fields` property? - const fieldsNorm = (fields ? normalizeFields(fields) : undefined) as + const fieldsNorm = (fields ? serviceMetadataDetailsMapping(fields) : undefined) as | ServiceMetadataDetailsRaw | undefined; diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts index 42a0e8f40d3f4..3108b07524ec2 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts @@ -7,6 +7,7 @@ import { rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; +import { TransactionRaw } from '@kbn/apm-types'; import { serviceMetadataIconsMapping } from '../../utils/es_fields_mappings'; import { AGENT_NAME, @@ -43,6 +44,8 @@ import { KUBERNETES_REPLICASET_NAME, } from '../../../common/es_fields/infra_metrics'; +type ServiceMetadataIconsRaw = Pick; + export interface ServiceMetadataIcons { agentName?: string; containerType?: ContainerType; @@ -129,7 +132,7 @@ export async function getServiceMetadataIcons({ const { kubernetes, cloud, container, agent } = serviceMetadataIconsMapping( response.hits.hits[0].fields - ); + ) as ServiceMetadataIconsRaw; let containerType: ContainerType; if (!!kubernetes) { diff --git a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts index 773d2ae9c2147..81fadf4424fbd 100644 --- a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts +++ b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts @@ -66,6 +66,7 @@ import { CONTAINER_ID, CONTAINER_IMAGE, CLOUD_PROVIDER, + CLOUD_INSTANCE_NAME, AGENT_ACTIVATION_METHOD, HOST_ARCHITECTURE, HOST_HOSTNAME, @@ -84,6 +85,16 @@ import { SERVICE_TARGET_NAME, SERVICE_TARGET_TYPE, SPAN_REPRESENTATIVE_COUNT, + SERVICE_LANGUAGE_VERSION, + CLOUD_AVAILABILITY_ZONE, + CLOUD_INSTANCE_ID, + CLOUD_MACHINE_TYPE, + CLOUD_PROJECT_ID, + CLOUD_PROJECT_NAME, + CLOUD_REGION, + CLOUD_ACCOUNT_ID, + CLOUD_ACCOUNT_NAME, + CLOUD_IMAGE_ID, } from '@kbn/apm-types'; import { KUBERNETES_CONTAINER_NAME, @@ -102,7 +113,10 @@ import { EventOutcome } from '../../common/event_outcome'; import { Exception } from '../../typings/es_schemas/raw/error_raw'; type ServiceMetadataIconsRaw = Pick; - +type ServiceMetadataDetailsRaw = Pick< + TransactionRaw, + 'service' | 'agent' | 'host' | 'container' | 'kubernetes' | 'cloud' | 'labels' +>; const normalizeValue = (field: unknown[] | unknown): T => { return (Array.isArray(field) && field.length > 0 ? field[0] : field) as T; }; @@ -533,43 +547,160 @@ export const errorSampleDetailsMapping = ( }; }; -export const serviceMetadataIconsMapping = ( +export const serviceMetadataDetailsMapping = ( fields: Partial> -): ServiceMetadataIconsRaw | undefined => { +): ServiceMetadataDetailsRaw | undefined => { if (!fields) return undefined; - + const kubernetesNamespace = normalizeValue(fields[KUBERNETES_NAMESPACE]); + const containerId = normalizeValue(fields[CONTAINER_ID]); + const cloudServiceName = normalizeValue(fields[CLOUD_PROVIDER]); return { - agent: { - name: normalizeValue(fields[AGENT_NAME]), - version: '', - }, - cloud: { - provider: normalizeValue(fields[CLOUD_PROVIDER]), - service: { - name: normalizeValue(fields[CLOUD_PROVIDER]), + service: { + name: normalizeValue(fields[SERVICE_NAME]), + version: normalizeValue(fields[SERVICE_VERSION]), + environment: normalizeValue(fields[SERVICE_ENVIRONMENT]), + framework: { + name: normalizeValue(fields[SERVICE_FRAMEWORK_NAME]), + version: normalizeValue(fields[SERVICE_FRAMEWORK_VERSION]), }, - }, - container: { - id: normalizeValue(fields[CONTAINER_ID]), - image: normalizeValue(fields[CONTAINER_IMAGE]), - }, - kubernetes: { - pod: { - name: normalizeValue(fields[KUBERNETES_POD_NAME]), - id: normalizeValue(fields[KUBERNETES_POD_UID]), + node: { + name: normalizeValue(fields[SERVICE_NODE_NAME]), }, - namespace: normalizeValue(fields[KUBERNETES_NAMESPACE]), - replicaset: { - name: normalizeValue(fields[KUBERNETES_REPLICASET_NAME]), + runtime: { + name: normalizeValue(fields[SERVICE_RUNTIME_NAME]), + version: normalizeValue(fields[SERVICE_RUNTIME_VERSION]), }, - deployment: { - name: normalizeValue(fields[KUBERNETES_DEPLOYMENT_NAME]), + language: { + name: normalizeValue(fields[SERVICE_LANGUAGE_NAME]), + version: normalizeValue(fields[SERVICE_LANGUAGE_VERSION]), }, - container: { - id: normalizeValue(fields[KUBERNETES_CONTAINER_ID]), - name: normalizeValue(fields[KUBERNETES_CONTAINER_NAME]), + }, + agent: { + name: normalizeValue(fields[AGENT_NAME]), + version: normalizeValue(fields[AGENT_VERSION]), + }, + host: { + architecture: normalizeValue(fields[HOST_ARCHITECTURE]), + hostname: normalizeValue(fields[HOST_HOSTNAME]), + name: normalizeValue(fields[HOST_NAME]), + ip: normalizeValue(fields[HOST_IP]), + os: { + platform: normalizeValue(fields[HOST_OS_PLATFORM]), }, }, + ...(containerId + ? { + container: { + id: containerId, + image: normalizeValue(fields[CONTAINER_IMAGE]), + }, + } + : undefined), + ...(kubernetesNamespace + ? { + kubernetes: { + pod: { + name: normalizeValue(fields[KUBERNETES_POD_NAME]), + uid: normalizeValue(fields[KUBERNETES_POD_UID]), + }, + namespace: kubernetesNamespace, + replicaset: { + name: normalizeValue(fields[KUBERNETES_REPLICASET_NAME]), + }, + deployment: { + name: normalizeValue(fields[KUBERNETES_DEPLOYMENT_NAME]), + }, + container: { + id: normalizeValue(fields[KUBERNETES_CONTAINER_ID]), + name: normalizeValue(fields[KUBERNETES_CONTAINER_NAME]), + }, + }, + } + : undefined), + ...(cloudServiceName + ? { + cloud: { + availability_zone: normalizeValue(fields[CLOUD_AVAILABILITY_ZONE]), + instance: { + name: normalizeValue(fields[CLOUD_INSTANCE_NAME]), + id: normalizeValue(fields[CLOUD_INSTANCE_ID]), + }, + machine: { + type: normalizeValue(fields[CLOUD_MACHINE_TYPE]), + }, + project: { + id: normalizeValue(fields[CLOUD_PROJECT_ID]), + name: normalizeValue(fields[CLOUD_PROJECT_NAME]), + }, + provider: normalizeValue(fields[CLOUD_PROVIDER]), + region: normalizeValue(fields[CLOUD_REGION]), + account: { + id: normalizeValue(fields[CLOUD_ACCOUNT_ID]), + name: normalizeValue(fields[CLOUD_ACCOUNT_NAME]), + }, + image: { + id: normalizeValue(fields[CLOUD_IMAGE_ID]), + }, + service: { + name: normalizeValue(fields[CLOUD_PROVIDER]), + }, + }, + } + : undefined), + }; +}; + +export const serviceMetadataIconsMapping = ( + fields: Partial> +): ServiceMetadataIconsRaw | undefined => { + if (!fields) return undefined; + const kubernetesNamespace = normalizeValue(fields[KUBERNETES_NAMESPACE]); + const containerId = normalizeValue(fields[CONTAINER_ID]); + const cloudServiceName = normalizeValue(fields[CLOUD_PROVIDER]); + return { + agent: { + name: normalizeValue(fields[AGENT_NAME]), + version: '', + }, + ...(cloudServiceName + ? { + cloud: { + provider: normalizeValue(fields[CLOUD_PROVIDER]), + service: { + name: normalizeValue(fields[CLOUD_PROVIDER]), + }, + }, + } + : undefined), + ...(containerId + ? { + container: { + id: containerId, + image: normalizeValue(fields[CONTAINER_IMAGE]), + }, + } + : undefined), + ...(kubernetesNamespace + ? { + kubernetes: { + pod: { + name: normalizeValue(fields[KUBERNETES_POD_NAME]), + uid: normalizeValue(fields[KUBERNETES_POD_UID]), + }, + namespace: kubernetesNamespace, + replicaset: { + name: normalizeValue(fields[KUBERNETES_REPLICASET_NAME]), + }, + deployment: { + name: normalizeValue(fields[KUBERNETES_DEPLOYMENT_NAME]), + }, + container: { + id: normalizeValue(fields[KUBERNETES_CONTAINER_ID]), + name: normalizeValue(fields[KUBERNETES_CONTAINER_NAME]), + }, + }, + } + : undefined), }; }; From 004f58730ea35f65f726d2aae16dbdec09f2bfb0 Mon Sep 17 00:00:00 2001 From: Bryce Buchanan Date: Fri, 27 Sep 2024 14:24:26 -0700 Subject: [PATCH 22/25] fixed kubernetes check in metadata mappings --- .../apm/server/utils/es_fields_mappings.ts | 50 ++++++++++++++----- 1 file changed, 38 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts index 81fadf4424fbd..26887fa08ee36 100644 --- a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts +++ b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts @@ -95,6 +95,7 @@ import { CLOUD_ACCOUNT_ID, CLOUD_ACCOUNT_NAME, CLOUD_IMAGE_ID, + CLOUD_SERVICE_NAME, } from '@kbn/apm-types'; import { KUBERNETES_CONTAINER_NAME, @@ -554,6 +555,7 @@ export const serviceMetadataDetailsMapping = ( const kubernetesNamespace = normalizeValue(fields[KUBERNETES_NAMESPACE]); const containerId = normalizeValue(fields[CONTAINER_ID]); const cloudServiceName = normalizeValue(fields[CLOUD_PROVIDER]); + const serviceRuntimeName = normalizeValue(fields[SERVICE_RUNTIME_NAME]); return { service: { name: normalizeValue(fields[SERVICE_NAME]), @@ -566,10 +568,14 @@ export const serviceMetadataDetailsMapping = ( node: { name: normalizeValue(fields[SERVICE_NODE_NAME]), }, - runtime: { - name: normalizeValue(fields[SERVICE_RUNTIME_NAME]), - version: normalizeValue(fields[SERVICE_RUNTIME_VERSION]), - }, + ...(serviceRuntimeName + ? { + runtime: { + name: serviceRuntimeName, + version: normalizeValue(fields[SERVICE_RUNTIME_VERSION]), + }, + } + : undefined), language: { name: normalizeValue(fields[SERVICE_LANGUAGE_NAME]), version: normalizeValue(fields[SERVICE_LANGUAGE_VERSION]), @@ -642,7 +648,7 @@ export const serviceMetadataDetailsMapping = ( id: normalizeValue(fields[CLOUD_IMAGE_ID]), }, service: { - name: normalizeValue(fields[CLOUD_PROVIDER]), + name: normalizeValue(fields[CLOUD_SERVICE_NAME]), }, }, } @@ -654,20 +660,40 @@ export const serviceMetadataIconsMapping = ( fields: Partial> ): ServiceMetadataIconsRaw | undefined => { if (!fields) return undefined; - const kubernetesNamespace = normalizeValue(fields[KUBERNETES_NAMESPACE]); + const kubernetesPodName = normalizeValue(fields[KUBERNETES_POD_NAME]); const containerId = normalizeValue(fields[CONTAINER_ID]); - const cloudServiceName = normalizeValue(fields[CLOUD_PROVIDER]); + const cloudProvider = normalizeValue(fields[CLOUD_PROVIDER]); return { agent: { name: normalizeValue(fields[AGENT_NAME]), version: '', }, - ...(cloudServiceName + ...(cloudProvider ? { cloud: { - provider: normalizeValue(fields[CLOUD_PROVIDER]), + availability_zone: normalizeValue(fields[CLOUD_AVAILABILITY_ZONE]), + instance: { + name: normalizeValue(fields[CLOUD_INSTANCE_NAME]), + id: normalizeValue(fields[CLOUD_INSTANCE_ID]), + }, + machine: { + type: normalizeValue(fields[CLOUD_MACHINE_TYPE]), + }, + project: { + id: normalizeValue(fields[CLOUD_PROJECT_ID]), + name: normalizeValue(fields[CLOUD_PROJECT_NAME]), + }, + provider: cloudProvider, + region: normalizeValue(fields[CLOUD_REGION]), + account: { + id: normalizeValue(fields[CLOUD_ACCOUNT_ID]), + name: normalizeValue(fields[CLOUD_ACCOUNT_NAME]), + }, + image: { + id: normalizeValue(fields[CLOUD_IMAGE_ID]), + }, service: { - name: normalizeValue(fields[CLOUD_PROVIDER]), + name: normalizeValue(fields[CLOUD_SERVICE_NAME]), }, }, } @@ -680,14 +706,14 @@ export const serviceMetadataIconsMapping = ( }, } : undefined), - ...(kubernetesNamespace + ...(kubernetesPodName ? { kubernetes: { pod: { name: normalizeValue(fields[KUBERNETES_POD_NAME]), uid: normalizeValue(fields[KUBERNETES_POD_UID]), }, - namespace: kubernetesNamespace, + namespace: normalizeValue(fields[KUBERNETES_NAMESPACE]), replicaset: { name: normalizeValue(fields[KUBERNETES_REPLICASET_NAME]), }, From 354d8218f7609bbc0a5897f90a0cd72e15abcab7 Mon Sep 17 00:00:00 2001 From: Milosz Marcinkowski Date: Mon, 30 Sep 2024 11:17:33 +0200 Subject: [PATCH 23/25] use one mapping for metadata details --- .../get_service_instance_metadata_details.ts | 13 +-- .../apm/server/utils/es_fields_mappings.ts | 106 ++---------------- 2 files changed, 15 insertions(+), 104 deletions(-) diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_metadata_details.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_metadata_details.ts index 23491821aa9cf..2054c29deae79 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_metadata_details.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_metadata_details.ts @@ -7,11 +7,7 @@ import { merge } from 'lodash'; import { rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; -import { - metadataAppMetricMapping, - metadataAppTransactionEventMapping, - metaDataAppTransactionMetricMapping, -} from '../../utils/es_fields_mappings'; +import { serviceInstanceMetadataDetailsMapping } from '../../utils/es_fields_mappings'; import { METRICSET_NAME, SERVICE_NAME, SERVICE_NODE_NAME } from '../../../common/es_fields/apm'; import { maybe } from '../../../common/utils/maybe'; import { @@ -76,7 +72,7 @@ export async function getServiceInstanceMetadataDetails({ } ); - return maybe(metadataAppMetricMapping(response.hits.hits[0]?.fields)); + return maybe(serviceInstanceMetadataDetailsMapping(response.hits.hits[0]?.fields)); } async function getTransactionEventSample() { @@ -96,7 +92,7 @@ export async function getServiceInstanceMetadataDetails({ } ); - return maybe(metadataAppTransactionEventMapping(response.hits.hits[0]?.fields)); + return maybe(serviceInstanceMetadataDetailsMapping(response.hits.hits[0]?.fields)); } async function getTransactionMetricSample() { @@ -119,7 +115,7 @@ export async function getServiceInstanceMetadataDetails({ }, } ); - return maybe(metaDataAppTransactionMetricMapping(response.hits.hits[0]?.fields)); + return maybe(serviceInstanceMetadataDetailsMapping(response.hits.hits[0]?.fields)); } // we can expect the most detail of application metrics, @@ -138,7 +134,6 @@ export async function getServiceInstanceMetadataDetails({ applicationMetricSample ); - // todo: find scenarios for container and cloud then add to mappings const { agent, service, container, kubernetes, host, cloud } = sample; return { diff --git a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts index b8b57b5f36b02..f48b6015105a0 100644 --- a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts +++ b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts @@ -87,6 +87,7 @@ import { SERVICE_TARGET_NAME, SERVICE_TARGET_TYPE, SPAN_REPRESENTATIVE_COUNT, + SERVICE_LANGUAGE_VERSION, } from '@kbn/apm-types'; import { Transaction } from '../../typings/es_schemas/ui/transaction'; import { TransactionRaw } from '../../typings/es_schemas/raw/transaction_raw'; @@ -566,35 +567,28 @@ export const serviceVersionMapping = ( }; }; -export const metadataAppMetricMapping = (fields: Partial>) => { +// todo: missing `container` and `could` mappings +export const serviceInstanceMetadataDetailsMapping = ( + fields: Partial> = {} +) => { if (!fields) return undefined; return { '@timestamp': normalizeValue(fields[AT_TIMESTAMP]), agent: { - name: normalizeValue(fields[AGENT_NAME]), + name: normalizeValue(fields[AGENT_NAME]), version: normalizeValue(fields[AGENT_VERSION]), activation_method: normalizeValue(fields[AGENT_ACTIVATION_METHOD]), }, host: { architecture: normalizeValue(fields[HOST_ARCHITECTURE]), hostname: normalizeValue(fields[HOST_HOSTNAME]), - ip: normalizeValue(fields[HOST_IP]), name: normalizeValue(fields[HOST_NAME]), + ip: normalizeValue(fields[HOST_IP]), os: { platform: normalizeValue(fields[HOST_OS_PLATFORM]), }, }, - kubernetes: { - namespace: normalizeValue(fields[KUBERNETES_NAMESPACE]), - node: { - name: normalizeValue(fields[KUBERNETES_NODE_NAME]), - }, - pod: { - name: normalizeValue(fields[KUBERNETES_POD_NAME]), - uid: normalizeValue(fields[KUBERNETES_POD_UID]), - }, - }, service: { name: normalizeValue(fields[SERVICE_NAME]), environment: normalizeValue(fields[SERVICE_ENVIRONMENT]), @@ -602,9 +596,6 @@ export const metadataAppMetricMapping = (fields: Partial(fields[SERVICE_FRAMEWORK_NAME]), versions: normalizeValue(fields[SERVICE_FRAMEWORK_VERSION]), }, - language: { - name: normalizeValue(fields[SERVICE_LANGUAGE_NAME]), - }, node: { name: normalizeValue(fields[SERVICE_NODE_NAME]), }, @@ -612,15 +603,12 @@ export const metadataAppMetricMapping = (fields: Partial(fields[SERVICE_RUNTIME_NAME]), version: normalizeValue(fields[SERVICE_RUNTIME_VERSION]), }, + language: { + name: normalizeValue(fields[SERVICE_LANGUAGE_NAME]), + version: normalizeValue(fields[SERVICE_LANGUAGE_VERSION]), + }, version: normalizeValue(fields[SERVICE_VERSION]), }, - }; -}; - -export const metadataAppTransactionEventMapping = (fields: Partial>) => { - if (!fields) return undefined; - - return { kubernetes: { namespace: normalizeValue(fields[KUBERNETES_NAMESPACE]), node: { @@ -631,78 +619,6 @@ export const metadataAppTransactionEventMapping = (fields: Partial(fields[KUBERNETES_POD_UID]), }, }, - agent: { - name: normalizeValue(fields[AGENT_NAME]), - version: normalizeValue(fields[AGENT_VERSION]), - activation_method: normalizeValue(fields[AGENT_ACTIVATION_METHOD]), - }, - '@timestamp': normalizeValue(fields[AT_TIMESTAMP]), - service: { - name: normalizeValue(fields[SERVICE_NAME]), - environment: normalizeValue(fields[SERVICE_ENVIRONMENT]), - framework: { - name: normalizeValue(fields[SERVICE_FRAMEWORK_NAME]), - versions: normalizeValue(fields[SERVICE_FRAMEWORK_VERSION]), - }, - language: { - name: normalizeValue(fields[SERVICE_LANGUAGE_NAME]), - }, - node: { - name: normalizeValue(fields[SERVICE_NODE_NAME]), - }, - runtime: { - name: normalizeValue(fields[SERVICE_RUNTIME_NAME]), - version: normalizeValue(fields[SERVICE_RUNTIME_VERSION]), - }, - version: normalizeValue(fields[SERVICE_VERSION]), - }, - host: { - architecture: normalizeValue(fields[HOST_ARCHITECTURE]), - hostname: normalizeValue(fields[HOST_HOSTNAME]), - ip: fields[HOST_IP] as string[] | undefined, - name: normalizeValue(fields[HOST_NAME]), - os: { - platform: normalizeValue(fields[HOST_OS_PLATFORM]), - }, - }, - }; -}; - -export const metaDataAppTransactionMetricMapping = (fields: Partial>) => { - if (!fields) return undefined; - - return { - '@timestamp': normalizeValue(fields[AT_TIMESTAMP]), - agent: { - name: normalizeValue(fields[AGENT_NAME]), - }, - host: { - hostname: normalizeValue(fields[HOST_HOSTNAME]), - name: normalizeValue(fields[HOST_NAME]), - os: { - platform: normalizeValue(fields[HOST_OS_PLATFORM]), - }, - }, - kubernetes: { - pod: { - name: normalizeValue(fields[KUBERNETES_POD_NAME]), - }, - }, - service: { - name: normalizeValue(fields[SERVICE_NAME]), - environment: normalizeValue(fields[SERVICE_ENVIRONMENT]), - language: { - name: normalizeValue(fields[SERVICE_LANGUAGE_NAME]), - }, - node: { - name: normalizeValue(fields[SERVICE_NODE_NAME]), - }, - runtime: { - name: normalizeValue(fields[SERVICE_RUNTIME_NAME]), - version: normalizeValue(fields[SERVICE_RUNTIME_VERSION]), - }, - version: normalizeValue(fields[SERVICE_VERSION]), - }, }; }; From aebea67779fc34029c1917c61670e539c80918bf Mon Sep 17 00:00:00 2001 From: Milosz Marcinkowski Date: Mon, 30 Sep 2024 12:37:15 +0200 Subject: [PATCH 24/25] update 'todo' and processor_name typo --- packages/kbn-apm-types/src/es_fields/apm.ts | 2 +- .../apm/server/utils/es_fields_mappings.ts | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/kbn-apm-types/src/es_fields/apm.ts b/packages/kbn-apm-types/src/es_fields/apm.ts index eaf6e4e0282f8..a8b7ae558b3e6 100644 --- a/packages/kbn-apm-types/src/es_fields/apm.ts +++ b/packages/kbn-apm-types/src/es_fields/apm.ts @@ -61,7 +61,7 @@ export const OBSERVER_TYPE = 'observer.type'; export const OBSERVER_VERSION = 'observer.version'; export const OBSERVER_VERSION_MAJOR = 'observer.version_major'; export const PROCESSOR_EVENT = 'processor.event'; -export const PROCESOR_NAME = 'processor.name'; +export const PROCESSOR_NAME = 'processor.name'; export const TRANSACTION_DURATION = 'transaction.duration.us'; export const TRANSACTION_DURATION_HISTOGRAM = 'transaction.duration.histogram'; diff --git a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts index f48b6015105a0..ccd116c8d5949 100644 --- a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts +++ b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts @@ -29,7 +29,7 @@ import { OBSERVER_VERSION, OBSERVER_VERSION_MAJOR, PARENT_ID, - PROCESOR_NAME, + PROCESSOR_NAME, PROCESSOR_EVENT, SERVICE_ENVIRONMENT, SERVICE_FRAMEWORK_NAME, @@ -198,7 +198,7 @@ export const spanMapping = (fields: Partial>) => { success_count: normalizeValue(fields[EVENT_SUCCESS_COUNT]), }, processor: { - name: normalizeValue<'transaction'>(fields[PROCESOR_NAME]), + name: normalizeValue<'transaction'>(fields[PROCESSOR_NAME]), event: normalizeValue<'span'>(fields[PROCESSOR_EVENT]), }, transaction: { @@ -518,7 +518,7 @@ export const errorSampleDetailsMapping = ( grouping_key: normalizeValue(fields[ERROR_GROUP_ID]), }, processor: { - name: normalizeValue<'error'>(fields[PROCESOR_NAME]), + name: normalizeValue<'error'>(fields[PROCESSOR_NAME]), event: normalizeValue<'error'>(fields[PROCESSOR_EVENT]), }, transaction: { @@ -553,7 +553,7 @@ export const serviceMetadataIconsMapping = ( }; }; -// todo(milosz): test it +// todo: check https://github.com/jennypavlova/kibana/pull/6#discussion_r1771611817 export const serviceVersionMapping = ( fields: Partial> ): Pick | undefined => { @@ -622,7 +622,7 @@ export const serviceInstanceMetadataDetailsMapping = ( }; }; -// todo: missing cloud and service properties +// todo: missing `cloud` and `service` properties export const serviceAgentNameMapping = (fields: Partial>) => { if (!fields) return undefined; From e4fac40d6dd74746a9252f5f943d1a6bcb321f3d Mon Sep 17 00:00:00 2001 From: Milosz Marcinkowski Date: Mon, 30 Sep 2024 15:53:29 +0200 Subject: [PATCH 25/25] add `container` mapping --- .../server/routes/services/get_service_metadata_details.ts | 1 - .../apm/server/utils/es_fields_mappings.ts | 5 ++++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts index dc1ac79b7ffd9..caec6b724bb06 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts @@ -245,7 +245,6 @@ export async function getServiceMetadataDetails({ const response = await apmEventClient.search('get_service_metadata_details', params); const fields = response.hits.hits[0]?.fields; - // todo: missing `fields` property? const fieldsNorm = (fields ? serviceMetadataDetailsMapping(fields) : undefined) as | ServiceMetadataDetailsRaw | undefined; diff --git a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts index 1bc570365b33d..a5335c56320db 100644 --- a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts +++ b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts @@ -744,7 +744,7 @@ export const serviceVersionMapping = ( }; }; -// todo: missing `container` and `could` mappings +// todo: missing `could` mapping export const serviceInstanceMetadataDetailsMapping = ( fields: Partial> = {} ) => { @@ -796,6 +796,9 @@ export const serviceInstanceMetadataDetailsMapping = ( uid: normalizeValue(fields[KUBERNETES_POD_UID]), }, }, + container: { + id: normalizeValue(fields[CONTAINER_ID]), + }, }; };