diff --git a/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap b/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap index 3ac20a05639fb..4ee7692222d68 100644 --- a/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap +++ b/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap @@ -567,6 +567,22 @@ Object { }, }, }, + "cloud": Object { + "properties": Object { + "availability_zone": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "provider": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "region": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, "counts": Object { "properties": Object { "agent_configuration": Object { diff --git a/x-pack/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap b/x-pack/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap index f3dc7abcf8239..06ca3145bfce9 100644 --- a/x-pack/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap +++ b/x-pack/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap @@ -8,6 +8,12 @@ exports[`Error CLIENT_GEO 1`] = `undefined`; exports[`Error CLIENT_GEO_COUNTRY_ISO_CODE 1`] = `undefined`; +exports[`Error CLOUD_AVAILABILITY_ZONE 1`] = `"europe-west1-c"`; + +exports[`Error CLOUD_PROVIDER 1`] = `"gcp"`; + +exports[`Error CLOUD_REGION 1`] = `"europe-west1"`; + exports[`Error CONTAINER_ID 1`] = `undefined`; exports[`Error DESTINATION_ADDRESS 1`] = `undefined`; @@ -146,6 +152,12 @@ exports[`Span CLIENT_GEO 1`] = `undefined`; exports[`Span CLIENT_GEO_COUNTRY_ISO_CODE 1`] = `undefined`; +exports[`Span CLOUD_AVAILABILITY_ZONE 1`] = `"europe-west1-c"`; + +exports[`Span CLOUD_PROVIDER 1`] = `"gcp"`; + +exports[`Span CLOUD_REGION 1`] = `"europe-west1"`; + exports[`Span CONTAINER_ID 1`] = `undefined`; exports[`Span DESTINATION_ADDRESS 1`] = `undefined`; @@ -284,6 +296,12 @@ exports[`Transaction CLIENT_GEO 1`] = `undefined`; exports[`Transaction CLIENT_GEO_COUNTRY_ISO_CODE 1`] = `undefined`; +exports[`Transaction CLOUD_AVAILABILITY_ZONE 1`] = `"europe-west1-c"`; + +exports[`Transaction CLOUD_PROVIDER 1`] = `"gcp"`; + +exports[`Transaction CLOUD_REGION 1`] = `"europe-west1"`; + exports[`Transaction CONTAINER_ID 1`] = `"container1234567890abcdef"`; exports[`Transaction DESTINATION_ADDRESS 1`] = `undefined`; diff --git a/x-pack/plugins/apm/common/apm_telemetry.ts b/x-pack/plugins/apm/common/apm_telemetry.ts index 1532058adf64f..5837648f3e505 100644 --- a/x-pack/plugins/apm/common/apm_telemetry.ts +++ b/x-pack/plugins/apm/common/apm_telemetry.ts @@ -91,6 +91,13 @@ export function getApmTelemetryMapping() { {} ), }, + cloud: { + properties: { + availability_zone: keyword, + provider: keyword, + region: keyword, + }, + }, counts: { properties: { agent_configuration: allProperties, diff --git a/x-pack/plugins/apm/common/elasticsearch_fieldnames.test.ts b/x-pack/plugins/apm/common/elasticsearch_fieldnames.test.ts index ae6eb7e9460cc..6d7e42363dfd1 100644 --- a/x-pack/plugins/apm/common/elasticsearch_fieldnames.test.ts +++ b/x-pack/plugins/apm/common/elasticsearch_fieldnames.test.ts @@ -22,6 +22,11 @@ describe('Transaction', () => { name: 'java', version: 'agent version', }, + cloud: { + availability_zone: 'europe-west1-c', + provider: 'gcp', + region: 'europe-west1', + }, http: { request: { method: 'GET' }, response: { status_code: 200 }, @@ -73,6 +78,11 @@ describe('Span', () => { name: 'java', version: 'agent version', }, + cloud: { + availability_zone: 'europe-west1-c', + provider: 'gcp', + region: 'europe-west1', + }, processor: { name: 'transaction', event: 'span', @@ -120,6 +130,11 @@ describe('Error', () => { name: 'java', version: 'agent version', }, + cloud: { + availability_zone: 'europe-west1-c', + provider: 'gcp', + region: 'europe-west1', + }, error: { exception: [ { diff --git a/x-pack/plugins/apm/common/elasticsearch_fieldnames.ts b/x-pack/plugins/apm/common/elasticsearch_fieldnames.ts index 7537dba7f8411..a5a42ccbb9a21 100644 --- a/x-pack/plugins/apm/common/elasticsearch_fieldnames.ts +++ b/x-pack/plugins/apm/common/elasticsearch_fieldnames.ts @@ -4,6 +4,10 @@ * you may not use this file except in compliance with the Elastic License. */ +export const CLOUD_AVAILABILITY_ZONE = 'cloud.availability_zone'; +export const CLOUD_PROVIDER = 'cloud.provider'; +export const CLOUD_REGION = 'cloud.region'; + export const SERVICE_NAME = 'service.name'; export const SERVICE_ENVIRONMENT = 'service.environment'; export const SERVICE_FRAMEWORK_NAME = 'service.framework.name'; diff --git a/x-pack/plugins/apm/dev_docs/telemetry.md b/x-pack/plugins/apm/dev_docs/telemetry.md index 9674d39e57177..fa8e057a59595 100644 --- a/x-pack/plugins/apm/dev_docs/telemetry.md +++ b/x-pack/plugins/apm/dev_docs/telemetry.md @@ -40,6 +40,9 @@ and/or config/kibana.dev.yml files. Running the script with `--clear` will delete the index first. +If you're using an Elasticsearch instance without TLS verification (if you have `elasticsearch.ssl.verificationMode: none` set in your kibana.yml) +you can run the script with `env NODE_TLS_REJECT_UNAUTHORIZED=0` to avoid TLS connection errors. + After running the script you should see sample telemetry data in the "xpack-phone-home" index. ### Updating Data Telemetry Mappings @@ -62,6 +65,10 @@ node ./scripts/merge-telemetry-mapping.js ../../../../telemetry/config/templates this will replace the contents of the mapping in the repository checkout with the updated mapping. You can then [follow the telemetry team's instructions](https://github.com/elastic/telemetry#mappings) for opening a pull request with the mapping changes. +The queries for the stats are in the [collect data telemetry tasks](../server/lib/apm_telemetry/collect_data_telemetry/tasks.ts). + +The collection tasks also use the [`APMDataTelemetry` type](../server/lib/apm_telemetry/types.ts) which also needs to be updated with any changes to the fields. + ## Behavioral Telemetry Behavioral telemetry is recorded with the ui_metrics and application_usage methods from the Usage Collection plugin. diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts new file mode 100644 index 0000000000000..c648cf4cc116a --- /dev/null +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { tasks } from './tasks'; +import { ApmIndicesConfig } from '../../settings/apm_indices/get_apm_indices'; + +describe('data telemetry collection tasks', () => { + const indices = { + 'apm_oss.errorIndices': 'apm-8.0.0-error', + 'apm_oss.metricsIndices': 'apm-8.0.0-metric', + 'apm_oss.spanIndices': 'apm-8.0.0-span', + 'apm_oss.transactionIndices': 'apm-8.0.0-transaction', + } as ApmIndicesConfig; + + describe('cloud', () => { + const cloudTask = tasks.find((task) => task.name === 'cloud'); + + it('returns a map of cloud provider data', async () => { + const search = jest.fn().mockResolvedValueOnce({ + aggregations: { + availability_zone: { + buckets: [ + { doc_count: 1, key: 'us-west-1' }, + { doc_count: 1, key: 'europe-west1-c' }, + ], + }, + provider: { + buckets: [ + { doc_count: 1, key: 'aws' }, + { doc_count: 1, key: 'gcp' }, + ], + }, + region: { + buckets: [ + { doc_count: 1, key: 'us-west' }, + { doc_count: 1, key: 'europe-west1' }, + ], + }, + }, + }); + + expect(await cloudTask?.executor({ indices, search } as any)).toEqual({ + cloud: { + availability_zone: ['us-west-1', 'europe-west1-c'], + provider: ['aws', 'gcp'], + region: ['us-west', 'europe-west1'], + }, + }); + }); + + describe('with no results', () => { + it('returns an empty map', async () => { + const search = jest.fn().mockResolvedValueOnce({}); + + expect(await cloudTask?.executor({ indices, search } as any)).toEqual({ + cloud: { + availability_zone: [], + provider: [], + region: [], + }, + }); + }); + }); + }); +}); diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts index ba0bfdf41d078..f27af9a2cc516 100644 --- a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts @@ -22,6 +22,9 @@ import { SERVICE_RUNTIME_NAME, SERVICE_RUNTIME_VERSION, USER_AGENT_ORIGINAL, + CLOUD_AVAILABILITY_ZONE, + CLOUD_PROVIDER, + CLOUD_REGION, } from '../../../../common/elasticsearch_fieldnames'; import { Span } from '../../../../typings/es_schemas/ui/span'; import { APMError } from '../../../../typings/es_schemas/ui/apm_error'; @@ -32,6 +35,66 @@ const TIME_RANGES = ['1d', 'all'] as const; type TimeRange = typeof TIME_RANGES[number]; export const tasks: TelemetryTask[] = [ + { + name: 'cloud', + executor: async ({ indices, search }) => { + function getBucketKeys({ + buckets, + }: { + buckets: Array<{ + doc_count: number; + key: string | number; + }>; + }) { + return buckets.map((bucket) => bucket.key as string); + } + + const az = 'availability_zone'; + const region = 'region'; + const provider = 'provider'; + + const response = await search({ + index: [ + indices['apm_oss.errorIndices'], + indices['apm_oss.metricsIndices'], + indices['apm_oss.spanIndices'], + indices['apm_oss.transactionIndices'], + ], + body: { + size: 0, + aggs: { + [az]: { + terms: { + field: CLOUD_AVAILABILITY_ZONE, + }, + }, + [provider]: { + terms: { + field: CLOUD_PROVIDER, + }, + }, + [region]: { + terms: { + field: CLOUD_REGION, + }, + }, + }, + }, + }); + + const { aggregations } = response; + + if (!aggregations) { + return { cloud: { [az]: [], [provider]: [], [region]: [] } }; + } + const cloud = { + [az]: getBucketKeys(aggregations[az]), + [provider]: getBucketKeys(aggregations[provider]), + [region]: getBucketKeys(aggregations[region]), + }; + return { cloud }; + }, + }, { name: 'processor_events', executor: async ({ indices, search }) => { diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/types.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/types.ts index 14807d50f3c31..a1d94333b1a08 100644 --- a/x-pack/plugins/apm/server/lib/apm_telemetry/types.ts +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/types.ts @@ -25,6 +25,11 @@ export type APMDataTelemetry = DeepPartial<{ patch: number; }; }; + cloud: { + availability_zone: string[]; + provider: string[]; + region: string[]; + }; counts: { transaction: TimeframeMap; span: TimeframeMap; @@ -102,6 +107,7 @@ export type APMDataTelemetry = DeepPartial<{ }; }; tasks: Record< + | 'cloud' | 'processor_events' | 'agent_configuration' | 'services'