From 146a149f853fe290a2e4d1a4b5a8ae6c4ff6abc6 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Tue, 24 Mar 2020 08:08:12 +0100 Subject: [PATCH 1/2] Revert "Revert "[APM] Collect telemetry about data/API performance (#51612)"" This reverts commit 6de7f2a62b2b078b703bbe6f18475909e1224f57. --- src/dev/run_check_lockfile_symlinks.js | 2 + x-pack/legacy/plugins/apm/index.ts | 19 +- x-pack/legacy/plugins/apm/mappings.json | 773 +++++++++++++++++- x-pack/legacy/plugins/apm/scripts/.gitignore | 1 + .../legacy/plugins/apm/scripts/package.json | 10 + .../apm/scripts/setup-kibana-security.js | 1 + .../apm/scripts/upload-telemetry-data.js | 21 + .../download-telemetry-template.ts | 26 + .../generate-sample-documents.ts | 124 +++ .../scripts/upload-telemetry-data/index.ts | 208 +++++ .../elasticsearch_fieldnames.test.ts.snap | 48 +- x-pack/plugins/apm/common/agent_name.ts | 44 +- .../apm/common/apm_saved_object_constants.ts | 10 +- .../common/elasticsearch_fieldnames.test.ts | 15 +- .../apm/common/elasticsearch_fieldnames.ts | 11 +- x-pack/plugins/apm/kibana.json | 5 +- x-pack/plugins/apm/server/index.ts | 6 +- .../lib/apm_telemetry/__test__/index.test.ts | 83 -- .../collect_data_telemetry/index.ts | 77 ++ .../collect_data_telemetry/tasks.ts | 725 ++++++++++++++++ .../apm/server/lib/apm_telemetry/index.ts | 155 +++- .../apm/server/lib/apm_telemetry/types.ts | 118 +++ .../server/lib/helpers/setup_request.test.ts | 13 + x-pack/plugins/apm/server/plugin.ts | 33 +- .../server/routes/create_api/index.test.ts | 1 + x-pack/plugins/apm/server/routes/services.ts | 16 - .../apm/typings/elasticsearch/aggregations.ts | 26 +- .../apm/typings/es_schemas/raw/error_raw.ts | 2 + .../typings/es_schemas/raw/fields/observer.ts | 10 + .../apm/typings/es_schemas/raw/span_raw.ts | 2 + .../typings/es_schemas/raw/transaction_raw.ts | 2 + 31 files changed, 2383 insertions(+), 204 deletions(-) create mode 100644 x-pack/legacy/plugins/apm/scripts/.gitignore create mode 100644 x-pack/legacy/plugins/apm/scripts/package.json create mode 100644 x-pack/legacy/plugins/apm/scripts/upload-telemetry-data.js create mode 100644 x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/download-telemetry-template.ts create mode 100644 x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/generate-sample-documents.ts create mode 100644 x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/index.ts delete mode 100644 x-pack/plugins/apm/server/lib/apm_telemetry/__test__/index.test.ts create mode 100644 x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/index.ts create mode 100644 x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts create mode 100644 x-pack/plugins/apm/server/lib/apm_telemetry/types.ts create mode 100644 x-pack/plugins/apm/typings/es_schemas/raw/fields/observer.ts diff --git a/src/dev/run_check_lockfile_symlinks.js b/src/dev/run_check_lockfile_symlinks.js index 54a8cdf638a7..6c6fc54638ee 100644 --- a/src/dev/run_check_lockfile_symlinks.js +++ b/src/dev/run_check_lockfile_symlinks.js @@ -36,6 +36,8 @@ const IGNORE_FILE_GLOBS = [ '**/*fixtures*/**/*', // cypress isn't used in production, ignore it 'x-pack/legacy/plugins/apm/e2e/*', + // apm scripts aren't used in production, ignore them + 'x-pack/legacy/plugins/apm/scripts/*', ]; run(async ({ log }) => { diff --git a/x-pack/legacy/plugins/apm/index.ts b/x-pack/legacy/plugins/apm/index.ts index 6cfd18d0c1cb..502e910caae5 100644 --- a/x-pack/legacy/plugins/apm/index.ts +++ b/x-pack/legacy/plugins/apm/index.ts @@ -14,7 +14,13 @@ import mappings from './mappings.json'; export const apm: LegacyPluginInitializer = kibana => { return new kibana.Plugin({ - require: ['kibana', 'elasticsearch', 'xpack_main', 'apm_oss'], + require: [ + 'kibana', + 'elasticsearch', + 'xpack_main', + 'apm_oss', + 'task_manager' + ], id: 'apm', configPrefix: 'xpack.apm', publicDir: resolve(__dirname, 'public'), @@ -76,7 +82,10 @@ export const apm: LegacyPluginInitializer = kibana => { serviceMapTraceIdBucketSize: Joi.number().default(65), serviceMapFingerprintGlobalBucketSize: Joi.number().default(1000), serviceMapTraceIdGlobalBucketSize: Joi.number().default(6), - serviceMapMaxTracesPerRequest: Joi.number().default(50) + serviceMapMaxTracesPerRequest: Joi.number().default(50), + + // telemetry + telemetryCollectionEnabled: Joi.boolean().default(true) }).default(); }, @@ -122,10 +131,12 @@ export const apm: LegacyPluginInitializer = kibana => { } } }); - const apmPlugin = server.newPlatform.setup.plugins .apm as APMPluginContract; - apmPlugin.registerLegacyAPI({ server }); + + apmPlugin.registerLegacyAPI({ + server + }); } }); }; diff --git a/x-pack/legacy/plugins/apm/mappings.json b/x-pack/legacy/plugins/apm/mappings.json index 61bc90da2875..ba4c7a89ceaa 100644 --- a/x-pack/legacy/plugins/apm/mappings.json +++ b/x-pack/legacy/plugins/apm/mappings.json @@ -1,20 +1,659 @@ { - "apm-services-telemetry": { + "apm-telemetry": { "properties": { + "agents": { + "properties": { + "dotnet": { + "properties": { + "agent": { + "properties": { + "version": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "service": { + "properties": { + "framework": { + "properties": { + "composite": { + "type": "keyword", + "ignore_above": 256 + }, + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "version": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "language": { + "properties": { + "name": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "runtime": { + "properties": { + "composite": { + "type": "keyword", + "ignore_above": 256 + }, + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "version": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + } + }, + "go": { + "properties": { + "agent": { + "properties": { + "version": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "service": { + "properties": { + "framework": { + "properties": { + "composite": { + "type": "keyword", + "ignore_above": 256 + }, + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "version": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "language": { + "properties": { + "composite": { + "type": "keyword", + "ignore_above": 256 + }, + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "version": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "runtime": { + "properties": { + "composite": { + "type": "keyword", + "ignore_above": 256 + }, + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "version": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + } + }, + "java": { + "properties": { + "agent": { + "properties": { + "version": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "service": { + "properties": { + "framework": { + "type": "object" + }, + "language": { + "properties": { + "composite": { + "type": "keyword", + "ignore_above": 256 + }, + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "version": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "runtime": { + "properties": { + "composite": { + "type": "keyword", + "ignore_above": 256 + }, + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "version": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + } + }, + "js-base": { + "properties": { + "agent": { + "properties": { + "version": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "service": { + "properties": { + "framework": { + "type": "object" + }, + "language": { + "properties": { + "name": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "runtime": { + "type": "object" + } + } + } + } + }, + "nodejs": { + "properties": { + "agent": { + "properties": { + "version": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "service": { + "properties": { + "framework": { + "properties": { + "composite": { + "type": "keyword", + "ignore_above": 256 + }, + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "version": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "language": { + "properties": { + "name": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "runtime": { + "properties": { + "composite": { + "type": "keyword", + "ignore_above": 256 + }, + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "version": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + } + }, + "python": { + "properties": { + "agent": { + "properties": { + "version": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "service": { + "properties": { + "framework": { + "properties": { + "composite": { + "type": "keyword", + "ignore_above": 256 + }, + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "version": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "language": { + "properties": { + "composite": { + "type": "keyword", + "ignore_above": 256 + }, + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "version": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "runtime": { + "properties": { + "composite": { + "type": "keyword", + "ignore_above": 256 + }, + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "version": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + } + }, + "ruby": { + "properties": { + "agent": { + "properties": { + "version": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "service": { + "properties": { + "framework": { + "properties": { + "composite": { + "type": "keyword", + "ignore_above": 256 + }, + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "version": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "language": { + "properties": { + "composite": { + "type": "keyword", + "ignore_above": 256 + }, + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "version": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "runtime": { + "properties": { + "composite": { + "type": "keyword", + "ignore_above": 256 + }, + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "version": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + } + }, + "rum-js": { + "properties": { + "agent": { + "type": "object" + }, + "service": { + "properties": { + "framework": { + "type": "object" + }, + "language": { + "type": "object" + }, + "runtime": { + "type": "object" + } + } + } + } + } + } + }, + "counts": { + "properties": { + "agent_configuration": { + "properties": { + "all": { + "type": "long" + } + } + }, + "error": { + "properties": { + "1d": { + "type": "long" + }, + "all": { + "type": "long" + } + } + }, + "max_error_groups_per_service": { + "properties": { + "1d": { + "type": "long" + } + } + }, + "max_transaction_groups_per_service": { + "properties": { + "1d": { + "type": "long" + } + } + }, + "metric": { + "properties": { + "1d": { + "type": "long" + }, + "all": { + "type": "long" + } + } + }, + "onboarding": { + "properties": { + "1d": { + "type": "long" + }, + "all": { + "type": "long" + } + } + }, + "services": { + "properties": { + "1d": { + "type": "long" + } + } + }, + "sourcemap": { + "properties": { + "1d": { + "type": "long" + }, + "all": { + "type": "long" + } + } + }, + "span": { + "properties": { + "1d": { + "type": "long" + }, + "all": { + "type": "long" + } + } + }, + "traces": { + "properties": { + "1d": { + "type": "long" + } + } + }, + "transaction": { + "properties": { + "1d": { + "type": "long" + }, + "all": { + "type": "long" + } + } + } + } + }, + "cardinality": { + "properties": { + "user_agent": { + "properties": { + "original": { + "properties": { + "all_agents": { + "properties": { + "1d": { + "type": "long" + } + } + }, + "rum": { + "properties": { + "1d": { + "type": "long" + } + } + } + } + } + } + }, + "transaction": { + "properties": { + "name": { + "properties": { + "all_agents": { + "properties": { + "1d": { + "type": "long" + } + } + }, + "rum": { + "properties": { + "1d": { + "type": "long" + } + } + } + } + } + } + } + } + }, "has_any_services": { "type": "boolean" }, + "indices": { + "properties": { + "all": { + "properties": { + "total": { + "properties": { + "docs": { + "properties": { + "count": { + "type": "long" + } + } + }, + "store": { + "properties": { + "size_in_bytes": { + "type": "long" + } + } + } + } + } + } + }, + "shards": { + "properties": { + "total": { + "type": "long" + } + } + } + } + }, + "integrations": { + "properties": { + "ml": { + "properties": { + "all_jobs_count": { + "type": "long" + } + } + } + } + }, + "retainment": { + "properties": { + "error": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "metric": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "onboarding": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "span": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "transaction": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, "services_per_agent": { "properties": { - "python": { + "dotnet": { "type": "long", "null_value": 0 }, - "java": { + "go": { "type": "long", "null_value": 0 }, - "nodejs": { + "java": { "type": "long", "null_value": 0 }, @@ -22,11 +661,11 @@ "type": "long", "null_value": 0 }, - "rum-js": { + "nodejs": { "type": "long", "null_value": 0 }, - "dotnet": { + "python": { "type": "long", "null_value": 0 }, @@ -34,11 +673,131 @@ "type": "long", "null_value": 0 }, - "go": { + "rum-js": { "type": "long", "null_value": 0 } } + }, + "tasks": { + "properties": { + "agent_configuration": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "agents": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "cardinality": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "groupings": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "indices_stats": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "integrations": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "processor_events": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "services": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "versions": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + } + } + }, + "version": { + "properties": { + "apm_server": { + "properties": { + "major": { + "type": "long" + }, + "minor": { + "type": "long" + }, + "patch": { + "type": "long" + } + } + } + } } } }, diff --git a/x-pack/legacy/plugins/apm/scripts/.gitignore b/x-pack/legacy/plugins/apm/scripts/.gitignore new file mode 100644 index 000000000000..8ee01d321b72 --- /dev/null +++ b/x-pack/legacy/plugins/apm/scripts/.gitignore @@ -0,0 +1 @@ +yarn.lock diff --git a/x-pack/legacy/plugins/apm/scripts/package.json b/x-pack/legacy/plugins/apm/scripts/package.json new file mode 100644 index 000000000000..9121449c5361 --- /dev/null +++ b/x-pack/legacy/plugins/apm/scripts/package.json @@ -0,0 +1,10 @@ +{ + "name": "apm-scripts", + "version": "1.0.0", + "main": "index.js", + "license": "MIT", + "dependencies": { + "@octokit/rest": "^16.35.0", + "console-stamp": "^0.2.9" + } +} diff --git a/x-pack/legacy/plugins/apm/scripts/setup-kibana-security.js b/x-pack/legacy/plugins/apm/scripts/setup-kibana-security.js index 825c1a526fcc..61ba2fdc7f7e 100644 --- a/x-pack/legacy/plugins/apm/scripts/setup-kibana-security.js +++ b/x-pack/legacy/plugins/apm/scripts/setup-kibana-security.js @@ -16,6 +16,7 @@ ******************************/ // compile typescript on the fly +// eslint-disable-next-line import/no-extraneous-dependencies require('@babel/register')({ extensions: ['.ts'], plugins: ['@babel/plugin-proposal-optional-chaining'], diff --git a/x-pack/legacy/plugins/apm/scripts/upload-telemetry-data.js b/x-pack/legacy/plugins/apm/scripts/upload-telemetry-data.js new file mode 100644 index 000000000000..a99651c62dd7 --- /dev/null +++ b/x-pack/legacy/plugins/apm/scripts/upload-telemetry-data.js @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +// compile typescript on the fly +// eslint-disable-next-line import/no-extraneous-dependencies +require('@babel/register')({ + extensions: ['.ts'], + plugins: [ + '@babel/plugin-proposal-optional-chaining', + '@babel/plugin-proposal-nullish-coalescing-operator' + ], + presets: [ + '@babel/typescript', + ['@babel/preset-env', { targets: { node: 'current' } }] + ] +}); + +require('./upload-telemetry-data/index.ts'); diff --git a/x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/download-telemetry-template.ts b/x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/download-telemetry-template.ts new file mode 100644 index 000000000000..dfed9223ef70 --- /dev/null +++ b/x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/download-telemetry-template.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +// @ts-ignore +import { Octokit } from '@octokit/rest'; + +export async function downloadTelemetryTemplate(octokit: Octokit) { + const file = await octokit.repos.getContents({ + owner: 'elastic', + repo: 'telemetry', + path: 'config/templates/xpack-phone-home.json', + // @ts-ignore + mediaType: { + format: 'application/vnd.github.VERSION.raw' + } + }); + + if (Array.isArray(file.data)) { + throw new Error('Expected single response, got array'); + } + + return JSON.parse(Buffer.from(file.data.content!, 'base64').toString()); +} diff --git a/x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/generate-sample-documents.ts b/x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/generate-sample-documents.ts new file mode 100644 index 000000000000..8d76063a7fdf --- /dev/null +++ b/x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/generate-sample-documents.ts @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { DeepPartial } from 'utility-types'; +import { + merge, + omit, + defaultsDeep, + range, + mapValues, + isPlainObject, + flatten +} from 'lodash'; +import uuid from 'uuid'; +import { + CollectTelemetryParams, + collectDataTelemetry + // eslint-disable-next-line @kbn/eslint/no-restricted-paths +} from '../../../../../plugins/apm/server/lib/apm_telemetry/collect_data_telemetry'; + +interface GenerateOptions { + days: number; + instances: number; + variation: { + min: number; + max: number; + }; +} + +const randomize = ( + value: unknown, + instanceVariation: number, + dailyGrowth: number +) => { + if (typeof value === 'boolean') { + return Math.random() > 0.5; + } + if (typeof value === 'number') { + return Math.round(instanceVariation * dailyGrowth * value); + } + return value; +}; + +const mapValuesDeep = ( + obj: Record, + iterator: (value: unknown, key: string, obj: Record) => unknown +): Record => + mapValues(obj, (val, key) => + isPlainObject(val) ? mapValuesDeep(val, iterator) : iterator(val, key!, obj) + ); + +export async function generateSampleDocuments( + options: DeepPartial & { + collectTelemetryParams: CollectTelemetryParams; + } +) { + const { collectTelemetryParams, ...preferredOptions } = options; + + const opts: GenerateOptions = defaultsDeep( + { + days: 100, + instances: 50, + variation: { + min: 0.1, + max: 4 + } + }, + preferredOptions + ); + + const sample = await collectDataTelemetry(collectTelemetryParams); + + console.log('Collected telemetry'); // eslint-disable-line no-console + console.log('\n' + JSON.stringify(sample, null, 2)); // eslint-disable-line no-console + + const dateOfScriptExecution = new Date(); + + return flatten( + range(0, opts.instances).map(instanceNo => { + const instanceId = uuid.v4(); + const defaults = { + cluster_uuid: instanceId, + stack_stats: { + kibana: { + versions: { + version: '8.0.0' + } + } + } + }; + + const instanceVariation = + Math.random() * (opts.variation.max - opts.variation.min) + + opts.variation.min; + + return range(0, opts.days).map(dayNo => { + const dailyGrowth = Math.pow(1.005, opts.days - 1 - dayNo); + + const timestamp = Date.UTC( + dateOfScriptExecution.getFullYear(), + dateOfScriptExecution.getMonth(), + -dayNo + ); + + const generated = mapValuesDeep(omit(sample, 'versions'), value => + randomize(value, instanceVariation, dailyGrowth) + ); + + return merge({}, defaults, { + timestamp, + stack_stats: { + kibana: { + plugins: { + apm: merge({}, sample, generated) + } + } + } + }); + }); + }) + ); +} diff --git a/x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/index.ts b/x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/index.ts new file mode 100644 index 000000000000..bdc57eac412f --- /dev/null +++ b/x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/index.ts @@ -0,0 +1,208 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +// This script downloads the telemetry mapping, runs the APM telemetry tasks, +// generates a bunch of randomized data based on the downloaded sample, +// and uploads it to a cluster of your choosing in the same format as it is +// stored in the telemetry cluster. Its purpose is twofold: +// - Easier testing of the telemetry tasks +// - Validate whether we can run the queries we want to on the telemetry data + +import fs from 'fs'; +import path from 'path'; +// @ts-ignore +import { Octokit } from '@octokit/rest'; +import { merge, chunk, flatten, pick, identity } from 'lodash'; +import axios from 'axios'; +import yaml from 'js-yaml'; +import { Client } from 'elasticsearch'; +import { argv } from 'yargs'; +import { promisify } from 'util'; +import { Logger } from 'kibana/server'; +// @ts-ignore +import consoleStamp from 'console-stamp'; +// eslint-disable-next-line @kbn/eslint/no-restricted-paths +import { CollectTelemetryParams } from '../../../../../plugins/apm/server/lib/apm_telemetry/collect_data_telemetry'; +import { downloadTelemetryTemplate } from './download-telemetry-template'; +import mapping from '../../mappings.json'; +import { generateSampleDocuments } from './generate-sample-documents'; + +consoleStamp(console, '[HH:MM:ss.l]'); + +const githubToken = process.env.GITHUB_TOKEN; + +if (!githubToken) { + throw new Error('GITHUB_TOKEN was not provided.'); +} + +const kibanaConfigDir = path.join(__filename, '../../../../../../../config'); +const kibanaDevConfig = path.join(kibanaConfigDir, 'kibana.dev.yml'); +const kibanaConfig = path.join(kibanaConfigDir, 'kibana.yml'); + +const xpackTelemetryIndexName = 'xpack-phone-home'; + +const loadedKibanaConfig = (yaml.safeLoad( + fs.readFileSync( + fs.existsSync(kibanaDevConfig) ? kibanaDevConfig : kibanaConfig, + 'utf8' + ) +) || {}) as {}; + +const cliEsCredentials = pick( + { + 'elasticsearch.username': process.env.ELASTICSEARCH_USERNAME, + 'elasticsearch.password': process.env.ELASTICSEARCH_PASSWORD, + 'elasticsearch.hosts': process.env.ELASTICSEARCH_HOST + }, + identity +) as { + 'elasticsearch.username': string; + 'elasticsearch.password': string; + 'elasticsearch.hosts': string; +}; + +const config = { + 'apm_oss.transactionIndices': 'apm-*', + 'apm_oss.metricsIndices': 'apm-*', + 'apm_oss.errorIndices': 'apm-*', + 'apm_oss.spanIndices': 'apm-*', + 'apm_oss.onboardingIndices': 'apm-*', + 'apm_oss.sourcemapIndices': 'apm-*', + 'elasticsearch.hosts': 'http://localhost:9200', + ...loadedKibanaConfig, + ...cliEsCredentials +}; + +async function uploadData() { + const octokit = new Octokit({ + auth: githubToken + }); + + const telemetryTemplate = await downloadTelemetryTemplate(octokit); + + const kibanaMapping = mapping['apm-telemetry']; + + const httpAuth = + config['elasticsearch.username'] && config['elasticsearch.password'] + ? { + username: config['elasticsearch.username'], + password: config['elasticsearch.password'] + } + : null; + + const client = new Client({ + host: config['elasticsearch.hosts'], + ...(httpAuth + ? { + httpAuth: `${httpAuth.username}:${httpAuth.password}` + } + : {}) + }); + + if (argv.clear) { + try { + await promisify(client.indices.delete.bind(client))({ + index: xpackTelemetryIndexName + }); + } catch (err) { + // 404 = index not found, totally okay + if (err.status !== 404) { + throw err; + } + } + } + + const axiosInstance = axios.create({ + baseURL: config['elasticsearch.hosts'], + ...(httpAuth ? { auth: httpAuth } : {}) + }); + + const newTemplate = merge(telemetryTemplate, { + settings: { + index: { mapping: { total_fields: { limit: 10000 } } } + } + }); + + // override apm mapping instead of merging + newTemplate.mappings.properties.stack_stats.properties.kibana.properties.plugins.properties.apm = kibanaMapping; + + await axiosInstance.put(`/_template/xpack-phone-home`, newTemplate); + + const sampleDocuments = await generateSampleDocuments({ + collectTelemetryParams: { + logger: (console as unknown) as Logger, + indices: { + ...config, + apmCustomLinkIndex: '.apm-custom-links', + apmAgentConfigurationIndex: '.apm-agent-configuration' + }, + search: body => { + return promisify(client.search.bind(client))({ + ...body, + requestTimeout: 120000 + }) as any; + }, + indicesStats: body => { + return promisify(client.indices.stats.bind(client))({ + ...body, + requestTimeout: 120000 + }) as any; + }, + transportRequest: (params => { + return axiosInstance[params.method](params.path); + }) as CollectTelemetryParams['transportRequest'] + } + }); + + const chunks = chunk(sampleDocuments, 250); + + await chunks.reduce>((prev, documents) => { + return prev.then(async () => { + const body = flatten( + documents.map(doc => [{ index: { _index: 'xpack-phone-home' } }, doc]) + ); + + return promisify(client.bulk.bind(client))({ + body, + refresh: true + }).then((response: any) => { + if (response.errors) { + const firstError = response.items.filter( + (item: any) => item.index.status >= 400 + )[0].index.error; + throw new Error(`Failed to upload documents: ${firstError.reason} `); + } + }); + }); + }, Promise.resolve()); +} + +uploadData() + .catch(e => { + if ('response' in e) { + if (typeof e.response === 'string') { + // eslint-disable-next-line no-console + console.log(e.response); + } else { + // eslint-disable-next-line no-console + console.log( + JSON.stringify( + e.response, + ['status', 'statusText', 'headers', 'data'], + 2 + ) + ); + } + } else { + // eslint-disable-next-line no-console + console.log(e); + } + process.exit(1); + }) + .then(() => { + // eslint-disable-next-line no-console + console.log('Finished uploading generated telemetry data'); + }); diff --git a/x-pack/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap b/x-pack/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap index 897d4e979fce..5de82a9ee878 100644 --- a/x-pack/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap +++ b/x-pack/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap @@ -2,6 +2,8 @@ exports[`Error AGENT_NAME 1`] = `"java"`; +exports[`Error AGENT_VERSION 1`] = `"agent version"`; + exports[`Error CLIENT_GEO_COUNTRY_ISO_CODE 1`] = `undefined`; exports[`Error CONTAINER_ID 1`] = `undefined`; @@ -56,7 +58,7 @@ exports[`Error METRIC_SYSTEM_TOTAL_MEMORY 1`] = `undefined`; exports[`Error OBSERVER_LISTENING 1`] = `undefined`; -exports[`Error OBSERVER_VERSION_MAJOR 1`] = `undefined`; +exports[`Error OBSERVER_VERSION_MAJOR 1`] = `8`; exports[`Error PARENT_ID 1`] = `"parentId"`; @@ -68,10 +70,20 @@ exports[`Error SERVICE_ENVIRONMENT 1`] = `undefined`; exports[`Error SERVICE_FRAMEWORK_NAME 1`] = `undefined`; +exports[`Error SERVICE_FRAMEWORK_VERSION 1`] = `undefined`; + +exports[`Error SERVICE_LANGUAGE_NAME 1`] = `"nodejs"`; + +exports[`Error SERVICE_LANGUAGE_VERSION 1`] = `"v1337"`; + exports[`Error SERVICE_NAME 1`] = `"service name"`; exports[`Error SERVICE_NODE_NAME 1`] = `undefined`; +exports[`Error SERVICE_RUNTIME_NAME 1`] = `undefined`; + +exports[`Error SERVICE_RUNTIME_VERSION 1`] = `undefined`; + exports[`Error SERVICE_VERSION 1`] = `undefined`; exports[`Error SPAN_ACTION 1`] = `undefined`; @@ -112,10 +124,14 @@ exports[`Error URL_FULL 1`] = `undefined`; exports[`Error USER_AGENT_NAME 1`] = `undefined`; +exports[`Error USER_AGENT_ORIGINAL 1`] = `undefined`; + exports[`Error USER_ID 1`] = `undefined`; exports[`Span AGENT_NAME 1`] = `"java"`; +exports[`Span AGENT_VERSION 1`] = `"agent version"`; + exports[`Span CLIENT_GEO_COUNTRY_ISO_CODE 1`] = `undefined`; exports[`Span CONTAINER_ID 1`] = `undefined`; @@ -170,7 +186,7 @@ exports[`Span METRIC_SYSTEM_TOTAL_MEMORY 1`] = `undefined`; exports[`Span OBSERVER_LISTENING 1`] = `undefined`; -exports[`Span OBSERVER_VERSION_MAJOR 1`] = `undefined`; +exports[`Span OBSERVER_VERSION_MAJOR 1`] = `8`; exports[`Span PARENT_ID 1`] = `"parentId"`; @@ -182,10 +198,20 @@ exports[`Span SERVICE_ENVIRONMENT 1`] = `undefined`; exports[`Span SERVICE_FRAMEWORK_NAME 1`] = `undefined`; +exports[`Span SERVICE_FRAMEWORK_VERSION 1`] = `undefined`; + +exports[`Span SERVICE_LANGUAGE_NAME 1`] = `undefined`; + +exports[`Span SERVICE_LANGUAGE_VERSION 1`] = `undefined`; + exports[`Span SERVICE_NAME 1`] = `"service name"`; exports[`Span SERVICE_NODE_NAME 1`] = `undefined`; +exports[`Span SERVICE_RUNTIME_NAME 1`] = `undefined`; + +exports[`Span SERVICE_RUNTIME_VERSION 1`] = `undefined`; + exports[`Span SERVICE_VERSION 1`] = `undefined`; exports[`Span SPAN_ACTION 1`] = `"my action"`; @@ -226,10 +252,14 @@ exports[`Span URL_FULL 1`] = `undefined`; exports[`Span USER_AGENT_NAME 1`] = `undefined`; +exports[`Span USER_AGENT_ORIGINAL 1`] = `undefined`; + exports[`Span USER_ID 1`] = `undefined`; exports[`Transaction AGENT_NAME 1`] = `"java"`; +exports[`Transaction AGENT_VERSION 1`] = `"agent version"`; + exports[`Transaction CLIENT_GEO_COUNTRY_ISO_CODE 1`] = `undefined`; exports[`Transaction CONTAINER_ID 1`] = `"container1234567890abcdef"`; @@ -284,7 +314,7 @@ exports[`Transaction METRIC_SYSTEM_TOTAL_MEMORY 1`] = `undefined`; exports[`Transaction OBSERVER_LISTENING 1`] = `undefined`; -exports[`Transaction OBSERVER_VERSION_MAJOR 1`] = `undefined`; +exports[`Transaction OBSERVER_VERSION_MAJOR 1`] = `8`; exports[`Transaction PARENT_ID 1`] = `"parentId"`; @@ -296,10 +326,20 @@ exports[`Transaction SERVICE_ENVIRONMENT 1`] = `undefined`; exports[`Transaction SERVICE_FRAMEWORK_NAME 1`] = `undefined`; +exports[`Transaction SERVICE_FRAMEWORK_VERSION 1`] = `undefined`; + +exports[`Transaction SERVICE_LANGUAGE_NAME 1`] = `"nodejs"`; + +exports[`Transaction SERVICE_LANGUAGE_VERSION 1`] = `"v1337"`; + exports[`Transaction SERVICE_NAME 1`] = `"service name"`; exports[`Transaction SERVICE_NODE_NAME 1`] = `undefined`; +exports[`Transaction SERVICE_RUNTIME_NAME 1`] = `undefined`; + +exports[`Transaction SERVICE_RUNTIME_VERSION 1`] = `undefined`; + exports[`Transaction SERVICE_VERSION 1`] = `undefined`; exports[`Transaction SPAN_ACTION 1`] = `undefined`; @@ -340,4 +380,6 @@ exports[`Transaction URL_FULL 1`] = `"http://www.elastic.co"`; exports[`Transaction USER_AGENT_NAME 1`] = `"Other"`; +exports[`Transaction USER_AGENT_ORIGINAL 1`] = `"test original"`; + exports[`Transaction USER_ID 1`] = `"1337"`; diff --git a/x-pack/plugins/apm/common/agent_name.ts b/x-pack/plugins/apm/common/agent_name.ts index bb68eb88b8e1..085828b729ea 100644 --- a/x-pack/plugins/apm/common/agent_name.ts +++ b/x-pack/plugins/apm/common/agent_name.ts @@ -4,36 +4,40 @@ * you may not use this file except in compliance with the Elastic License. */ +import { AgentName } from '../typings/es_schemas/ui/fields/agent'; + /* * Agent names can be any string. This list only defines the official agents * that we might want to target specifically eg. linking to their documentation * & telemetry reporting. Support additional agent types by appending * definitions in mappings.json (for telemetry), the AgentName type, and the - * agentNames object. + * AGENT_NAMES array. */ -import { AgentName } from '../typings/es_schemas/ui/fields/agent'; -const agentNames: { [agentName in AgentName]: agentName } = { - python: 'python', - java: 'java', - nodejs: 'nodejs', - 'js-base': 'js-base', - 'rum-js': 'rum-js', - dotnet: 'dotnet', - ruby: 'ruby', - go: 'go' -}; +export const AGENT_NAMES: AgentName[] = [ + 'java', + 'js-base', + 'rum-js', + 'dotnet', + 'go', + 'java', + 'nodejs', + 'python', + 'ruby' +]; -export function isAgentName(agentName: string): boolean { - return Object.values(agentNames).includes(agentName as AgentName); +export function isAgentName(agentName: string): agentName is AgentName { + return AGENT_NAMES.includes(agentName as AgentName); } -export function isRumAgentName(agentName: string | undefined) { - return ( - agentName === agentNames['js-base'] || agentName === agentNames['rum-js'] - ); +export function isRumAgentName( + agentName: string | undefined +): agentName is 'js-base' | 'rum-js' { + return agentName === 'js-base' || agentName === 'rum-js'; } -export function isJavaAgentName(agentName: string | undefined) { - return agentName === agentNames.java; +export function isJavaAgentName( + agentName: string | undefined +): agentName is 'java' { + return agentName === 'java'; } diff --git a/x-pack/plugins/apm/common/apm_saved_object_constants.ts b/x-pack/plugins/apm/common/apm_saved_object_constants.ts index ac43b700117c..0529d90fe940 100644 --- a/x-pack/plugins/apm/common/apm_saved_object_constants.ts +++ b/x-pack/plugins/apm/common/apm_saved_object_constants.ts @@ -4,11 +4,13 @@ * you may not use this file except in compliance with the Elastic License. */ -// APM Services telemetry -export const APM_SERVICES_TELEMETRY_SAVED_OBJECT_TYPE = - 'apm-services-telemetry'; -export const APM_SERVICES_TELEMETRY_SAVED_OBJECT_ID = 'apm-services-telemetry'; +// the types have to match the names of the saved object mappings +// in /x-pack/legacy/plugins/apm/mappings.json // APM indices export const APM_INDICES_SAVED_OBJECT_TYPE = 'apm-indices'; export const APM_INDICES_SAVED_OBJECT_ID = 'apm-indices'; + +// APM telemetry +export const APM_TELEMETRY_SAVED_OBJECT_TYPE = 'apm-telemetry'; +export const APM_TELEMETRY_SAVED_OBJECT_ID = 'apm-telemetry'; diff --git a/x-pack/plugins/apm/common/elasticsearch_fieldnames.test.ts b/x-pack/plugins/apm/common/elasticsearch_fieldnames.test.ts index 1add2427d16a..63fa749cd9f2 100644 --- a/x-pack/plugins/apm/common/elasticsearch_fieldnames.test.ts +++ b/x-pack/plugins/apm/common/elasticsearch_fieldnames.test.ts @@ -15,7 +15,10 @@ describe('Transaction', () => { const transaction: AllowUnknownProperties = { '@timestamp': new Date().toString(), '@metadata': 'whatever', - observer: 'whatever', + observer: { + version: 'whatever', + version_major: 8 + }, agent: { name: 'java', version: 'agent version' @@ -63,7 +66,10 @@ describe('Span', () => { const span: AllowUnknownProperties = { '@timestamp': new Date().toString(), '@metadata': 'whatever', - observer: 'whatever', + observer: { + version: 'whatever', + version_major: 8 + }, agent: { name: 'java', version: 'agent version' @@ -107,7 +113,10 @@ describe('Span', () => { describe('Error', () => { const errorDoc: AllowUnknownProperties = { '@metadata': 'whatever', - observer: 'whatever', + observer: { + version: 'whatever', + version_major: 8 + }, agent: { name: 'java', version: 'agent version' diff --git a/x-pack/plugins/apm/common/elasticsearch_fieldnames.ts b/x-pack/plugins/apm/common/elasticsearch_fieldnames.ts index 822201baddd8..bc1b346f50da 100644 --- a/x-pack/plugins/apm/common/elasticsearch_fieldnames.ts +++ b/x-pack/plugins/apm/common/elasticsearch_fieldnames.ts @@ -4,15 +4,24 @@ * you may not use this file except in compliance with the Elastic License. */ -export const AGENT_NAME = 'agent.name'; export const SERVICE_NAME = 'service.name'; export const SERVICE_ENVIRONMENT = 'service.environment'; export const SERVICE_FRAMEWORK_NAME = 'service.framework.name'; +export const SERVICE_FRAMEWORK_VERSION = 'service.framework.version'; +export const SERVICE_LANGUAGE_NAME = 'service.language.name'; +export const SERVICE_LANGUAGE_VERSION = 'service.language.version'; +export const SERVICE_RUNTIME_NAME = 'service.runtime.name'; +export const SERVICE_RUNTIME_VERSION = 'service.runtime.version'; export const SERVICE_NODE_NAME = 'service.node.name'; export const SERVICE_VERSION = 'service.version'; + +export const AGENT_NAME = 'agent.name'; +export const AGENT_VERSION = 'agent.version'; + export const URL_FULL = 'url.full'; export const HTTP_REQUEST_METHOD = 'http.request.method'; export const USER_ID = 'user.id'; +export const USER_AGENT_ORIGINAL = 'user_agent.original'; export const USER_AGENT_NAME = 'user_agent.name'; export const DESTINATION_ADDRESS = 'destination.address'; diff --git a/x-pack/plugins/apm/kibana.json b/x-pack/plugins/apm/kibana.json index 931fd92e1ecc..7ffdb676c740 100644 --- a/x-pack/plugins/apm/kibana.json +++ b/x-pack/plugins/apm/kibana.json @@ -3,7 +3,10 @@ "server": true, "version": "8.0.0", "kibanaVersion": "kibana", - "configPath": ["xpack", "apm"], + "configPath": [ + "xpack", + "apm" + ], "ui": false, "requiredPlugins": ["apm_oss", "data", "home", "licensing"], "optionalPlugins": ["cloud", "usageCollection", "taskManager","actions", "alerting"] diff --git a/x-pack/plugins/apm/server/index.ts b/x-pack/plugins/apm/server/index.ts index 8afdb9e99c1a..77655568a7e9 100644 --- a/x-pack/plugins/apm/server/index.ts +++ b/x-pack/plugins/apm/server/index.ts @@ -29,7 +29,8 @@ export const config = { enabled: schema.boolean({ defaultValue: true }), transactionGroupBucketSize: schema.number({ defaultValue: 100 }), maxTraceItems: schema.number({ defaultValue: 1000 }) - }) + }), + telemetryCollectionEnabled: schema.boolean({ defaultValue: true }) }) }; @@ -62,7 +63,8 @@ export function mergeConfigs( 'xpack.apm.ui.maxTraceItems': apmConfig.ui.maxTraceItems, 'xpack.apm.ui.transactionGroupBucketSize': apmConfig.ui.transactionGroupBucketSize, - 'xpack.apm.autocreateApmIndexPattern': apmConfig.autocreateApmIndexPattern + 'xpack.apm.autocreateApmIndexPattern': apmConfig.autocreateApmIndexPattern, + 'xpack.apm.telemetryCollectionEnabled': apmConfig.telemetryCollectionEnabled }; } diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/__test__/index.test.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/__test__/index.test.ts deleted file mode 100644 index c45c74a791ae..000000000000 --- a/x-pack/plugins/apm/server/lib/apm_telemetry/__test__/index.test.ts +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { SavedObjectAttributes } from '../../../../../../../src/core/server'; -import { createApmTelementry, storeApmServicesTelemetry } from '../index'; -import { - APM_SERVICES_TELEMETRY_SAVED_OBJECT_TYPE, - APM_SERVICES_TELEMETRY_SAVED_OBJECT_ID -} from '../../../../common/apm_saved_object_constants'; - -describe('apm_telemetry', () => { - describe('createApmTelementry', () => { - it('should create a ApmTelemetry object with boolean flag and frequency map of the given list of AgentNames', () => { - const apmTelemetry = createApmTelementry([ - 'go', - 'nodejs', - 'go', - 'js-base' - ]); - expect(apmTelemetry.has_any_services).toBe(true); - expect(apmTelemetry.services_per_agent).toMatchObject({ - go: 2, - nodejs: 1, - 'js-base': 1 - }); - }); - it('should ignore undefined or unknown AgentName values', () => { - const apmTelemetry = createApmTelementry([ - 'go', - 'nodejs', - 'go', - 'js-base', - 'example-platform' as any, - undefined as any - ]); - expect(apmTelemetry.services_per_agent).toMatchObject({ - go: 2, - nodejs: 1, - 'js-base': 1 - }); - }); - }); - - describe('storeApmServicesTelemetry', () => { - let apmTelemetry: SavedObjectAttributes; - let savedObjectsClient: any; - - beforeEach(() => { - apmTelemetry = { - has_any_services: true, - services_per_agent: { - go: 2, - nodejs: 1, - 'js-base': 1 - } - }; - savedObjectsClient = { create: jest.fn() }; - }); - - it('should call savedObjectsClient create with the given ApmTelemetry object', () => { - storeApmServicesTelemetry(savedObjectsClient, apmTelemetry); - expect(savedObjectsClient.create.mock.calls[0][1]).toBe(apmTelemetry); - }); - - it('should call savedObjectsClient create with the apm-telemetry document type and ID', () => { - storeApmServicesTelemetry(savedObjectsClient, apmTelemetry); - expect(savedObjectsClient.create.mock.calls[0][0]).toBe( - APM_SERVICES_TELEMETRY_SAVED_OBJECT_TYPE - ); - expect(savedObjectsClient.create.mock.calls[0][2].id).toBe( - APM_SERVICES_TELEMETRY_SAVED_OBJECT_ID - ); - }); - - it('should call savedObjectsClient create with overwrite: true', () => { - storeApmServicesTelemetry(savedObjectsClient, apmTelemetry); - expect(savedObjectsClient.create.mock.calls[0][2].overwrite).toBe(true); - }); - }); -}); diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/index.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/index.ts new file mode 100644 index 000000000000..729ccb73d73f --- /dev/null +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/index.ts @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { merge } from 'lodash'; +import { Logger, CallAPIOptions } from 'kibana/server'; +import { IndicesStatsParams, Client } from 'elasticsearch'; +import { + ESSearchRequest, + ESSearchResponse +} from '../../../../typings/elasticsearch'; +import { ApmIndicesConfig } from '../../settings/apm_indices/get_apm_indices'; +import { tasks } from './tasks'; +import { APMDataTelemetry } from '../types'; + +type TelemetryTaskExecutor = (params: { + indices: ApmIndicesConfig; + search( + params: TSearchRequest + ): Promise>; + indicesStats( + params: IndicesStatsParams, + options?: CallAPIOptions + ): ReturnType; + transportRequest: (params: { + path: string; + method: 'get'; + }) => Promise; +}) => Promise; + +export interface TelemetryTask { + name: string; + executor: TelemetryTaskExecutor; +} + +export type CollectTelemetryParams = Parameters[0] & { + logger: Logger; +}; + +export function collectDataTelemetry({ + search, + indices, + logger, + indicesStats, + transportRequest +}: CollectTelemetryParams) { + return tasks.reduce((prev, task) => { + return prev.then(async data => { + logger.debug(`Executing APM telemetry task ${task.name}`); + try { + const time = process.hrtime(); + const next = await task.executor({ + search, + indices, + indicesStats, + transportRequest + }); + const took = process.hrtime(time); + + return merge({}, data, next, { + tasks: { + [task.name]: { + took: { + ms: Math.round(took[0] * 1000 + took[1] / 1e6) + } + } + } + }); + } catch (err) { + logger.warn(`Failed executing APM telemetry task ${task.name}`); + logger.warn(err); + return data; + } + }); + }, Promise.resolve({} as APMDataTelemetry)); +} diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts new file mode 100644 index 000000000000..415076b6ae11 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts @@ -0,0 +1,725 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { flatten, merge, sortBy, sum } from 'lodash'; +import { AgentName } from '../../../../typings/es_schemas/ui/fields/agent'; +import { AGENT_NAMES } from '../../../../common/agent_name'; +import { Transaction } from '../../../../typings/es_schemas/ui/transaction'; +import { + PROCESSOR_EVENT, + SERVICE_NAME, + AGENT_NAME, + AGENT_VERSION, + ERROR_GROUP_ID, + TRANSACTION_NAME, + PARENT_ID, + SERVICE_FRAMEWORK_NAME, + SERVICE_FRAMEWORK_VERSION, + SERVICE_LANGUAGE_NAME, + SERVICE_LANGUAGE_VERSION, + SERVICE_RUNTIME_NAME, + SERVICE_RUNTIME_VERSION, + USER_AGENT_ORIGINAL +} from '../../../../common/elasticsearch_fieldnames'; +import { Span } from '../../../../typings/es_schemas/ui/span'; +import { APMError } from '../../../../typings/es_schemas/ui/apm_error'; +import { TelemetryTask } from '.'; +import { APMTelemetry } from '../types'; + +const TIME_RANGES = ['1d', 'all'] as const; +type TimeRange = typeof TIME_RANGES[number]; + +export const tasks: TelemetryTask[] = [ + { + name: 'processor_events', + executor: async ({ indices, search }) => { + const indicesByProcessorEvent = { + error: indices['apm_oss.errorIndices'], + metric: indices['apm_oss.metricsIndices'], + span: indices['apm_oss.spanIndices'], + transaction: indices['apm_oss.transactionIndices'], + onboarding: indices['apm_oss.onboardingIndices'], + sourcemap: indices['apm_oss.sourcemapIndices'] + }; + + type ProcessorEvent = keyof typeof indicesByProcessorEvent; + + const jobs: Array<{ + processorEvent: ProcessorEvent; + timeRange: TimeRange; + }> = flatten( + (Object.keys( + indicesByProcessorEvent + ) as ProcessorEvent[]).map(processorEvent => + TIME_RANGES.map(timeRange => ({ processorEvent, timeRange })) + ) + ); + + const allData = await jobs.reduce((prevJob, current) => { + return prevJob.then(async data => { + const { processorEvent, timeRange } = current; + + const response = await search({ + index: indicesByProcessorEvent[processorEvent], + body: { + size: 1, + query: { + bool: { + filter: [ + { term: { [PROCESSOR_EVENT]: processorEvent } }, + ...(timeRange !== 'all' + ? [ + { + range: { + '@timestamp': { + gte: `now-${timeRange}` + } + } + } + ] + : []) + ] + } + }, + sort: { + '@timestamp': 'asc' + }, + _source: ['@timestamp'], + track_total_hits: true + } + }); + + const event = response.hits.hits[0]?._source as { + '@timestamp': number; + }; + + return merge({}, data, { + counts: { + [processorEvent]: { + [timeRange]: response.hits.total.value + } + }, + ...(timeRange === 'all' && event + ? { + retainment: { + [processorEvent]: { + ms: + new Date().getTime() - + new Date(event['@timestamp']).getTime() + } + } + } + : {}) + }); + }); + }, Promise.resolve({} as Record> }>)); + + return allData; + } + }, + { + name: 'agent_configuration', + executor: async ({ indices, search }) => { + const agentConfigurationCount = ( + await search({ + index: indices.apmAgentConfigurationIndex, + body: { + size: 0, + track_total_hits: true + } + }) + ).hits.total.value; + + return { + counts: { + agent_configuration: { + all: agentConfigurationCount + } + } + }; + } + }, + { + name: 'services', + executor: async ({ indices, search }) => { + const servicesPerAgent = await AGENT_NAMES.reduce( + (prevJob, agentName) => { + return prevJob.then(async data => { + const response = await search({ + index: [ + indices['apm_oss.errorIndices'], + indices['apm_oss.spanIndices'], + indices['apm_oss.metricsIndices'], + indices['apm_oss.transactionIndices'] + ], + body: { + size: 0, + query: { + bool: { + filter: [ + { + term: { + [AGENT_NAME]: agentName + } + }, + { + range: { + '@timestamp': { + gte: 'now-1d' + } + } + } + ] + } + }, + aggs: { + services: { + cardinality: { + field: SERVICE_NAME + } + } + } + } + }); + + return { + ...data, + [agentName]: response.aggregations?.services.value || 0 + }; + }); + }, + Promise.resolve({} as Record) + ); + + return { + has_any_services: sum(Object.values(servicesPerAgent)) > 0, + services_per_agent: servicesPerAgent + }; + } + }, + { + name: 'versions', + executor: async ({ search, indices }) => { + const response = await search({ + index: [ + indices['apm_oss.transactionIndices'], + indices['apm_oss.spanIndices'], + indices['apm_oss.errorIndices'] + ], + terminateAfter: 1, + body: { + query: { + exists: { + field: 'observer.version' + } + }, + size: 1, + sort: { + '@timestamp': 'desc' + } + } + }); + + const hit = response.hits.hits[0]?._source as Pick< + Transaction | Span | APMError, + 'observer' + >; + + if (!hit || !hit.observer?.version) { + return {}; + } + + const [major, minor, patch] = hit.observer.version + .split('.') + .map(part => Number(part)); + + return { + versions: { + apm_server: { + major, + minor, + patch + } + } + }; + } + }, + { + name: 'groupings', + executor: async ({ search, indices }) => { + const range1d = { range: { '@timestamp': { gte: 'now-1d' } } }; + const errorGroupsCount = ( + await search({ + index: indices['apm_oss.errorIndices'], + body: { + size: 0, + query: { + bool: { + filter: [{ term: { [PROCESSOR_EVENT]: 'error' } }, range1d] + } + }, + aggs: { + top_service: { + terms: { + field: SERVICE_NAME, + order: { + error_groups: 'desc' + }, + size: 1 + }, + aggs: { + error_groups: { + cardinality: { + field: ERROR_GROUP_ID + } + } + } + } + } + } + }) + ).aggregations?.top_service.buckets[0]?.error_groups.value; + + const transactionGroupsCount = ( + await search({ + index: indices['apm_oss.transactionIndices'], + body: { + size: 0, + query: { + bool: { + filter: [ + { term: { [PROCESSOR_EVENT]: 'transaction' } }, + range1d + ] + } + }, + aggs: { + top_service: { + terms: { + field: SERVICE_NAME, + order: { + transaction_groups: 'desc' + }, + size: 1 + }, + aggs: { + transaction_groups: { + cardinality: { + field: TRANSACTION_NAME + } + } + } + } + } + } + }) + ).aggregations?.top_service.buckets[0]?.transaction_groups.value; + + const tracesPerDayCount = ( + await search({ + index: indices['apm_oss.transactionIndices'], + body: { + query: { + bool: { + filter: [ + { term: { [PROCESSOR_EVENT]: 'transaction' } }, + range1d + ], + must_not: { + exists: { field: PARENT_ID } + } + } + }, + track_total_hits: true, + size: 0 + } + }) + ).hits.total.value; + + const servicesCount = ( + await search({ + index: [ + indices['apm_oss.transactionIndices'], + indices['apm_oss.errorIndices'], + indices['apm_oss.metricsIndices'] + ], + body: { + size: 0, + query: { + bool: { + filter: [range1d] + } + }, + aggs: { + service_name: { + cardinality: { + field: SERVICE_NAME + } + } + } + } + }) + ).aggregations?.service_name.value; + + return { + counts: { + max_error_groups_per_service: { + '1d': errorGroupsCount || 0 + }, + max_transaction_groups_per_service: { + '1d': transactionGroupsCount || 0 + }, + traces: { + '1d': tracesPerDayCount || 0 + }, + services: { + '1d': servicesCount || 0 + } + } + }; + } + }, + { + name: 'integrations', + executor: async ({ transportRequest }) => { + const apmJobs = ['*-high_mean_response_time']; + + const response = (await transportRequest({ + method: 'get', + path: `/_ml/anomaly_detectors/${apmJobs.join(',')}` + })) as { data?: { count: number } }; + + return { + integrations: { + ml: { + all_jobs_count: response.data?.count ?? 0 + } + } + }; + } + }, + { + name: 'agents', + executor: async ({ search, indices }) => { + const size = 3; + + const agentData = await AGENT_NAMES.reduce(async (prevJob, agentName) => { + const data = await prevJob; + + const response = await search({ + index: [ + indices['apm_oss.errorIndices'], + indices['apm_oss.metricsIndices'], + indices['apm_oss.transactionIndices'] + ], + body: { + size: 0, + query: { + bool: { + filter: [ + { term: { [AGENT_NAME]: agentName } }, + { range: { '@timestamp': { gte: 'now-1d' } } } + ] + } + }, + sort: { + '@timestamp': 'desc' + }, + aggs: { + [AGENT_VERSION]: { + terms: { + field: AGENT_VERSION, + size + } + }, + [SERVICE_FRAMEWORK_NAME]: { + terms: { + field: SERVICE_FRAMEWORK_NAME, + size + }, + aggs: { + [SERVICE_FRAMEWORK_VERSION]: { + terms: { + field: SERVICE_FRAMEWORK_VERSION, + size + } + } + } + }, + [SERVICE_FRAMEWORK_VERSION]: { + terms: { + field: SERVICE_FRAMEWORK_VERSION, + size + } + }, + [SERVICE_LANGUAGE_NAME]: { + terms: { + field: SERVICE_LANGUAGE_NAME, + size + }, + aggs: { + [SERVICE_LANGUAGE_VERSION]: { + terms: { + field: SERVICE_LANGUAGE_VERSION, + size + } + } + } + }, + [SERVICE_LANGUAGE_VERSION]: { + terms: { + field: SERVICE_LANGUAGE_VERSION, + size + } + }, + [SERVICE_RUNTIME_NAME]: { + terms: { + field: SERVICE_RUNTIME_NAME, + size + }, + aggs: { + [SERVICE_RUNTIME_VERSION]: { + terms: { + field: SERVICE_RUNTIME_VERSION, + size + } + } + } + }, + [SERVICE_RUNTIME_VERSION]: { + terms: { + field: SERVICE_RUNTIME_VERSION, + size + } + } + } + } + }); + + const { aggregations } = response; + + if (!aggregations) { + return data; + } + + const toComposite = ( + outerKey: string | number, + innerKey: string | number + ) => `${outerKey}/${innerKey}`; + + return { + ...data, + [agentName]: { + agent: { + version: aggregations[AGENT_VERSION].buckets.map( + bucket => bucket.key as string + ) + }, + service: { + framework: { + name: aggregations[SERVICE_FRAMEWORK_NAME].buckets + .map(bucket => bucket.key as string) + .slice(0, size), + version: aggregations[SERVICE_FRAMEWORK_VERSION].buckets + .map(bucket => bucket.key as string) + .slice(0, size), + composite: sortBy( + flatten( + aggregations[SERVICE_FRAMEWORK_NAME].buckets.map(bucket => + bucket[SERVICE_FRAMEWORK_VERSION].buckets.map( + versionBucket => ({ + doc_count: versionBucket.doc_count, + name: toComposite(bucket.key, versionBucket.key) + }) + ) + ) + ), + 'doc_count' + ) + .reverse() + .slice(0, size) + .map(composite => composite.name) + }, + language: { + name: aggregations[SERVICE_LANGUAGE_NAME].buckets + .map(bucket => bucket.key as string) + .slice(0, size), + version: aggregations[SERVICE_LANGUAGE_VERSION].buckets + .map(bucket => bucket.key as string) + .slice(0, size), + composite: sortBy( + flatten( + aggregations[SERVICE_LANGUAGE_NAME].buckets.map(bucket => + bucket[SERVICE_LANGUAGE_VERSION].buckets.map( + versionBucket => ({ + doc_count: versionBucket.doc_count, + name: toComposite(bucket.key, versionBucket.key) + }) + ) + ) + ), + 'doc_count' + ) + .reverse() + .slice(0, size) + .map(composite => composite.name) + }, + runtime: { + name: aggregations[SERVICE_RUNTIME_NAME].buckets + .map(bucket => bucket.key as string) + .slice(0, size), + version: aggregations[SERVICE_RUNTIME_VERSION].buckets + .map(bucket => bucket.key as string) + .slice(0, size), + composite: sortBy( + flatten( + aggregations[SERVICE_RUNTIME_NAME].buckets.map(bucket => + bucket[SERVICE_RUNTIME_VERSION].buckets.map( + versionBucket => ({ + doc_count: versionBucket.doc_count, + name: toComposite(bucket.key, versionBucket.key) + }) + ) + ) + ), + 'doc_count' + ) + .reverse() + .slice(0, size) + .map(composite => composite.name) + } + } + } + }; + }, Promise.resolve({} as APMTelemetry['agents'])); + + return { + agents: agentData + }; + } + }, + { + name: 'indices_stats', + executor: async ({ indicesStats, indices }) => { + const response = await indicesStats({ + index: [ + indices.apmAgentConfigurationIndex, + indices['apm_oss.errorIndices'], + indices['apm_oss.metricsIndices'], + indices['apm_oss.onboardingIndices'], + indices['apm_oss.sourcemapIndices'], + indices['apm_oss.spanIndices'], + indices['apm_oss.transactionIndices'] + ] + }); + + return { + indices: { + shards: { + total: response._shards.total + }, + all: { + total: { + docs: { + count: response._all.total.docs.count + }, + store: { + size_in_bytes: response._all.total.store.size_in_bytes + } + } + } + } + }; + } + }, + { + name: 'cardinality', + executor: async ({ search }) => { + const allAgentsCardinalityResponse = await search({ + body: { + size: 0, + query: { + bool: { + filter: [{ range: { '@timestamp': { gte: 'now-1d' } } }] + } + }, + aggs: { + [TRANSACTION_NAME]: { + cardinality: { + field: TRANSACTION_NAME + } + }, + [USER_AGENT_ORIGINAL]: { + cardinality: { + field: USER_AGENT_ORIGINAL + } + } + } + } + }); + + const rumAgentCardinalityResponse = await search({ + body: { + size: 0, + query: { + bool: { + filter: [ + { range: { '@timestamp': { gte: 'now-1d' } } }, + { terms: { [AGENT_NAME]: ['rum-js', 'js-base'] } } + ] + } + }, + aggs: { + [TRANSACTION_NAME]: { + cardinality: { + field: TRANSACTION_NAME + } + }, + [USER_AGENT_ORIGINAL]: { + cardinality: { + field: USER_AGENT_ORIGINAL + } + } + } + } + }); + + return { + cardinality: { + transaction: { + name: { + all_agents: { + '1d': + allAgentsCardinalityResponse.aggregations?.[TRANSACTION_NAME] + .value + }, + rum: { + '1d': + rumAgentCardinalityResponse.aggregations?.[TRANSACTION_NAME] + .value + } + } + }, + user_agent: { + original: { + all_agents: { + '1d': + allAgentsCardinalityResponse.aggregations?.[ + USER_AGENT_ORIGINAL + ].value + }, + rum: { + '1d': + rumAgentCardinalityResponse.aggregations?.[ + USER_AGENT_ORIGINAL + ].value + } + } + } + } + }; + } + } +]; diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/index.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/index.ts index a2b049473082..c80057a2894d 100644 --- a/x-pack/plugins/apm/server/lib/apm_telemetry/index.ts +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/index.ts @@ -3,60 +3,127 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ - -import { countBy } from 'lodash'; -import { SavedObjectAttributes } from '../../../../../../src/core/server'; -import { isAgentName } from '../../../common/agent_name'; +import { CoreSetup, Logger } from 'src/core/server'; +import { Observable } from 'rxjs'; +import { take } from 'rxjs/operators'; +import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; +import { + TaskManagerStartContract, + TaskManagerSetupContract +} from '../../../../task_manager/server'; +import { getApmIndices } from '../settings/apm_indices/get_apm_indices'; import { - APM_SERVICES_TELEMETRY_SAVED_OBJECT_TYPE, - APM_SERVICES_TELEMETRY_SAVED_OBJECT_ID + APM_TELEMETRY_SAVED_OBJECT_ID, + APM_TELEMETRY_SAVED_OBJECT_TYPE } from '../../../common/apm_saved_object_constants'; -import { UsageCollectionSetup } from '../../../../../../src/plugins/usage_collection/server'; -import { InternalSavedObjectsClient } from '../helpers/get_internal_saved_objects_client'; - -export function createApmTelementry( - agentNames: string[] = [] -): SavedObjectAttributes { - const validAgentNames = agentNames.filter(isAgentName); - return { - has_any_services: validAgentNames.length > 0, - services_per_agent: countBy(validAgentNames) +import { + collectDataTelemetry, + CollectTelemetryParams +} from './collect_data_telemetry'; +import { APMConfig } from '../..'; +import { getInternalSavedObjectsClient } from '../helpers/get_internal_saved_objects_client'; + +const APM_TELEMETRY_TASK_NAME = 'apm-telemetry-task'; + +export async function createApmTelemetry({ + core, + config$, + usageCollector, + taskManager, + logger +}: { + core: CoreSetup; + config$: Observable; + usageCollector: UsageCollectionSetup; + taskManager: TaskManagerSetupContract; + logger: Logger; +}) { + const savedObjectsClient = await getInternalSavedObjectsClient(core); + + const collectAndStore = async () => { + const config = await config$.pipe(take(1)).toPromise(); + const esClient = core.elasticsearch.dataClient; + + const indices = await getApmIndices({ + config, + savedObjectsClient + }); + + const search = esClient.callAsInternalUser.bind( + esClient, + 'search' + ) as CollectTelemetryParams['search']; + + const indicesStats = esClient.callAsInternalUser.bind( + esClient, + 'indices.stats' + ) as CollectTelemetryParams['indicesStats']; + + const transportRequest = esClient.callAsInternalUser.bind( + esClient, + 'transport.request' + ) as CollectTelemetryParams['transportRequest']; + + const dataTelemetry = await collectDataTelemetry({ + search, + indices, + logger, + indicesStats, + transportRequest + }); + + await savedObjectsClient.create( + APM_TELEMETRY_SAVED_OBJECT_TYPE, + dataTelemetry, + { id: APM_TELEMETRY_SAVED_OBJECT_TYPE, overwrite: true } + ); }; -} -export async function storeApmServicesTelemetry( - savedObjectsClient: InternalSavedObjectsClient, - apmTelemetry: SavedObjectAttributes -) { - return savedObjectsClient.create( - APM_SERVICES_TELEMETRY_SAVED_OBJECT_TYPE, - apmTelemetry, - { - id: APM_SERVICES_TELEMETRY_SAVED_OBJECT_ID, - overwrite: true + taskManager.registerTaskDefinitions({ + [APM_TELEMETRY_TASK_NAME]: { + title: 'Collect APM telemetry', + type: APM_TELEMETRY_TASK_NAME, + createTaskRunner: () => { + return { + run: async () => { + await collectAndStore(); + } + }; + } } - ); -} + }); -export function makeApmUsageCollector( - usageCollector: UsageCollectionSetup, - savedObjectsRepository: InternalSavedObjectsClient -) { - const apmUsageCollector = usageCollector.makeUsageCollector({ + const collector = usageCollector.makeUsageCollector({ type: 'apm', fetch: async () => { - try { - const apmTelemetrySavedObject = await savedObjectsRepository.get( - APM_SERVICES_TELEMETRY_SAVED_OBJECT_TYPE, - APM_SERVICES_TELEMETRY_SAVED_OBJECT_ID - ); - return apmTelemetrySavedObject.attributes; - } catch (err) { - return createApmTelementry(); - } + const data = ( + await savedObjectsClient.get( + APM_TELEMETRY_SAVED_OBJECT_TYPE, + APM_TELEMETRY_SAVED_OBJECT_ID + ) + ).attributes; + + return data; }, isReady: () => true }); - usageCollector.registerCollector(apmUsageCollector); + usageCollector.registerCollector(collector); + + core.getStartServices().then(([coreStart, pluginsStart]) => { + const { taskManager: taskManagerStart } = pluginsStart as { + taskManager: TaskManagerStartContract; + }; + + taskManagerStart.ensureScheduled({ + id: APM_TELEMETRY_TASK_NAME, + taskType: APM_TELEMETRY_TASK_NAME, + schedule: { + interval: '720m' + }, + scope: ['apm'], + params: {}, + state: {} + }); + }); } diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/types.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/types.ts new file mode 100644 index 000000000000..f68dc517a222 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/types.ts @@ -0,0 +1,118 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { DeepPartial } from 'utility-types'; +import { AgentName } from '../../../typings/es_schemas/ui/fields/agent'; + +export interface TimeframeMap { + '1d': number; + all: number; +} + +export type TimeframeMap1d = Pick; +export type TimeframeMapAll = Pick; + +export type APMDataTelemetry = DeepPartial<{ + has_any_services: boolean; + services_per_agent: Record; + versions: { + apm_server: { + minor: number; + major: number; + patch: number; + }; + }; + counts: { + transaction: TimeframeMap; + span: TimeframeMap; + error: TimeframeMap; + metric: TimeframeMap; + sourcemap: TimeframeMap; + onboarding: TimeframeMap; + agent_configuration: TimeframeMapAll; + max_transaction_groups_per_service: TimeframeMap; + max_error_groups_per_service: TimeframeMap; + traces: TimeframeMap; + services: TimeframeMap; + }; + cardinality: { + user_agent: { + original: { + all_agents: TimeframeMap1d; + rum: TimeframeMap1d; + }; + }; + transaction: { + name: { + all_agents: TimeframeMap1d; + rum: TimeframeMap1d; + }; + }; + }; + retainment: Record< + 'span' | 'transaction' | 'error' | 'metric' | 'sourcemap' | 'onboarding', + { ms: number } + >; + integrations: { + ml: { + all_jobs_count: number; + }; + }; + agents: Record< + AgentName, + { + agent: { + version: string[]; + }; + service: { + framework: { + name: string[]; + version: string[]; + composite: string[]; + }; + language: { + name: string[]; + version: string[]; + composite: string[]; + }; + runtime: { + name: string[]; + version: string[]; + composite: string[]; + }; + }; + } + >; + indices: { + shards: { + total: number; + }; + all: { + total: { + docs: { + count: number; + }; + store: { + size_in_bytes: number; + }; + }; + }; + }; + tasks: Record< + | 'processor_events' + | 'agent_configuration' + | 'services' + | 'versions' + | 'groupings' + | 'integrations' + | 'agents' + | 'indices_stats' + | 'cardinality', + { took: { ms: number } } + >; +}>; + +export type APMTelemetry = APMDataTelemetry; diff --git a/x-pack/plugins/apm/server/lib/helpers/setup_request.test.ts b/x-pack/plugins/apm/server/lib/helpers/setup_request.test.ts index 40a2a0e7216a..8e8cf698a84c 100644 --- a/x-pack/plugins/apm/server/lib/helpers/setup_request.test.ts +++ b/x-pack/plugins/apm/server/lib/helpers/setup_request.test.ts @@ -39,6 +39,19 @@ function getMockRequest() { _debug: false } }, + __LEGACY: { + server: { + plugins: { + elasticsearch: { + getCluster: jest.fn().mockReturnValue({ callWithInternalUser: {} }) + } + }, + savedObjects: { + SavedObjectsClient: jest.fn(), + getSavedObjectsRepository: jest.fn() + } + } + }, core: { elasticsearch: { dataClient: { diff --git a/x-pack/plugins/apm/server/plugin.ts b/x-pack/plugins/apm/server/plugin.ts index e140340786e8..e18b6d33ca41 100644 --- a/x-pack/plugins/apm/server/plugin.ts +++ b/x-pack/plugins/apm/server/plugin.ts @@ -13,7 +13,6 @@ import { TaskManagerSetupContract } from '../../task_manager/server'; import { AlertingPlugin } from '../../alerting/server'; import { ActionsPlugin } from '../../actions/server'; import { APMOSSPluginSetup } from '../../../../src/plugins/apm_oss/server'; -import { makeApmUsageCollector } from './lib/apm_telemetry'; import { createApmAgentConfigurationIndex } from './lib/settings/agent_configuration/create_agent_config_index'; import { createApmCustomLinkIndex } from './lib/settings/custom_link/create_custom_link_index'; import { createApmApi } from './routes/create_apm_api'; @@ -25,6 +24,7 @@ import { CloudSetup } from '../../cloud/server'; import { getInternalSavedObjectsClient } from './lib/helpers/get_internal_saved_objects_client'; import { LicensingPluginSetup } from '../../licensing/public'; import { registerApmAlerts } from './lib/alerts/register_apm_alerts'; +import { createApmTelemetry } from './lib/apm_telemetry'; export interface LegacySetup { server: Server; @@ -56,7 +56,7 @@ export class APMPlugin implements Plugin { actions?: ActionsPlugin['setup']; } ) { - const logger = this.initContext.logger.get('apm'); + const logger = this.initContext.logger.get(); const config$ = this.initContext.config.create(); const mergedConfig$ = combineLatest(plugins.apm_oss.config$, config$).pipe( map(([apmOssConfig, apmConfig]) => mergeConfigs(apmOssConfig, apmConfig)) @@ -76,6 +76,20 @@ export class APMPlugin implements Plugin { const currentConfig = await mergedConfig$.pipe(take(1)).toPromise(); + if ( + plugins.taskManager && + plugins.usageCollection && + currentConfig['xpack.apm.telemetryCollectionEnabled'] + ) { + createApmTelemetry({ + core, + config$: mergedConfig$, + usageCollector: plugins.usageCollection, + taskManager: plugins.taskManager, + logger + }); + } + // create agent configuration index without blocking setup lifecycle createApmAgentConfigurationIndex({ esClient: core.elasticsearch.dataClient, @@ -104,18 +118,6 @@ export class APMPlugin implements Plugin { }) ); - const usageCollection = plugins.usageCollection; - if (usageCollection) { - getInternalSavedObjectsClient(core) - .then(savedObjectsClient => { - makeApmUsageCollector(usageCollection, savedObjectsClient); - }) - .catch(error => { - logger.error('Unable to initialize use collection'); - logger.error(error.message); - }); - } - return { config$: mergedConfig$, registerLegacyAPI: once((__LEGACY: LegacySetup) => { @@ -130,6 +132,7 @@ export class APMPlugin implements Plugin { }; } - public start() {} + public async start() {} + public stop() {} } diff --git a/x-pack/plugins/apm/server/routes/create_api/index.test.ts b/x-pack/plugins/apm/server/routes/create_api/index.test.ts index e639bb5101e2..312dae1d1f9d 100644 --- a/x-pack/plugins/apm/server/routes/create_api/index.test.ts +++ b/x-pack/plugins/apm/server/routes/create_api/index.test.ts @@ -36,6 +36,7 @@ const getCoreMock = () => { put, createRouter, context: { + measure: () => undefined, config$: new BehaviorSubject({} as APMConfig), logger: ({ error: jest.fn() diff --git a/x-pack/plugins/apm/server/routes/services.ts b/x-pack/plugins/apm/server/routes/services.ts index 2d4fae9d2707..1c6561ee24c9 100644 --- a/x-pack/plugins/apm/server/routes/services.ts +++ b/x-pack/plugins/apm/server/routes/services.ts @@ -5,11 +5,6 @@ */ import * as t from 'io-ts'; -import { AgentName } from '../../typings/es_schemas/ui/fields/agent'; -import { - createApmTelementry, - storeApmServicesTelemetry -} from '../lib/apm_telemetry'; import { setupRequest } from '../lib/helpers/setup_request'; import { getServiceAgentName } from '../lib/services/get_service_agent_name'; import { getServices } from '../lib/services/get_services'; @@ -18,7 +13,6 @@ import { getServiceNodeMetadata } from '../lib/services/get_service_node_metadat import { createRoute } from './create_route'; import { uiFiltersRt, rangeRt } from './default_api_types'; import { getServiceAnnotations } from '../lib/services/annotations'; -import { getInternalSavedObjectsClient } from '../lib/helpers/get_internal_saved_objects_client'; export const servicesRoute = createRoute(core => ({ path: '/api/apm/services', @@ -29,16 +23,6 @@ export const servicesRoute = createRoute(core => ({ const setup = await setupRequest(context, request); const services = await getServices(setup); - // Store telemetry data derived from services - const agentNames = services.items.map( - ({ agentName }) => agentName as AgentName - ); - const apmTelemetry = createApmTelementry(agentNames); - const savedObjectsClient = await getInternalSavedObjectsClient(core); - storeApmServicesTelemetry(savedObjectsClient, apmTelemetry).catch(error => { - context.logger.error(error.message); - }); - return services; } })); diff --git a/x-pack/plugins/apm/typings/elasticsearch/aggregations.ts b/x-pack/plugins/apm/typings/elasticsearch/aggregations.ts index 6d3620f11a87..8a8d256cf427 100644 --- a/x-pack/plugins/apm/typings/elasticsearch/aggregations.ts +++ b/x-pack/plugins/apm/typings/elasticsearch/aggregations.ts @@ -126,6 +126,16 @@ export interface AggregationOptionsByType { combine_script: Script; reduce_script: Script; }; + date_range: { + field: string; + format?: string; + ranges: Array< + | { from: string | number } + | { to: string | number } + | { from: string | number; to: string | number } + >; + keyed?: boolean; + }; } type AggregationType = keyof AggregationOptionsByType; @@ -136,6 +146,15 @@ type AggregationOptionsMap = Unionize< } > & { aggs?: AggregationInputMap }; +interface DateRangeBucket { + key: string; + to?: number; + from?: number; + to_as_string?: string; + from_as_string?: string; + doc_count: number; +} + export interface AggregationInputMap { [key: string]: AggregationOptionsMap; } @@ -276,6 +295,11 @@ interface AggregationResponsePart< scripted_metric: { value: unknown; }; + date_range: { + buckets: TAggregationOptionsMap extends { date_range: { keyed: true } } + ? Record + : { buckets: DateRangeBucket[] }; + }; } // Type for debugging purposes. If you see an error in AggregationResponseMap @@ -285,7 +309,7 @@ interface AggregationResponsePart< // type MissingAggregationResponseTypes = Exclude< // AggregationType, -// keyof AggregationResponsePart<{}> +// keyof AggregationResponsePart<{}, unknown> // >; export type AggregationResponseMap< diff --git a/x-pack/plugins/apm/typings/es_schemas/raw/error_raw.ts b/x-pack/plugins/apm/typings/es_schemas/raw/error_raw.ts index daf65e44980b..8e49d02beb90 100644 --- a/x-pack/plugins/apm/typings/es_schemas/raw/error_raw.ts +++ b/x-pack/plugins/apm/typings/es_schemas/raw/error_raw.ts @@ -15,6 +15,7 @@ import { Service } from './fields/service'; import { IStackframe } from './fields/stackframe'; import { Url } from './fields/url'; import { User } from './fields/user'; +import { Observer } from './fields/observer'; interface Processor { name: 'error'; @@ -61,4 +62,5 @@ export interface ErrorRaw extends APMBaseDoc { service: Service; url?: Url; user?: User; + observer?: Observer; } diff --git a/x-pack/plugins/apm/typings/es_schemas/raw/fields/observer.ts b/x-pack/plugins/apm/typings/es_schemas/raw/fields/observer.ts new file mode 100644 index 000000000000..42843130ec47 --- /dev/null +++ b/x-pack/plugins/apm/typings/es_schemas/raw/fields/observer.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export interface Observer { + version: string; + version_major: number; +} diff --git a/x-pack/plugins/apm/typings/es_schemas/raw/span_raw.ts b/x-pack/plugins/apm/typings/es_schemas/raw/span_raw.ts index dbd9e7ede425..4d5d2c5c4a12 100644 --- a/x-pack/plugins/apm/typings/es_schemas/raw/span_raw.ts +++ b/x-pack/plugins/apm/typings/es_schemas/raw/span_raw.ts @@ -6,6 +6,7 @@ import { APMBaseDoc } from './apm_base_doc'; import { IStackframe } from './fields/stackframe'; +import { Observer } from './fields/observer'; interface Processor { name: 'transaction'; @@ -50,4 +51,5 @@ export interface SpanRaw extends APMBaseDoc { transaction?: { id: string; }; + observer?: Observer; } diff --git a/x-pack/plugins/apm/typings/es_schemas/raw/transaction_raw.ts b/x-pack/plugins/apm/typings/es_schemas/raw/transaction_raw.ts index 3673f1f13c40..b8ebb4cf8da5 100644 --- a/x-pack/plugins/apm/typings/es_schemas/raw/transaction_raw.ts +++ b/x-pack/plugins/apm/typings/es_schemas/raw/transaction_raw.ts @@ -15,6 +15,7 @@ import { Service } from './fields/service'; import { Url } from './fields/url'; import { User } from './fields/user'; import { UserAgent } from './fields/user_agent'; +import { Observer } from './fields/observer'; interface Processor { name: 'transaction'; @@ -61,4 +62,5 @@ export interface TransactionRaw extends APMBaseDoc { url?: Url; user?: User; user_agent?: UserAgent; + observer?: Observer; } From 418e093998c160b3cb261922a03aae48ef37ccc2 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Tue, 24 Mar 2020 08:13:14 +0100 Subject: [PATCH 2/2] Update transaction mock data to reflect the type --- .../shared/TransactionActionMenu/__test__/mockData.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/legacy/plugins/apm/public/components/shared/TransactionActionMenu/__test__/mockData.ts b/x-pack/legacy/plugins/apm/public/components/shared/TransactionActionMenu/__test__/mockData.ts index be8f379ce62e..70be1a474476 100644 --- a/x-pack/legacy/plugins/apm/public/components/shared/TransactionActionMenu/__test__/mockData.ts +++ b/x-pack/legacy/plugins/apm/public/components/shared/TransactionActionMenu/__test__/mockData.ts @@ -8,7 +8,10 @@ import { Location } from 'history'; const bareTransaction = { '@metadata': 'whatever', - observer: 'whatever', + observer: { + version: '8.0.0', + version_major: 8 + }, agent: { name: 'java', version: '7.0.0'