diff --git a/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap b/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap index a154ededc2a33..29ae8156230d7 100644 --- a/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap +++ b/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap @@ -845,16 +845,6 @@ exports[`APM telemetry helpers getApmTelemetry generates a JSON object with the } } }, - "sourcemap": { - "properties": { - "1d": { - "type": "long" - }, - "all": { - "type": "long" - } - } - }, "onboarding": { "properties": { "1d": { @@ -1011,13 +1001,6 @@ exports[`APM telemetry helpers getApmTelemetry generates a JSON object with the } } }, - "sourcemap": { - "properties": { - "ms": { - "type": "long" - } - } - }, "onboarding": { "properties": { "ms": { diff --git a/x-pack/plugins/apm/ftr_e2e/cypress/e2e/power_user/no_data_screen.cy.ts b/x-pack/plugins/apm/ftr_e2e/cypress/e2e/power_user/no_data_screen.cy.ts index ae08bc1ea9b63..e73a9e6b608c4 100644 --- a/x-pack/plugins/apm/ftr_e2e/cypress/e2e/power_user/no_data_screen.cy.ts +++ b/x-pack/plugins/apm/ftr_e2e/cypress/e2e/power_user/no_data_screen.cy.ts @@ -10,7 +10,6 @@ describe('No data screen', () => { before(() => { // Change indices setApmIndices({ - sourcemap: 'foo-*', error: 'foo-*', onboarding: 'foo-*', span: 'foo-*', @@ -37,7 +36,6 @@ describe('No data screen', () => { after(() => { // reset to default indices setApmIndices({ - sourcemap: '', error: '', onboarding: '', span: '', diff --git a/x-pack/plugins/apm/public/components/app/settings/apm_indices/index.tsx b/x-pack/plugins/apm/public/components/app/settings/apm_indices/index.tsx index 81b2ec53ddd33..419229fe3ac0f 100644 --- a/x-pack/plugins/apm/public/components/app/settings/apm_indices/index.tsx +++ b/x-pack/plugins/apm/public/components/app/settings/apm_indices/index.tsx @@ -33,13 +33,6 @@ import { } from '../../../../services/rest/create_call_apm_api'; const APM_INDEX_LABELS = [ - { - configurationName: 'sourcemap', - label: i18n.translate( - 'xpack.apm.settings.apmIndices.sourcemapIndicesLabel', - { defaultMessage: 'Sourcemap Indices' } - ), - }, { configurationName: 'error', label: i18n.translate('xpack.apm.settings.apmIndices.errorIndicesLabel', { diff --git a/x-pack/plugins/apm/scripts/shared/read_kibana_config.ts b/x-pack/plugins/apm/scripts/shared/read_kibana_config.ts index f3e2b48390468..fffe4586d3167 100644 --- a/x-pack/plugins/apm/scripts/shared/read_kibana_config.ts +++ b/x-pack/plugins/apm/scripts/shared/read_kibana_config.ts @@ -43,7 +43,6 @@ export const readKibanaConfig = () => { 'xpack.apm.indices.error': 'logs-apm*,apm-*', 'xpack.apm.indices.span': 'traces-apm*,apm-*', 'xpack.apm.indices.onboarding': 'apm-*', - 'xpack.apm.indices.sourcemap': 'apm-*', 'elasticsearch.hosts': 'http://localhost:9200', ...loadedKibanaConfig, ...cliEsCredentials, diff --git a/x-pack/plugins/apm/server/index.ts b/x-pack/plugins/apm/server/index.ts index 19b34766a1bf2..d04d10a1cccad 100644 --- a/x-pack/plugins/apm/server/index.ts +++ b/x-pack/plugins/apm/server/index.ts @@ -51,7 +51,6 @@ const configSchema = schema.object({ span: schema.string({ defaultValue: 'traces-apm*,apm-*' }), error: schema.string({ defaultValue: 'logs-apm*,apm-*' }), metric: schema.string({ defaultValue: 'metrics-apm*,apm-*' }), - sourcemap: schema.string({ defaultValue: 'apm-*' }), onboarding: schema.string({ defaultValue: 'apm-*' }), }), forceSyntheticSource: schema.boolean({ defaultValue: false }), @@ -61,10 +60,12 @@ const configSchema = schema.object({ export const config: PluginConfigDescriptor = { deprecations: ({ rename, + unused, renameFromRoot, deprecateFromRoot, unusedFromRoot, }) => [ + unused('indices.sourcemap', { level: 'warning' }), rename('autocreateApmIndexPattern', 'autoCreateApmDataView', { level: 'warning', }), diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts index 1b9103aeefc6a..d48ab312a5674 100644 --- a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts @@ -298,10 +298,6 @@ describe('data telemetry collection tasks', () => { '1d': 1, all: 1, }, - sourcemap: { - '1d': 1, - all: 1, - }, span: { '1d': 1, all: 1, @@ -321,9 +317,6 @@ describe('data telemetry collection tasks', () => { onboarding: { ms: 0, }, - sourcemap: { - ms: 0, - }, span: { ms: 0, }, diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts index d7d14c37e819f..88cbcdc842d11 100644 --- a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts @@ -54,7 +54,10 @@ import { AgentName } from '../../../../typings/es_schemas/ui/fields/agent'; import { Span } from '../../../../typings/es_schemas/ui/span'; import { Transaction } from '../../../../typings/es_schemas/ui/transaction'; import { APMTelemetry, APMPerService, APMDataTelemetry } from '../types'; -import { ApmIndicesConfig } from '../../../routes/settings/apm_indices/get_apm_indices'; +import { + ApmIndicesConfig, + APM_AGENT_CONFIGURATION_INDEX, +} from '../../../routes/settings/apm_indices/get_apm_indices'; import { TelemetryClient } from '../telemetry_client'; type ISavedObjectsClient = Pick; @@ -464,7 +467,6 @@ export const tasks: TelemetryTask[] = [ span: indices.span, transaction: indices.transaction, onboarding: indices.onboarding, - sourcemap: indices.sourcemap, }; type ProcessorEvent = keyof typeof indicesByProcessorEvent; @@ -558,7 +560,7 @@ export const tasks: TelemetryTask[] = [ name: 'agent_configuration', executor: async ({ indices, telemetryClient }) => { const agentConfigurationCount = await telemetryClient.search({ - index: indices.apmAgentConfigurationIndex, + index: APM_AGENT_CONFIGURATION_INDEX, body: { size: 0, timeout, @@ -1033,11 +1035,10 @@ export const tasks: TelemetryTask[] = [ executor: async ({ indices, telemetryClient }) => { const response = await telemetryClient.indicesStats({ index: [ - indices.apmAgentConfigurationIndex, + APM_AGENT_CONFIGURATION_INDEX, indices.error, indices.metric, indices.onboarding, - indices.sourcemap, indices.span, indices.transaction, ], diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/schema.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/schema.ts index c73eeae128cfe..11b8d34da19ad 100644 --- a/x-pack/plugins/apm/server/lib/apm_telemetry/schema.ts +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/schema.ts @@ -193,7 +193,6 @@ export const apmSchema: MakeSchemaFrom = { span: timeframeMapSchema, error: timeframeMapSchema, metric: timeframeMapSchema, - sourcemap: timeframeMapSchema, onboarding: timeframeMapSchema, agent_configuration: timeframeMapAllSchema, max_transaction_groups_per_service: timeframeMapSchema, @@ -221,7 +220,6 @@ export const apmSchema: MakeSchemaFrom = { transaction: { ms: long }, error: { ms: long }, metric: { ms: long }, - sourcemap: { ms: long }, onboarding: { ms: long }, }, integrations: { ml: { all_jobs_count: long } }, diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/types.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/types.ts index 518bb969bcb0e..54f0963dfbb62 100644 --- a/x-pack/plugins/apm/server/lib/apm_telemetry/types.ts +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/types.ts @@ -101,7 +101,6 @@ export interface APMUsage { span: TimeframeMap; error: TimeframeMap; metric: TimeframeMap; - sourcemap: TimeframeMap; onboarding: TimeframeMap; agent_configuration: TimeframeMapAll; max_transaction_groups_per_service: TimeframeMap; @@ -125,7 +124,7 @@ export interface APMUsage { }; }; retainment: Record< - 'span' | 'transaction' | 'error' | 'metric' | 'sourcemap' | 'onboarding', + 'span' | 'transaction' | 'error' | 'metric' | 'onboarding', { ms: number } >; integrations: { diff --git a/x-pack/plugins/apm/server/lib/helpers/create_es_client/create_apm_event_client/unpack_processor_events.test.ts b/x-pack/plugins/apm/server/lib/helpers/create_es_client/create_apm_event_client/unpack_processor_events.test.ts index c0aa066798611..abeb548271b04 100644 --- a/x-pack/plugins/apm/server/lib/helpers/create_es_client/create_apm_event_client/unpack_processor_events.test.ts +++ b/x-pack/plugins/apm/server/lib/helpers/create_es_client/create_apm_event_client/unpack_processor_events.test.ts @@ -27,7 +27,6 @@ describe('unpackProcessorEvents', () => { error: 'my-apm-*-error-*', span: 'my-apm-*-span-*', onboarding: 'my-apm-*-onboarding-*', - sourcemap: 'my-apm-*-sourcemap-*', } as ApmIndicesConfig; res = unpackProcessorEvents(request, indices); diff --git a/x-pack/plugins/apm/server/plugin.ts b/x-pack/plugins/apm/server/plugin.ts index 60c3750b1aa3a..17b6d2c0e8250 100644 --- a/x-pack/plugins/apm/server/plugin.ts +++ b/x-pack/plugins/apm/server/plugin.ts @@ -56,6 +56,8 @@ import { } from '../common/es_fields/apm'; import { tutorialProvider } from './tutorial'; import { migrateLegacyAPMIndicesToSpaceAware } from './saved_objects/migrations/migrate_legacy_apm_indices_to_space_aware'; +import { scheduleSourceMapMigration } from './routes/source_maps/schedule_source_map_migration'; +import { createApmSourceMapIndexTemplate } from './routes/source_maps/create_apm_source_map_index_template'; export class APMPlugin implements @@ -110,6 +112,9 @@ export class APMPlugin const getCoreStart = () => core.getStartServices().then(([coreStart]) => coreStart); + const getPluginStart = () => + core.getStartServices().then(([coreStart, pluginStart]) => pluginStart); + const { ruleDataService } = plugins.ruleRegistry; const ruleDataClient = ruleDataService.initializeIndex({ feature: APM_SERVER_FEATURE_ID, @@ -220,6 +225,21 @@ export class APMPlugin kibanaVersion: this.initContext.env.packageInfo.version, }); + const fleetStartPromise = resourcePlugins.fleet?.start(); + const taskManager = plugins.taskManager; + + // create source map index and run migrations + scheduleSourceMapMigration({ + coreStartPromise: getCoreStart(), + pluginStartPromise: getPluginStart(), + fleetStartPromise, + taskManager, + logger: this.logger, + }).catch((e) => { + this.logger?.error('Failed to schedule APM source map migration'); + this.logger?.error(e); + }); + return { config$, getApmIndices: boundGetApmIndices, @@ -254,28 +274,39 @@ export class APMPlugin }; } - public start(core: CoreStart) { + public start(core: CoreStart, plugins: APMPluginStartDependencies) { if (this.currentConfig == null || this.logger == null) { throw new Error('APMPlugin needs to be setup before calling start()'); } - // create agent configuration index without blocking start lifecycle - createApmAgentConfigurationIndex({ - client: core.elasticsearch.client.asInternalUser, - config: this.currentConfig, - logger: this.logger, + const logger = this.logger; + const client = core.elasticsearch.client.asInternalUser; + + // create .apm-agent-configuration index without blocking start lifecycle + createApmAgentConfigurationIndex({ client, logger }).catch((e) => { + logger.error('Failed to create .apm-agent-configuration index'); + logger.error(e); }); - // create custom action index without blocking start lifecycle - createApmCustomLinkIndex({ - client: core.elasticsearch.client.asInternalUser, - config: this.currentConfig, - logger: this.logger, + + // create .apm-custom-link index without blocking start lifecycle + createApmCustomLinkIndex({ client, logger }).catch((e) => { + logger.error('Failed to create .apm-custom-link index'); + logger.error(e); }); - migrateLegacyAPMIndicesToSpaceAware({ - coreStart: core, - logger: this.logger, + // create .apm-source-map index without blocking start lifecycle + createApmSourceMapIndexTemplate({ client, logger }).catch((e) => { + logger.error('Failed to create apm-source-map index template'); + logger.error(e); }); + + // TODO: remove in 9.0 + migrateLegacyAPMIndicesToSpaceAware({ coreStart: core, logger }).catch( + (e) => { + logger.error('Failed to run migration making APM indices space aware'); + logger.error(e); + } + ); } public stop() {} diff --git a/x-pack/plugins/apm/server/routes/fleet/source_maps.ts b/x-pack/plugins/apm/server/routes/fleet/source_maps.ts index 9cba744cdb689..c2e7ca7110fe8 100644 --- a/x-pack/plugins/apm/server/routes/fleet/source_maps.ts +++ b/x-pack/plugins/apm/server/routes/fleet/source_maps.ts @@ -18,32 +18,26 @@ import { APMPluginStartDependencies } from '../../types'; import { getApmPackagePolicies } from './get_apm_package_policies'; import { APM_SERVER, PackagePolicy } from './register_fleet_policy_callbacks'; -export interface ApmArtifactBody { +const doUnzip = promisify(unzip); + +interface ApmSourceMapArtifactBody { serviceName: string; serviceVersion: string; bundleFilepath: string; sourceMap: SourceMap; } export type ArtifactSourceMap = Omit & { - body: ApmArtifactBody; + body: ApmSourceMapArtifactBody; }; export type FleetPluginStart = NonNullable; -const doUnzip = promisify(unzip); - -async function unzipArtifactBody( - artifact: Artifact -): Promise { - const body = await doUnzip(Buffer.from(artifact.body, 'base64')); - - return { - ...artifact, - body: JSON.parse(body.toString()) as ApmArtifactBody, - }; +export async function getUnzippedArtifactBody(artifactBody: string) { + const unzippedBody = await doUnzip(Buffer.from(artifactBody, 'base64')); + return JSON.parse(unzippedBody.toString()) as ApmSourceMapArtifactBody; } -function getApmArtifactClient(fleetPluginStart: FleetPluginStart) { +export function getApmArtifactClient(fleetPluginStart: FleetPluginStart) { return fleetPluginStart.createArtifactsClient('apm'); } @@ -66,17 +60,20 @@ export async function listSourceMapArtifacts({ }); const artifacts = await Promise.all( - artifactsResponse.items.map(unzipArtifactBody) + artifactsResponse.items.map(async (item) => { + const body = await getUnzippedArtifactBody(item.body); + return { ...item, body }; + }) ); return { artifacts, total: artifactsResponse.total }; } -export async function createApmArtifact({ +export async function createFleetSourceMapArtifact({ apmArtifactBody, fleetPluginStart, }: { - apmArtifactBody: ApmArtifactBody; + apmArtifactBody: ApmSourceMapArtifactBody; fleetPluginStart: FleetPluginStart; }) { const apmArtifactClient = getApmArtifactClient(fleetPluginStart); @@ -89,7 +86,7 @@ export async function createApmArtifact({ }); } -export async function deleteApmArtifact({ +export async function deleteFleetSourcemapArtifact({ id, fleetPluginStart, }: { @@ -141,12 +138,12 @@ export async function updateSourceMapsOnFleetPolicies({ core, fleetPluginStart, savedObjectsClient, - elasticsearchClient, + internalESClient, }: { core: { setup: CoreSetup; start: () => Promise }; fleetPluginStart: FleetPluginStart; savedObjectsClient: SavedObjectsClientContract; - elasticsearchClient: ElasticsearchClient; + internalESClient: ElasticsearchClient; }) { const { artifacts } = await listSourceMapArtifacts({ fleetPluginStart }); const apmFleetPolicies = await getApmPackagePolicies({ @@ -171,7 +168,7 @@ export async function updateSourceMapsOnFleetPolicies({ await fleetPluginStart.packagePolicyService.update( savedObjectsClient, - elasticsearchClient, + internalESClient, id, updatedPackagePolicy ); @@ -179,11 +176,11 @@ export async function updateSourceMapsOnFleetPolicies({ ); } -export function getCleanedBundleFilePath(bundleFilePath: string) { +export function getCleanedBundleFilePath(bundleFilepath: string) { try { - const cleanedBundleFilepath = new URL(bundleFilePath); + const cleanedBundleFilepath = new URL(bundleFilepath); return cleanedBundleFilepath.href; } catch (e) { - return bundleFilePath; + return bundleFilepath; } } diff --git a/x-pack/plugins/apm/server/routes/settings/agent_configuration/__snapshots__/queries.test.ts.snap b/x-pack/plugins/apm/server/routes/settings/agent_configuration/__snapshots__/queries.test.ts.snap index e626209fe75db..4edd9dcfe99dc 100644 --- a/x-pack/plugins/apm/server/routes/settings/agent_configuration/__snapshots__/queries.test.ts.snap +++ b/x-pack/plugins/apm/server/routes/settings/agent_configuration/__snapshots__/queries.test.ts.snap @@ -1,10 +1,88 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`agent configuration queries findExactConfiguration find configuration by service.environment 1`] = `undefined`; +exports[`agent configuration queries findExactConfiguration find configuration by service.environment 1`] = ` +Object { + "body": Object { + "query": Object { + "bool": Object { + "filter": Array [ + Object { + "bool": Object { + "must_not": Array [ + Object { + "exists": Object { + "field": "service.name", + }, + }, + ], + }, + }, + Object { + "term": Object { + "service.environment": "bar", + }, + }, + ], + }, + }, + }, + "index": ".apm-agent-configuration", +} +`; -exports[`agent configuration queries findExactConfiguration find configuration by service.name 1`] = `undefined`; +exports[`agent configuration queries findExactConfiguration find configuration by service.name 1`] = ` +Object { + "body": Object { + "query": Object { + "bool": Object { + "filter": Array [ + Object { + "term": Object { + "service.name": "foo", + }, + }, + Object { + "bool": Object { + "must_not": Array [ + Object { + "exists": Object { + "field": "service.environment", + }, + }, + ], + }, + }, + ], + }, + }, + }, + "index": ".apm-agent-configuration", +} +`; -exports[`agent configuration queries findExactConfiguration find configuration by service.name and service.environment 1`] = `undefined`; +exports[`agent configuration queries findExactConfiguration find configuration by service.name and service.environment 1`] = ` +Object { + "body": Object { + "query": Object { + "bool": Object { + "filter": Array [ + Object { + "term": Object { + "service.name": "foo", + }, + }, + Object { + "term": Object { + "service.environment": "bar", + }, + }, + ], + }, + }, + }, + "index": ".apm-agent-configuration", +} +`; exports[`agent configuration queries getAllEnvironments fetches all environments 1`] = ` Object { @@ -42,10 +120,142 @@ Object { } `; -exports[`agent configuration queries getExistingEnvironmentsForService fetches unavailable environments 1`] = `undefined`; +exports[`agent configuration queries getExistingEnvironmentsForService fetches unavailable environments 1`] = ` +Object { + "body": Object { + "aggs": Object { + "environments": Object { + "terms": Object { + "field": "service.environment", + "missing": "ALL_OPTION_VALUE", + "size": 50, + }, + }, + }, + "query": Object { + "bool": Object { + "filter": Array [ + Object { + "term": Object { + "service.name": "foo", + }, + }, + ], + }, + }, + "size": 0, + }, + "index": ".apm-agent-configuration", +} +`; -exports[`agent configuration queries listConfigurations fetches configurations 1`] = `undefined`; +exports[`agent configuration queries listConfigurations fetches configurations 1`] = ` +Object { + "index": ".apm-agent-configuration", + "size": 200, +} +`; -exports[`agent configuration queries searchConfigurations fetches filtered configurations with an environment 1`] = `undefined`; +exports[`agent configuration queries searchConfigurations fetches filtered configurations with an environment 1`] = ` +Object { + "body": Object { + "query": Object { + "bool": Object { + "minimum_should_match": 2, + "should": Array [ + Object { + "constant_score": Object { + "boost": 2, + "filter": Object { + "term": Object { + "service.name": "foo", + }, + }, + }, + }, + Object { + "constant_score": Object { + "boost": 1, + "filter": Object { + "term": Object { + "service.environment": "bar", + }, + }, + }, + }, + Object { + "bool": Object { + "must_not": Array [ + Object { + "exists": Object { + "field": "service.name", + }, + }, + ], + }, + }, + Object { + "bool": Object { + "must_not": Array [ + Object { + "exists": Object { + "field": "service.environment", + }, + }, + ], + }, + }, + ], + }, + }, + }, + "index": ".apm-agent-configuration", +} +`; -exports[`agent configuration queries searchConfigurations fetches filtered configurations without an environment 1`] = `undefined`; +exports[`agent configuration queries searchConfigurations fetches filtered configurations without an environment 1`] = ` +Object { + "body": Object { + "query": Object { + "bool": Object { + "minimum_should_match": 2, + "should": Array [ + Object { + "constant_score": Object { + "boost": 2, + "filter": Object { + "term": Object { + "service.name": "foo", + }, + }, + }, + }, + Object { + "bool": Object { + "must_not": Array [ + Object { + "exists": Object { + "field": "service.name", + }, + }, + ], + }, + }, + Object { + "bool": Object { + "must_not": Array [ + Object { + "exists": Object { + "field": "service.environment", + }, + }, + ], + }, + }, + ], + }, + }, + }, + "index": ".apm-agent-configuration", +} +`; diff --git a/x-pack/plugins/apm/server/routes/settings/agent_configuration/create_agent_config_index.ts b/x-pack/plugins/apm/server/routes/settings/agent_configuration/create_agent_config_index.ts index 05e9b4d87589a..4aebdfaf03e6a 100644 --- a/x-pack/plugins/apm/server/routes/settings/agent_configuration/create_agent_config_index.ts +++ b/x-pack/plugins/apm/server/routes/settings/agent_configuration/create_agent_config_index.ts @@ -10,21 +10,17 @@ import { createOrUpdateIndex, Mappings, } from '@kbn/observability-plugin/server'; -import { APMConfig } from '../../..'; -import { getApmIndicesConfig } from '../apm_indices/get_apm_indices'; +import { APM_AGENT_CONFIGURATION_INDEX } from '../apm_indices/get_apm_indices'; export async function createApmAgentConfigurationIndex({ client, - config, logger, }: { client: ElasticsearchClient; - config: APMConfig; logger: Logger; }) { - const index = getApmIndicesConfig(config).apmAgentConfigurationIndex; return createOrUpdateIndex({ - index, + index: APM_AGENT_CONFIGURATION_INDEX, client, logger, mappings, diff --git a/x-pack/plugins/apm/server/routes/settings/agent_configuration/create_or_update_configuration.ts b/x-pack/plugins/apm/server/routes/settings/agent_configuration/create_or_update_configuration.ts index 5e7a1551fdc6c..666195217ccdc 100644 --- a/x-pack/plugins/apm/server/routes/settings/agent_configuration/create_or_update_configuration.ts +++ b/x-pack/plugins/apm/server/routes/settings/agent_configuration/create_or_update_configuration.ts @@ -14,6 +14,7 @@ import { APMIndexDocumentParams, APMInternalESClient, } from '../../../lib/helpers/create_es_client/create_internal_es_client'; +import { APM_AGENT_CONFIGURATION_INDEX } from '../apm_indices/get_apm_indices'; export function createOrUpdateConfiguration({ configurationId, @@ -26,7 +27,7 @@ export function createOrUpdateConfiguration({ }) { const params: APMIndexDocumentParams = { refresh: true, - index: internalESClient.apmIndices.apmAgentConfigurationIndex, + index: APM_AGENT_CONFIGURATION_INDEX, body: { agent_name: configurationIntake.agent_name, service: { diff --git a/x-pack/plugins/apm/server/routes/settings/agent_configuration/delete_configuration.ts b/x-pack/plugins/apm/server/routes/settings/agent_configuration/delete_configuration.ts index 58896f40b2ede..d0897fb48941b 100644 --- a/x-pack/plugins/apm/server/routes/settings/agent_configuration/delete_configuration.ts +++ b/x-pack/plugins/apm/server/routes/settings/agent_configuration/delete_configuration.ts @@ -6,6 +6,7 @@ */ import { APMInternalESClient } from '../../../lib/helpers/create_es_client/create_internal_es_client'; +import { APM_AGENT_CONFIGURATION_INDEX } from '../apm_indices/get_apm_indices'; export async function deleteConfiguration({ configurationId, @@ -16,7 +17,7 @@ export async function deleteConfiguration({ }) { const params = { refresh: 'wait_for' as const, - index: internalESClient.apmIndices.apmAgentConfigurationIndex, + index: APM_AGENT_CONFIGURATION_INDEX, id: configurationId, }; diff --git a/x-pack/plugins/apm/server/routes/settings/agent_configuration/find_exact_configuration.ts b/x-pack/plugins/apm/server/routes/settings/agent_configuration/find_exact_configuration.ts index b9a0d5cb36fd0..3d65d534e9545 100644 --- a/x-pack/plugins/apm/server/routes/settings/agent_configuration/find_exact_configuration.ts +++ b/x-pack/plugins/apm/server/routes/settings/agent_configuration/find_exact_configuration.ts @@ -12,6 +12,7 @@ import { SERVICE_NAME, } from '../../../../common/es_fields/apm'; import { APMInternalESClient } from '../../../lib/helpers/create_es_client/create_internal_es_client'; +import { APM_AGENT_CONFIGURATION_INDEX } from '../apm_indices/get_apm_indices'; import { convertConfigSettingsToString } from './convert_settings_to_string'; import { getConfigsAppliedToAgentsThroughFleet } from './get_config_applied_to_agent_through_fleet'; @@ -31,7 +32,7 @@ export async function findExactConfiguration({ : { bool: { must_not: [{ exists: { field: SERVICE_ENVIRONMENT } }] } }; const params = { - index: internalESClient.apmIndices.apmAgentConfigurationIndex, + index: APM_AGENT_CONFIGURATION_INDEX, body: { query: { bool: { filter: [serviceNameFilter, environmentFilter] }, diff --git a/x-pack/plugins/apm/server/routes/settings/agent_configuration/get_environments/get_existing_environments_for_service.ts b/x-pack/plugins/apm/server/routes/settings/agent_configuration/get_environments/get_existing_environments_for_service.ts index 4223ecdf0ed21..3e588a5c8d9e5 100644 --- a/x-pack/plugins/apm/server/routes/settings/agent_configuration/get_environments/get_existing_environments_for_service.ts +++ b/x-pack/plugins/apm/server/routes/settings/agent_configuration/get_environments/get_existing_environments_for_service.ts @@ -11,6 +11,7 @@ import { } from '../../../../../common/es_fields/apm'; import { ALL_OPTION_VALUE } from '../../../../../common/agent_configuration/all_option'; import { APMInternalESClient } from '../../../../lib/helpers/create_es_client/create_internal_es_client'; +import { APM_AGENT_CONFIGURATION_INDEX } from '../../apm_indices/get_apm_indices'; export async function getExistingEnvironmentsForService({ serviceName, @@ -26,7 +27,7 @@ export async function getExistingEnvironmentsForService({ : { must_not: [{ exists: { field: SERVICE_NAME } }] }; const params = { - index: internalESClient.apmIndices.apmAgentConfigurationIndex, + index: APM_AGENT_CONFIGURATION_INDEX, body: { size: 0, query: { bool }, diff --git a/x-pack/plugins/apm/server/routes/settings/agent_configuration/list_configurations.ts b/x-pack/plugins/apm/server/routes/settings/agent_configuration/list_configurations.ts index 7f6a5dd20bf0b..6a7e53097945e 100644 --- a/x-pack/plugins/apm/server/routes/settings/agent_configuration/list_configurations.ts +++ b/x-pack/plugins/apm/server/routes/settings/agent_configuration/list_configurations.ts @@ -9,12 +9,13 @@ import { AgentConfiguration } from '../../../../common/agent_configuration/confi import { convertConfigSettingsToString } from './convert_settings_to_string'; import { getConfigsAppliedToAgentsThroughFleet } from './get_config_applied_to_agent_through_fleet'; import { APMInternalESClient } from '../../../lib/helpers/create_es_client/create_internal_es_client'; +import { APM_AGENT_CONFIGURATION_INDEX } from '../apm_indices/get_apm_indices'; export async function listConfigurations( internalESClient: APMInternalESClient ) { const params = { - index: internalESClient.apmIndices.apmAgentConfigurationIndex, + index: APM_AGENT_CONFIGURATION_INDEX, size: 200, }; diff --git a/x-pack/plugins/apm/server/routes/settings/agent_configuration/mark_applied_by_agent.ts b/x-pack/plugins/apm/server/routes/settings/agent_configuration/mark_applied_by_agent.ts index c104b4743df91..6b32bbacd3b2a 100644 --- a/x-pack/plugins/apm/server/routes/settings/agent_configuration/mark_applied_by_agent.ts +++ b/x-pack/plugins/apm/server/routes/settings/agent_configuration/mark_applied_by_agent.ts @@ -7,6 +7,7 @@ import { AgentConfiguration } from '../../../../common/agent_configuration/configuration_types'; import { APMInternalESClient } from '../../../lib/helpers/create_es_client/create_internal_es_client'; +import { APM_AGENT_CONFIGURATION_INDEX } from '../apm_indices/get_apm_indices'; // We're not wrapping this function with a span as it is not blocking the request export async function markAppliedByAgent({ @@ -19,7 +20,7 @@ export async function markAppliedByAgent({ internalESClient: APMInternalESClient; }) { const params = { - index: internalESClient.apmIndices.apmAgentConfigurationIndex, + index: APM_AGENT_CONFIGURATION_INDEX, id, // by specifying the `id` elasticsearch will do an "upsert" body: { ...body, diff --git a/x-pack/plugins/apm/server/routes/settings/agent_configuration/search_configurations.ts b/x-pack/plugins/apm/server/routes/settings/agent_configuration/search_configurations.ts index 51f6702a0fb57..77da47b62c1c8 100644 --- a/x-pack/plugins/apm/server/routes/settings/agent_configuration/search_configurations.ts +++ b/x-pack/plugins/apm/server/routes/settings/agent_configuration/search_configurations.ts @@ -13,6 +13,7 @@ import { import { AgentConfiguration } from '../../../../common/agent_configuration/configuration_types'; import { convertConfigSettingsToString } from './convert_settings_to_string'; import { APMInternalESClient } from '../../../lib/helpers/create_es_client/create_internal_es_client'; +import { APM_AGENT_CONFIGURATION_INDEX } from '../apm_indices/get_apm_indices'; export async function searchConfigurations({ service, @@ -47,7 +48,7 @@ export async function searchConfigurations({ : []; const params = { - index: internalESClient.apmIndices.apmAgentConfigurationIndex, + index: APM_AGENT_CONFIGURATION_INDEX, body: { query: { bool: { diff --git a/x-pack/plugins/apm/server/routes/settings/apm_indices/get_apm_indices.ts b/x-pack/plugins/apm/server/routes/settings/apm_indices/get_apm_indices.ts index f80c81d0bd0ef..9153c7113ccba 100644 --- a/x-pack/plugins/apm/server/routes/settings/apm_indices/get_apm_indices.ts +++ b/x-pack/plugins/apm/server/routes/settings/apm_indices/get_apm_indices.ts @@ -16,16 +16,17 @@ import { withApmSpan } from '../../../utils/with_apm_span'; import { APMIndices } from '../../../saved_objects/apm_indices'; export type ApmIndicesConfig = Readonly<{ - sourcemap: string; error: string; onboarding: string; span: string; transaction: string; metric: string; - apmAgentConfigurationIndex: string; - apmCustomLinkIndex: string; }>; +export const APM_AGENT_CONFIGURATION_INDEX = '.apm-agent-configuration'; +export const APM_CUSTOM_LINK_INDEX = '.apm-custom-link'; +export const APM_SOURCE_MAP_INDEX = '.apm-source-map'; + type ISavedObjectsClient = Pick; async function getApmIndicesSavedObject( @@ -44,15 +45,11 @@ async function getApmIndicesSavedObject( export function getApmIndicesConfig(config: APMConfig): ApmIndicesConfig { return { - sourcemap: config.indices.sourcemap, error: config.indices.error, onboarding: config.indices.onboarding, span: config.indices.span, transaction: config.indices.transaction, metric: config.indices.metric, - // system indices, not configurable - apmAgentConfigurationIndex: '.apm-agent-configuration', - apmCustomLinkIndex: '.apm-custom-link', }; } diff --git a/x-pack/plugins/apm/server/routes/settings/apm_indices/route.ts b/x-pack/plugins/apm/server/routes/settings/apm_indices/route.ts index 81d37c8d34608..4217fd7c19d6e 100644 --- a/x-pack/plugins/apm/server/routes/settings/apm_indices/route.ts +++ b/x-pack/plugins/apm/server/routes/settings/apm_indices/route.ts @@ -25,7 +25,6 @@ const apmIndexSettingsRoute = createApmServerRoute({ | 'span' | 'error' | 'metric' - | 'sourcemap' | 'onboarding'; defaultValue: string; savedValue: string | undefined; @@ -66,7 +65,6 @@ const saveApmIndicesRoute = createApmServerRoute({ }, params: t.type({ body: t.partial({ - sourcemap: t.string, error: t.string, onboarding: t.string, span: t.string, diff --git a/x-pack/plugins/apm/server/routes/settings/custom_link/__snapshots__/list_custom_links.test.ts.snap b/x-pack/plugins/apm/server/routes/settings/custom_link/__snapshots__/list_custom_links.test.ts.snap index d05f0bc081eba..520b185fc797e 100644 --- a/x-pack/plugins/apm/server/routes/settings/custom_link/__snapshots__/list_custom_links.test.ts.snap +++ b/x-pack/plugins/apm/server/routes/settings/custom_link/__snapshots__/list_custom_links.test.ts.snap @@ -1,5 +1,90 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`List Custom Links fetches all custom links 1`] = `undefined`; +exports[`List Custom Links fetches all custom links 1`] = ` +Object { + "body": Object { + "query": Object { + "bool": Object { + "filter": Array [], + }, + }, + "sort": Array [ + Object { + "label.keyword": Object { + "order": "asc", + }, + }, + ], + }, + "index": ".apm-custom-link", + "size": 500, +} +`; -exports[`List Custom Links filters custom links 1`] = `undefined`; +exports[`List Custom Links filters custom links 1`] = ` +Object { + "body": Object { + "query": Object { + "bool": Object { + "filter": Array [ + Object { + "bool": Object { + "minimum_should_match": 1, + "should": Array [ + Object { + "term": Object { + "service.name": "foo", + }, + }, + Object { + "bool": Object { + "must_not": Array [ + Object { + "exists": Object { + "field": "service.name", + }, + }, + ], + }, + }, + ], + }, + }, + Object { + "bool": Object { + "minimum_should_match": 1, + "should": Array [ + Object { + "term": Object { + "transaction.name": "bar", + }, + }, + Object { + "bool": Object { + "must_not": Array [ + Object { + "exists": Object { + "field": "transaction.name", + }, + }, + ], + }, + }, + ], + }, + }, + ], + }, + }, + "sort": Array [ + Object { + "label.keyword": Object { + "order": "asc", + }, + }, + ], + }, + "index": ".apm-custom-link", + "size": 500, +} +`; diff --git a/x-pack/plugins/apm/server/routes/settings/custom_link/create_custom_link_index.ts b/x-pack/plugins/apm/server/routes/settings/custom_link/create_custom_link_index.ts index 4f9fc321a19f9..b8bfc3b3f2f72 100644 --- a/x-pack/plugins/apm/server/routes/settings/custom_link/create_custom_link_index.ts +++ b/x-pack/plugins/apm/server/routes/settings/custom_link/create_custom_link_index.ts @@ -11,21 +11,17 @@ import { createOrUpdateIndex, Mappings, } from '@kbn/observability-plugin/server'; -import { APMConfig } from '../../..'; -import { getApmIndicesConfig } from '../apm_indices/get_apm_indices'; +import { APM_CUSTOM_LINK_INDEX } from '../apm_indices/get_apm_indices'; export const createApmCustomLinkIndex = async ({ client, - config, logger, }: { client: ElasticsearchClient; - config: APMConfig; logger: Logger; }) => { - const index = getApmIndicesConfig(config).apmCustomLinkIndex; return createOrUpdateIndex({ - index, + index: APM_CUSTOM_LINK_INDEX, client, logger, mappings, diff --git a/x-pack/plugins/apm/server/routes/settings/custom_link/create_or_update_custom_link.test.ts b/x-pack/plugins/apm/server/routes/settings/custom_link/create_or_update_custom_link.test.ts index f93e1f87b7360..4d768cdff09d4 100644 --- a/x-pack/plugins/apm/server/routes/settings/custom_link/create_or_update_custom_link.test.ts +++ b/x-pack/plugins/apm/server/routes/settings/custom_link/create_or_update_custom_link.test.ts @@ -8,18 +8,12 @@ import { mockNow } from '../../../utils/test_helpers'; import { CustomLink } from '../../../../common/custom_link/custom_link_types'; import { createOrUpdateCustomLink } from './create_or_update_custom_link'; -import { ApmIndicesConfig } from '../apm_indices/get_apm_indices'; import { APMInternalESClient } from '../../../lib/helpers/create_es_client/create_internal_es_client'; describe('Create or Update Custom link', () => { const internalClientIndexMock = jest.fn(); - const mockIndices = { - apmCustomLinkIndex: 'apmCustomLinkIndex', - } as unknown as ApmIndicesConfig; - const mockInternalESClient = { - apmIndices: mockIndices, index: internalClientIndexMock, } as unknown as APMInternalESClient; @@ -48,7 +42,7 @@ describe('Create or Update Custom link', () => { 'create_or_update_custom_link', { refresh: true, - index: 'apmCustomLinkIndex', + index: '.apm-custom-link', body: { '@timestamp': 1570737000000, label: 'foo', @@ -69,7 +63,7 @@ describe('Create or Update Custom link', () => { 'create_or_update_custom_link', { refresh: true, - index: 'apmCustomLinkIndex', + index: '.apm-custom-link', id: 'bar', body: { '@timestamp': 1570737000000, diff --git a/x-pack/plugins/apm/server/routes/settings/custom_link/create_or_update_custom_link.ts b/x-pack/plugins/apm/server/routes/settings/custom_link/create_or_update_custom_link.ts index 0eaada68ba9b0..1bda27f7eb78b 100644 --- a/x-pack/plugins/apm/server/routes/settings/custom_link/create_or_update_custom_link.ts +++ b/x-pack/plugins/apm/server/routes/settings/custom_link/create_or_update_custom_link.ts @@ -14,6 +14,7 @@ import { APMIndexDocumentParams, APMInternalESClient, } from '../../../lib/helpers/create_es_client/create_internal_es_client'; +import { APM_CUSTOM_LINK_INDEX } from '../apm_indices/get_apm_indices'; export function createOrUpdateCustomLink({ customLinkId, @@ -26,7 +27,7 @@ export function createOrUpdateCustomLink({ }) { const params: APMIndexDocumentParams = { refresh: true, - index: internalESClient.apmIndices.apmCustomLinkIndex, + index: APM_CUSTOM_LINK_INDEX, body: { '@timestamp': Date.now(), ...toESFormat(customLink), diff --git a/x-pack/plugins/apm/server/routes/settings/custom_link/delete_custom_link.ts b/x-pack/plugins/apm/server/routes/settings/custom_link/delete_custom_link.ts index 53f6b830aa44a..16318fe696712 100644 --- a/x-pack/plugins/apm/server/routes/settings/custom_link/delete_custom_link.ts +++ b/x-pack/plugins/apm/server/routes/settings/custom_link/delete_custom_link.ts @@ -6,6 +6,7 @@ */ import { APMInternalESClient } from '../../../lib/helpers/create_es_client/create_internal_es_client'; +import { APM_CUSTOM_LINK_INDEX } from '../apm_indices/get_apm_indices'; export function deleteCustomLink({ customLinkId, @@ -16,7 +17,7 @@ export function deleteCustomLink({ }) { const params = { refresh: 'wait_for' as const, - index: internalESClient.apmIndices.apmCustomLinkIndex, + index: APM_CUSTOM_LINK_INDEX, id: customLinkId, }; diff --git a/x-pack/plugins/apm/server/routes/settings/custom_link/list_custom_links.ts b/x-pack/plugins/apm/server/routes/settings/custom_link/list_custom_links.ts index c435767ffd424..27b7c08ca9a4b 100644 --- a/x-pack/plugins/apm/server/routes/settings/custom_link/list_custom_links.ts +++ b/x-pack/plugins/apm/server/routes/settings/custom_link/list_custom_links.ts @@ -14,6 +14,7 @@ import { import { fromESFormat } from './helper'; import { filterOptionsRt } from './custom_link_types'; import { APMInternalESClient } from '../../../lib/helpers/create_es_client/create_internal_es_client'; +import { APM_CUSTOM_LINK_INDEX } from '../apm_indices/get_apm_indices'; export async function listCustomLinks({ internalESClient, @@ -35,7 +36,7 @@ export async function listCustomLinks({ }); const params = { - index: internalESClient.apmIndices.apmCustomLinkIndex, + index: APM_CUSTOM_LINK_INDEX, size: 500, body: { query: { diff --git a/x-pack/plugins/apm/server/routes/source_maps/bulk_create_apm_source_maps.ts b/x-pack/plugins/apm/server/routes/source_maps/bulk_create_apm_source_maps.ts new file mode 100644 index 0000000000000..f681642c0bc4b --- /dev/null +++ b/x-pack/plugins/apm/server/routes/source_maps/bulk_create_apm_source_maps.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { Artifact } from '@kbn/fleet-plugin/server'; +import { getUnzippedArtifactBody } from '../fleet/source_maps'; +import { APM_SOURCE_MAP_INDEX } from '../settings/apm_indices/get_apm_indices'; +import { ApmSourceMap } from './create_apm_source_map_index_template'; +import { getEncodedContent, getSourceMapId } from './sourcemap_utils'; + +export async function bulkCreateApmSourceMaps({ + artifacts, + internalESClient, +}: { + artifacts: Artifact[]; + internalESClient: ElasticsearchClient; +}) { + const docs = await Promise.all( + artifacts.map(async (artifact): Promise => { + const { serviceName, serviceVersion, bundleFilepath, sourceMap } = + await getUnzippedArtifactBody(artifact.body); + + const { contentEncoded, contentHash } = await getEncodedContent( + sourceMap + ); + + return { + fleet_id: artifact.id, + created: artifact.created, + content: contentEncoded, + content_sha256: contentHash, + file: { + path: bundleFilepath, + }, + service: { + name: serviceName, + version: serviceVersion, + }, + }; + }) + ); + + return internalESClient.bulk({ + body: docs.flatMap((doc) => { + const id = getSourceMapId({ + serviceName: doc.service.name, + serviceVersion: doc.service.version, + bundleFilepath: doc.file.path, + }); + return [{ create: { _index: APM_SOURCE_MAP_INDEX, _id: id } }, doc]; + }), + }); +} diff --git a/x-pack/plugins/apm/server/routes/source_maps/create_apm_source_map.ts b/x-pack/plugins/apm/server/routes/source_maps/create_apm_source_map.ts new file mode 100644 index 0000000000000..f4a49d475ceef --- /dev/null +++ b/x-pack/plugins/apm/server/routes/source_maps/create_apm_source_map.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { isElasticsearchVersionConflictError } from '@kbn/fleet-plugin/server/errors/utils'; +import { Logger } from '@kbn/core/server'; +import { APM_SOURCE_MAP_INDEX } from '../settings/apm_indices/get_apm_indices'; +import { ApmSourceMap } from './create_apm_source_map_index_template'; +import { SourceMap } from './route'; +import { getEncodedContent, getSourceMapId } from './sourcemap_utils'; + +export async function createApmSourceMap({ + internalESClient, + logger, + fleetId, + created, + sourceMapContent, + bundleFilepath, + serviceName, + serviceVersion, +}: { + internalESClient: ElasticsearchClient; + logger: Logger; + fleetId: string; + created: string; + sourceMapContent: SourceMap; + bundleFilepath: string; + serviceName: string; + serviceVersion: string; +}) { + const { contentEncoded, contentHash } = await getEncodedContent( + sourceMapContent + ); + const doc: ApmSourceMap = { + fleet_id: fleetId, + created, + content: contentEncoded, + content_sha256: contentHash, + file: { path: bundleFilepath }, + service: { name: serviceName, version: serviceVersion }, + }; + + try { + const id = getSourceMapId({ serviceName, serviceVersion, bundleFilepath }); + logger.debug(`Create APM source map: "${id}"`); + return await internalESClient.create({ + index: APM_SOURCE_MAP_INDEX, + id, + body: doc, + }); + } catch (e) { + // we ignore 409 errors from the create (document already exists) + if (!isElasticsearchVersionConflictError(e)) { + throw e; + } + } +} diff --git a/x-pack/plugins/apm/server/routes/source_maps/create_apm_source_map_index_template.ts b/x-pack/plugins/apm/server/routes/source_maps/create_apm_source_map_index_template.ts new file mode 100644 index 0000000000000..25a7cf5ae5815 --- /dev/null +++ b/x-pack/plugins/apm/server/routes/source_maps/create_apm_source_map_index_template.ts @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { createOrUpdateIndexTemplate } from '@kbn/observability-plugin/server'; +import { APM_SOURCE_MAP_INDEX } from '../settings/apm_indices/get_apm_indices'; + +const indexTemplate: IndicesPutIndexTemplateRequest = { + name: 'apm-source-map', + body: { + version: 1, + index_patterns: [APM_SOURCE_MAP_INDEX], + template: { + settings: { + number_of_shards: 1, + index: { + hidden: true, + }, + }, + mappings: { + dynamic: 'strict', + properties: { + fleet_id: { + type: 'keyword', + }, + created: { + type: 'date', + }, + content: { + type: 'binary', + }, + content_sha256: { + type: 'keyword', + }, + 'file.path': { + type: 'keyword', + }, + 'service.name': { + type: 'keyword', + }, + 'service.version': { + type: 'keyword', + }, + }, + }, + }, + }, +}; + +export async function createApmSourceMapIndexTemplate({ + client, + logger, +}: { + client: ElasticsearchClient; + logger: Logger; +}) { + // create index template + await createOrUpdateIndexTemplate({ indexTemplate, client, logger }); + + // create index if it doesn't exist + const indexExists = await client.indices.exists({ + index: APM_SOURCE_MAP_INDEX, + }); + + if (!indexExists) { + logger.debug(`Create index: "${APM_SOURCE_MAP_INDEX}"`); + await client.indices.create({ index: APM_SOURCE_MAP_INDEX }); + } +} + +export interface ApmSourceMap { + fleet_id: string; + created: string; + content: string; + content_sha256: string; + file: { + path: string; + }; + service: { + name: string; + version: string; + }; +} diff --git a/x-pack/plugins/apm/server/routes/source_maps/delete_apm_sourcemap.ts b/x-pack/plugins/apm/server/routes/source_maps/delete_apm_sourcemap.ts new file mode 100644 index 0000000000000..ffdaf02c165d8 --- /dev/null +++ b/x-pack/plugins/apm/server/routes/source_maps/delete_apm_sourcemap.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { APM_SOURCE_MAP_INDEX } from '../settings/apm_indices/get_apm_indices'; + +export async function deleteApmSourceMap({ + internalESClient, + fleetId, +}: { + internalESClient: ElasticsearchClient; + fleetId: string; +}) { + return internalESClient.deleteByQuery({ + index: APM_SOURCE_MAP_INDEX, + query: { + bool: { + filter: [{ term: { fleet_id: fleetId } }], + }, + }, + }); +} diff --git a/x-pack/plugins/apm/server/routes/source_maps/route.ts b/x-pack/plugins/apm/server/routes/source_maps/route.ts index b4d005373c4be..a15e6b07f9f17 100644 --- a/x-pack/plugins/apm/server/routes/source_maps/route.ts +++ b/x-pack/plugins/apm/server/routes/source_maps/route.ts @@ -10,8 +10,8 @@ import { SavedObjectsClientContract } from '@kbn/core/server'; import { jsonRt, toNumberRt } from '@kbn/io-ts-utils'; import { Artifact } from '@kbn/fleet-plugin/server'; import { - createApmArtifact, - deleteApmArtifact, + createFleetSourceMapArtifact, + deleteFleetSourcemapArtifact, listSourceMapArtifacts, updateSourceMapsOnFleetPolicies, getCleanedBundleFilePath, @@ -20,6 +20,9 @@ import { import { getInternalSavedObjectsClient } from '../../lib/helpers/get_internal_saved_objects_client'; import { createApmServerRoute } from '../apm_routes/create_apm_server_route'; import { stringFromBufferRt } from '../../utils/string_from_buffer_rt'; +import { createApmSourceMap } from './create_apm_source_map'; +import { deleteApmSourceMap } from './delete_apm_sourcemap'; +import { runFleetSourcemapArtifactsMigration } from './schedule_source_map_migration'; export const sourceMapRt = t.intersection([ t.type({ @@ -89,35 +92,55 @@ const uploadSourceMapRoute = createApmServerRoute({ .pipe(sourceMapRt), }), }), - handler: async ({ params, plugins, core }): Promise => { + handler: async ({ + params, + plugins, + core, + logger, + }): Promise => { const { service_name: serviceName, service_version: serviceVersion, bundle_filepath: bundleFilepath, - sourcemap: sourceMap, + sourcemap: sourceMapContent, } = params.body; const cleanedBundleFilepath = getCleanedBundleFilePath(bundleFilepath); const fleetPluginStart = await plugins.fleet?.start(); const coreStart = await core.start(); - const esClient = coreStart.elasticsearch.client.asInternalUser; + const internalESClient = coreStart.elasticsearch.client.asInternalUser; const savedObjectsClient = await getInternalSavedObjectsClient(core.setup); try { if (fleetPluginStart) { - const artifact = await createApmArtifact({ + // create source map as fleet artifact + const artifact = await createFleetSourceMapArtifact({ fleetPluginStart, apmArtifactBody: { serviceName, serviceVersion, bundleFilepath: cleanedBundleFilepath, - sourceMap, + sourceMap: sourceMapContent, }, }); + + // sync source map to APM managed index + await createApmSourceMap({ + internalESClient, + logger, + fleetId: artifact.id, + created: artifact.created, + sourceMapContent, + bundleFilepath: cleanedBundleFilepath, + serviceName, + serviceVersion, + }); + + // sync source map to fleet policy await updateSourceMapsOnFleetPolicies({ core, fleetPluginStart, savedObjectsClient: savedObjectsClient as unknown as SavedObjectsClientContract, - elasticsearchClient: esClient, + internalESClient, }); return artifact; @@ -143,17 +166,18 @@ const deleteSourceMapRoute = createApmServerRoute({ const fleetPluginStart = await plugins.fleet?.start(); const { id } = params.path; const coreStart = await core.start(); - const esClient = coreStart.elasticsearch.client.asInternalUser; + const internalESClient = coreStart.elasticsearch.client.asInternalUser; const savedObjectsClient = await getInternalSavedObjectsClient(core.setup); try { if (fleetPluginStart) { - await deleteApmArtifact({ id, fleetPluginStart }); + await deleteFleetSourcemapArtifact({ id, fleetPluginStart }); + await deleteApmSourceMap({ internalESClient, fleetId: id }); await updateSourceMapsOnFleetPolicies({ core, fleetPluginStart, savedObjectsClient: savedObjectsClient as unknown as SavedObjectsClientContract, - elasticsearchClient: esClient, + internalESClient, }); } } catch (e) { @@ -165,8 +189,27 @@ const deleteSourceMapRoute = createApmServerRoute({ }, }); +const migrateFleetArtifactsSourceMapRoute = createApmServerRoute({ + endpoint: 'POST /internal/apm/sourcemaps/migrate_fleet_artifacts', + options: { tags: ['access:apm', 'access:apm_write'] }, + handler: async ({ plugins, core, logger }): Promise => { + const fleet = await plugins.fleet?.start(); + const coreStart = await core.start(); + const internalESClient = coreStart.elasticsearch.client.asInternalUser; + + if (fleet) { + return runFleetSourcemapArtifactsMigration({ + fleet, + internalESClient, + logger, + }); + } + }, +}); + export const sourceMapsRouteRepository = { ...listSourceMapRoute, ...uploadSourceMapRoute, ...deleteSourceMapRoute, + ...migrateFleetArtifactsSourceMapRoute, }; diff --git a/x-pack/plugins/apm/server/routes/source_maps/schedule_source_map_migration.ts b/x-pack/plugins/apm/server/routes/source_maps/schedule_source_map_migration.ts new file mode 100644 index 0000000000000..d0a172581fc46 --- /dev/null +++ b/x-pack/plugins/apm/server/routes/source_maps/schedule_source_map_migration.ts @@ -0,0 +1,225 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { FleetStartContract } from '@kbn/fleet-plugin/server'; +import { FleetArtifactsClient } from '@kbn/fleet-plugin/server/services'; +import { TaskManagerSetupContract } from '@kbn/task-manager-plugin/server'; +import { CoreStart, Logger } from '@kbn/core/server'; +import { getApmArtifactClient } from '../fleet/source_maps'; +import { bulkCreateApmSourceMaps } from './bulk_create_apm_source_maps'; +import { APM_SOURCE_MAP_INDEX } from '../settings/apm_indices/get_apm_indices'; +import { ApmSourceMap } from './create_apm_source_map_index_template'; +import { APMPluginStartDependencies } from '../../types'; +import { createApmSourceMapIndexTemplate } from './create_apm_source_map_index_template'; + +const PER_PAGE = 10; +const TASK_ID = 'apm-source-map-migration-task-id'; +const TASK_TYPE = 'apm-source-map-migration-task'; + +export async function scheduleSourceMapMigration({ + coreStartPromise, + pluginStartPromise, + fleetStartPromise, + taskManager, + logger, +}: { + coreStartPromise: Promise; + pluginStartPromise: Promise; + fleetStartPromise?: Promise; + taskManager?: TaskManagerSetupContract; + logger: Logger; +}) { + if (!taskManager) { + return; + } + + logger.debug(`Register task "${TASK_TYPE}"`); + + taskManager.registerTaskDefinitions({ + [TASK_TYPE]: { + title: 'Migrate fleet source map artifacts', + description: + 'Migrates fleet source map artifacts to `.apm-source-map` index', + timeout: '1h', + maxAttempts: 5, + maxConcurrency: 1, + createTaskRunner() { + const taskState: TaskState = { isAborted: false }; + + return { + async run() { + logger.debug(`Run task: "${TASK_TYPE}"`); + const coreStart = await coreStartPromise; + const internalESClient = + coreStart.elasticsearch.client.asInternalUser; + + // ensure that the index template has been created before running migration + await createApmSourceMapIndexTemplate({ + client: internalESClient, + logger, + }); + + const fleet = await fleetStartPromise; + if (fleet) { + await runFleetSourcemapArtifactsMigration({ + taskState, + fleet, + internalESClient, + logger, + }); + } + }, + + async cancel() { + taskState.isAborted = true; + logger.debug(`Task cancelled: "${TASK_TYPE}"`); + }, + }; + }, + }, + }); + + const pluginStart = await pluginStartPromise; + const taskManagerStart = pluginStart.taskManager; + + if (taskManagerStart) { + logger.debug(`Task scheduled: "${TASK_TYPE}"`); + await pluginStart.taskManager?.ensureScheduled({ + id: TASK_ID, + taskType: TASK_TYPE, + scope: ['apm'], + params: {}, + state: {}, + }); + } +} + +interface TaskState { + isAborted: boolean; +} + +export async function runFleetSourcemapArtifactsMigration({ + taskState, + fleet, + internalESClient, + logger, +}: { + taskState?: TaskState; + fleet: FleetStartContract; + internalESClient: ElasticsearchClient; + logger: Logger; +}) { + try { + const latestApmSourceMapTimestamp = await getLatestApmSourceMap( + internalESClient + ); + const createdDateFilter = latestApmSourceMapTimestamp + ? ` AND created:>${asLuceneEncoding(latestApmSourceMapTimestamp)}` + : ''; + + await paginateArtifacts({ + taskState, + page: 1, + apmArtifactClient: getApmArtifactClient(fleet), + kuery: `type: sourcemap${createdDateFilter}`, + logger, + internalESClient, + }); + } catch (e) { + logger.error('Failed to migrate APM fleet source map artifacts'); + logger.error(e); + } +} + +// will convert "2022-12-12T21:21:51.203Z" to "2022-12-12T21\:21\:51.203Z" because colons are not allowed when using Lucene syntax +function asLuceneEncoding(timestamp: string) { + return timestamp.replaceAll(':', '\\:'); +} + +async function getArtifactsForPage({ + page, + apmArtifactClient, + kuery, +}: { + page: number; + apmArtifactClient: FleetArtifactsClient; + kuery: string; +}) { + return await apmArtifactClient.listArtifacts({ + kuery, + perPage: PER_PAGE, + page, + sortOrder: 'asc', + sortField: 'created', + }); +} + +async function paginateArtifacts({ + taskState, + page, + apmArtifactClient, + kuery, + logger, + internalESClient, +}: { + taskState?: TaskState; + page: number; + apmArtifactClient: FleetArtifactsClient; + kuery: string; + logger: Logger; + internalESClient: ElasticsearchClient; +}) { + if (taskState?.isAborted) { + return; + } + + const { total, items: artifacts } = await getArtifactsForPage({ + page, + apmArtifactClient, + kuery, + }); + + if (artifacts.length === 0) { + logger.debug('No source maps need to be migrated'); + return; + } + + const migratedCount = (page - 1) * PER_PAGE + artifacts.length; + logger.info(`Migrating ${migratedCount} of ${total} source maps`); + + await bulkCreateApmSourceMaps({ artifacts, internalESClient }); + + const hasMorePages = total > migratedCount; + if (hasMorePages) { + await paginateArtifacts({ + taskState, + page: page + 1, + apmArtifactClient, + kuery, + logger, + internalESClient, + }); + } else { + logger.info(`Successfully migrated ${total} source maps`); + } +} + +async function getLatestApmSourceMap(internalESClient: ElasticsearchClient) { + const params = { + index: APM_SOURCE_MAP_INDEX, + track_total_hits: false, + size: 1, + _source: ['created'], + sort: [{ created: { order: 'desc' } }], + body: { + query: { match_all: {} }, + }, + }; + const res = await internalESClient.search(params); + return res.hits.hits[0]?._source?.created; +} diff --git a/x-pack/plugins/apm/server/routes/source_maps/sourcemap_utils.ts b/x-pack/plugins/apm/server/routes/source_maps/sourcemap_utils.ts new file mode 100644 index 0000000000000..20ff2fa4bd41c --- /dev/null +++ b/x-pack/plugins/apm/server/routes/source_maps/sourcemap_utils.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { deflate } from 'zlib'; +import { BinaryLike, createHash } from 'crypto'; +import { promisify } from 'util'; +import { SourceMap } from './route'; + +const deflateAsync = promisify(deflate); + +function asSha256Encoded(content: BinaryLike): string { + return createHash('sha256').update(content).digest('hex'); +} + +export async function getEncodedContent(sourceMapContent: SourceMap) { + const contentBuffer = Buffer.from(JSON.stringify(sourceMapContent)); + const contentZipped = await deflateAsync(contentBuffer); + const contentEncoded = contentZipped.toString('base64'); + const contentHash = asSha256Encoded(contentZipped); + return { contentEncoded, contentHash }; +} + +export function getSourceMapId({ + serviceName, + serviceVersion, + bundleFilepath, +}: { + serviceName: string; + serviceVersion: string; + bundleFilepath: string; +}) { + return [serviceName, serviceVersion, bundleFilepath].join('-'); +} diff --git a/x-pack/plugins/apm/server/routes/storage_explorer/is_cross_cluster_search.test.ts b/x-pack/plugins/apm/server/routes/storage_explorer/is_cross_cluster_search.test.ts index d4604df07e3f6..edb28d9bfa490 100644 --- a/x-pack/plugins/apm/server/routes/storage_explorer/is_cross_cluster_search.test.ts +++ b/x-pack/plugins/apm/server/routes/storage_explorer/is_cross_cluster_search.test.ts @@ -44,7 +44,6 @@ describe('isCrossClusterSearch', () => { metric: '', error: '', onboarding: 'apm-*,remote_cluster:apm-*', - sourcemap: 'apm-*,remote_cluster:apm-*', } as ApmIndicesConfig, } as unknown as APMEventClient; diff --git a/x-pack/plugins/apm/server/saved_objects/apm_indices.ts b/x-pack/plugins/apm/server/saved_objects/apm_indices.ts index 19c90f41461c2..b31a97bcb9760 100644 --- a/x-pack/plugins/apm/server/saved_objects/apm_indices.ts +++ b/x-pack/plugins/apm/server/saved_objects/apm_indices.ts @@ -11,7 +11,6 @@ import { updateApmOssIndexPaths } from './migrations/update_apm_oss_index_paths' export interface APMIndices { apmIndices?: { - sourcemap?: string; error?: string; onboarding?: string; span?: string; diff --git a/x-pack/plugins/apm/server/saved_objects/migrations/update_apm_oss_index_paths.ts b/x-pack/plugins/apm/server/saved_objects/migrations/update_apm_oss_index_paths.ts index 72ba40db0ce05..1d129eda16e6d 100644 --- a/x-pack/plugins/apm/server/saved_objects/migrations/update_apm_oss_index_paths.ts +++ b/x-pack/plugins/apm/server/saved_objects/migrations/update_apm_oss_index_paths.ts @@ -6,7 +6,6 @@ */ const apmIndexConfigs = [ - ['sourcemap', 'apm_oss.sourcemapIndices'], ['error', 'apm_oss.errorIndices'], ['onboarding', 'apm_oss.onboardingIndices'], ['span', 'apm_oss.spanIndices'], diff --git a/x-pack/plugins/apm/server/utils/test_helpers.tsx b/x-pack/plugins/apm/server/utils/test_helpers.tsx index 22b0de8af8e26..37db53111903c 100644 --- a/x-pack/plugins/apm/server/utils/test_helpers.tsx +++ b/x-pack/plugins/apm/server/utils/test_helpers.tsx @@ -52,7 +52,6 @@ export async function inspectSearchParams( const indices: { [Property in keyof APMConfig['indices']]: string; } = { - sourcemap: 'myIndex', error: 'myIndex', onboarding: 'myIndex', span: 'myIndex', @@ -86,14 +85,10 @@ export async function inspectSearchParams( } ) as APMConfig; const mockInternalESClient = { search: spy } as any; - const mockIndices = { - ...indices, - apmAgentConfigurationIndex: 'myIndex', - apmCustomLinkIndex: 'myIndex', - }; + try { response = await fn({ - mockIndices, + mockIndices: indices, mockApmEventClient, mockConfig, mockInternalESClient, diff --git a/x-pack/plugins/observability/common/typings.ts b/x-pack/plugins/observability/common/typings.ts index 31376f8cb5c46..ce2fdb7460688 100644 --- a/x-pack/plugins/observability/common/typings.ts +++ b/x-pack/plugins/observability/common/typings.ts @@ -19,14 +19,11 @@ export const alertWorkflowStatusRt = t.keyof({ export type AlertWorkflowStatus = t.TypeOf; export interface ApmIndicesConfig { - sourcemap: string; error: string; onboarding: string; span: string; transaction: string; metric: string; - apmAgentConfigurationIndex: string; - apmCustomLinkIndex: string; } export type AlertStatus = diff --git a/x-pack/plugins/observability/server/index.ts b/x-pack/plugins/observability/server/index.ts index e8b6582a9184e..ec930b813fb36 100644 --- a/x-pack/plugins/observability/server/index.ts +++ b/x-pack/plugins/observability/server/index.ts @@ -12,6 +12,7 @@ import { schema, TypeOf } from '@kbn/config-schema'; import { PluginConfigDescriptor, PluginInitializerContext } from '@kbn/core/server'; import { ObservabilityPlugin, ObservabilityPluginSetup } from './plugin'; import { createOrUpdateIndex, Mappings } from './utils/create_or_update_index'; +import { createOrUpdateIndexTemplate } from './utils/create_or_update_index_template'; import { ScopedAnnotationsClient } from './lib/annotations/bootstrap_annotations'; import { unwrapEsResponse, @@ -61,6 +62,11 @@ export const plugin = (initContext: PluginInitializerContext) => new ObservabilityPlugin(initContext); export type { Mappings, ObservabilityPluginSetup, ScopedAnnotationsClient }; -export { createOrUpdateIndex, unwrapEsResponse, WrappedElasticsearchClientError }; +export { + createOrUpdateIndex, + createOrUpdateIndexTemplate, + unwrapEsResponse, + WrappedElasticsearchClientError, +}; export { uiSettings } from './ui_settings'; diff --git a/x-pack/plugins/observability/server/utils/create_or_update_index_template.ts b/x-pack/plugins/observability/server/utils/create_or_update_index_template.ts new file mode 100644 index 0000000000000..4c747c71614ef --- /dev/null +++ b/x-pack/plugins/observability/server/utils/create_or_update_index_template.ts @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import pRetry from 'p-retry'; +import { Logger, ElasticsearchClient } from '@kbn/core/server'; +import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + +export async function createOrUpdateIndexTemplate({ + indexTemplate, + client, + logger, +}: { + indexTemplate: IndicesPutIndexTemplateRequest; + client: ElasticsearchClient; + logger: Logger; +}) { + try { + /* + * In some cases we could be trying to create the index template before ES is ready. + * When this happens, we retry creating the template with exponential backoff. + * We use retry's default formula, meaning that the first retry happens after 2s, + * the 5th after 32s, and the final attempt after around 17m. If the final attempt fails, + * the error is logged to the console. + * See https://github.com/sindresorhus/p-retry and https://github.com/tim-kos/node-retry. + */ + return await pRetry( + async () => { + logger.debug( + `Create index template: "${indexTemplate.name}" for index pattern "${indexTemplate.body?.index_patterns}"` + ); + + const result = await client.indices.putIndexTemplate(indexTemplate); + + if (!result.acknowledged) { + // @ts-expect-error + const resultError = JSON.stringify(result?.body?.error); + throw new Error(resultError); + } + + return result; + }, + { + onFailedAttempt: (e) => { + logger.warn(`Could not create index template: '${indexTemplate.name}'. Retrying...`); + logger.warn(e); + }, + } + ); + } catch (e) { + logger.error(`Could not create index template: '${indexTemplate.name}'. Error: ${e.message}.`); + } +} diff --git a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json index 409f29bfc437b..06cd0479b1169 100644 --- a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json +++ b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @@ -3876,16 +3876,6 @@ } } }, - "sourcemap": { - "properties": { - "1d": { - "type": "long" - }, - "all": { - "type": "long" - } - } - }, "onboarding": { "properties": { "1d": { @@ -4042,13 +4032,6 @@ } } }, - "sourcemap": { - "properties": { - "ms": { - "type": "long" - } - } - }, "onboarding": { "properties": { "ms": { diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index ce5486e70c5b3..f9ac8c45ad4bc 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -7823,7 +7823,6 @@ "xpack.apm.settings.apmIndices.metricsIndicesLabel": "Index des indicateurs", "xpack.apm.settings.apmIndices.noPermissionTooltipLabel": "Votre rôle d'utilisateur ne dispose pas d'autorisations pour changer les index APM", "xpack.apm.settings.apmIndices.onboardingIndicesLabel": "Intégration des index", - "xpack.apm.settings.apmIndices.sourcemapIndicesLabel": "Index des source maps", "xpack.apm.settings.apmIndices.spanIndicesLabel": "Index des intervalles", "xpack.apm.settings.apmIndices.title": "Index", "xpack.apm.settings.apmIndices.transactionIndicesLabel": "Index des transactions", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 475eb687a9b09..39ecebf35103e 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -7812,7 +7812,6 @@ "xpack.apm.settings.apmIndices.metricsIndicesLabel": "メトリックインデックス", "xpack.apm.settings.apmIndices.noPermissionTooltipLabel": "ユーザーロールには、APMインデックスを変更する権限がありません", "xpack.apm.settings.apmIndices.onboardingIndicesLabel": "オンボーディングインデックス", - "xpack.apm.settings.apmIndices.sourcemapIndicesLabel": "ソースマップインデックス", "xpack.apm.settings.apmIndices.spanIndicesLabel": "スパンインデックス", "xpack.apm.settings.apmIndices.title": "インデックス", "xpack.apm.settings.apmIndices.transactionIndicesLabel": "トランザクションインデックス", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index a2cdf2f2c4b31..9b02e8607bdce 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -7826,7 +7826,6 @@ "xpack.apm.settings.apmIndices.metricsIndicesLabel": "指标索引", "xpack.apm.settings.apmIndices.noPermissionTooltipLabel": "您的用户角色无权更改 APM 索引", "xpack.apm.settings.apmIndices.onboardingIndicesLabel": "载入索引", - "xpack.apm.settings.apmIndices.sourcemapIndicesLabel": "源地图索引", "xpack.apm.settings.apmIndices.spanIndicesLabel": "跨度索引", "xpack.apm.settings.apmIndices.title": "索引", "xpack.apm.settings.apmIndices.transactionIndicesLabel": "事务索引", diff --git a/x-pack/test/apm_api_integration/common/config.ts b/x-pack/test/apm_api_integration/common/config.ts index 86b5ae9820199..82dd9d5b8b26f 100644 --- a/x-pack/test/apm_api_integration/common/config.ts +++ b/x-pack/test/apm_api_integration/common/config.ts @@ -28,7 +28,7 @@ import { RegistryProvider } from './registry'; export interface ApmFtrConfig { name: APMFtrConfigName; license: 'basic' | 'trial'; - kibanaConfig?: Record; + kibanaConfig?: Record; } async function getApmApiClient({ diff --git a/x-pack/test/apm_api_integration/configs/index.ts b/x-pack/test/apm_api_integration/configs/index.ts index e4dfc0b8e387c..eed3a3c9f2443 100644 --- a/x-pack/test/apm_api_integration/configs/index.ts +++ b/x-pack/test/apm_api_integration/configs/index.ts @@ -8,17 +8,25 @@ import { mapValues } from 'lodash'; import { createTestConfig, CreateTestConfig } from '../common/config'; +const apmDebugLogger = { + name: 'plugins.apm', + level: 'debug', + appenders: ['console'], +}; + const apmFtrConfigs = { basic: { license: 'basic' as const, kibanaConfig: { 'xpack.apm.forceSyntheticSource': 'true', + 'logging.loggers': [apmDebugLogger], }, }, trial: { license: 'trial' as const, kibanaConfig: { 'xpack.apm.forceSyntheticSource': 'true', + 'logging.loggers': [apmDebugLogger], }, }, rules: { @@ -26,6 +34,7 @@ const apmFtrConfigs = { kibanaConfig: { 'xpack.ruleRegistry.write.enabled': 'true', 'xpack.apm.forceSyntheticSource': 'true', + 'logging.loggers': [apmDebugLogger], }, }, }; diff --git a/x-pack/test/apm_api_integration/tests/sourcemaps/sourcemaps.ts b/x-pack/test/apm_api_integration/tests/sourcemaps/sourcemaps.ts index 0b3dbf2c6ebfc..ed5adcbcab685 100644 --- a/x-pack/test/apm_api_integration/tests/sourcemaps/sourcemaps.ts +++ b/x-pack/test/apm_api_integration/tests/sourcemaps/sourcemaps.ts @@ -4,7 +4,8 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { APIReturnType } from '@kbn/apm-plugin/public/services/rest/create_call_apm_api'; +import type { APIReturnType } from '@kbn/apm-plugin/public/services/rest/create_call_apm_api'; +import type { ApmSourceMap } from '@kbn/apm-plugin/server/routes/source_maps/create_apm_source_map_index_template'; import type { SourceMap } from '@kbn/apm-plugin/server/routes/source_maps/route'; import expect from '@kbn/expect'; import { first, last, times } from 'lodash'; @@ -13,6 +14,65 @@ import { FtrProviderContext } from '../../common/ftr_provider_context'; export default function ApiTest({ getService }: FtrProviderContext) { const registry = getService('registry'); const apmApiClient = getService('apmApiClient'); + const esClient = getService('es'); + + function sleep(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)); + } + + async function waitFor( + cb: () => Promise, + { retries = 50, delay = 100 }: { retries?: number; delay?: number } = {} + ): Promise { + if (retries === 0) { + throw new Error(`Maximum number of retries reached`); + } + + const res = await cb(); + if (!res) { + await sleep(delay); + return waitFor(cb, { retries: retries - 1, delay }); + } + } + + async function waitForSourceMapCount( + count: number, + { retries, delay }: { retries?: number; delay?: number } = {} + ) { + await waitFor( + async () => { + const res = await esClient.search({ + index: '.apm-source-map', + size: 0, + track_total_hits: true, + }); + // @ts-expect-error + return res.hits.total.value === count; + }, + { retries, delay } + ); + return count; + } + + async function deleteAllApmSourceMaps() { + await esClient.deleteByQuery({ + index: '.apm-source-map*', + refresh: true, + query: { match_all: {} }, + }); + } + + async function deleteAllFleetSourceMaps() { + return esClient.deleteByQuery({ + index: '.fleet-artifacts*', + refresh: true, + query: { + bool: { + filter: [{ term: { type: 'sourcemap' } }, { term: { package_name: 'apm' } }], + }, + }, + }); + } async function uploadSourcemap({ bundleFilePath, @@ -40,6 +100,12 @@ export default function ApiTest({ getService }: FtrProviderContext) { return response.body; } + async function runSourceMapMigration() { + await apmApiClient.writeUser({ + endpoint: 'POST /internal/apm/sourcemaps/migrate_fleet_artifacts', + }); + } + async function deleteSourcemap(id: string) { await apmApiClient.writeUser({ endpoint: 'DELETE /api/apm/sourcemaps/{id}', @@ -58,6 +124,10 @@ export default function ApiTest({ getService }: FtrProviderContext) { } registry.when('source maps', { config: 'basic', archives: [] }, () => { + before(async () => { + await Promise.all([deleteAllFleetSourceMaps(), deleteAllApmSourceMaps()]); + }); + let resp: APIReturnType<'POST /api/apm/sourcemaps'>; describe('upload source map', () => { after(async () => { @@ -67,9 +137,9 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); }); - it('can upload a source map', async () => { + before(async () => { resp = await uploadSourcemap({ - serviceName: 'my_service', + serviceName: 'uploading-test', serviceVersion: '1.0.0', bundleFilePath: 'bar', sourcemap: { @@ -78,17 +148,50 @@ export default function ApiTest({ getService }: FtrProviderContext) { mappings: '', }, }); - expect(resp).to.not.empty(); + + await waitForSourceMapCount(1); + }); + + it('is uploaded as a fleet artifact', async () => { + const res = await esClient.search({ + index: '.fleet-artifacts', + size: 1, + query: { + bool: { + filter: [{ term: { type: 'sourcemap' } }, { term: { package_name: 'apm' } }], + }, + }, + }); + + // @ts-expect-error + expect(res.hits.hits[0]._source.identifier).to.be('uploading-test-1.0.0'); + }); + + it('is added to .apm-source-map index', async () => { + const res = await esClient.search({ + index: '.apm-source-map', + }); + + const doc = res.hits.hits[0]._source as ApmSourceMap; + expect(doc.content).to.be( + 'eJyrVipLLSrOzM9TsjI0MtZRKs4vLUpOLVayilZSitVRyk0sKMjMSwfylZRqAURLDgo=' + ); + expect(doc.content_sha256).to.be( + '02dd950aa88a66183d312a7a5f44d72fc9e3914cdbbe5e3a04f1509a8a3d7d83' + ); + expect(doc.file.path).to.be('bar'); + expect(doc.service.name).to.be('uploading-test'); + expect(doc.service.version).to.be('1.0.0'); }); }); describe('list source maps', async () => { - const uploadedSourcemapIds: string[] = []; before(async () => { - const sourcemapCount = times(15); + const totalCount = 6; + const sourcemapCount = times(totalCount); for (const i of sourcemapCount) { - const sourcemap = await uploadSourcemap({ - serviceName: 'my_service', + await uploadSourcemap({ + serviceName: 'list-test', serviceVersion: `1.0.${i}`, bundleFilePath: 'bar', sourcemap: { @@ -97,44 +200,44 @@ export default function ApiTest({ getService }: FtrProviderContext) { mappings: '', }, }); - uploadedSourcemapIds.push(sourcemap.id); } + await waitForSourceMapCount(totalCount); }); after(async () => { - await Promise.all(uploadedSourcemapIds.map((id) => deleteSourcemap(id))); + await Promise.all([deleteAllFleetSourceMaps(), deleteAllApmSourceMaps()]); }); describe('pagination', () => { it('can retrieve the first page', async () => { - const firstPageItems = await listSourcemaps({ page: 1, perPage: 5 }); - expect(first(firstPageItems.artifacts)?.identifier).to.eql('my_service-1.0.14'); - expect(last(firstPageItems.artifacts)?.identifier).to.eql('my_service-1.0.10'); - expect(firstPageItems.artifacts.length).to.be(5); - expect(firstPageItems.total).to.be(15); + const res = await listSourcemaps({ page: 1, perPage: 2 }); + expect(first(res.artifacts)?.identifier).to.eql('list-test-1.0.5'); + expect(last(res.artifacts)?.identifier).to.eql('list-test-1.0.4'); + expect(res.artifacts.length).to.be(2); + expect(res.total).to.be(6); }); it('can retrieve the second page', async () => { - const secondPageItems = await listSourcemaps({ page: 2, perPage: 5 }); - expect(first(secondPageItems.artifacts)?.identifier).to.eql('my_service-1.0.9'); - expect(last(secondPageItems.artifacts)?.identifier).to.eql('my_service-1.0.5'); - expect(secondPageItems.artifacts.length).to.be(5); - expect(secondPageItems.total).to.be(15); + const res = await listSourcemaps({ page: 2, perPage: 2 }); + expect(first(res.artifacts)?.identifier).to.eql('list-test-1.0.3'); + expect(last(res.artifacts)?.identifier).to.eql('list-test-1.0.2'); + expect(res.artifacts.length).to.be(2); + expect(res.total).to.be(6); }); it('can retrieve the third page', async () => { - const thirdPageItems = await listSourcemaps({ page: 3, perPage: 5 }); - expect(first(thirdPageItems.artifacts)?.identifier).to.eql('my_service-1.0.4'); - expect(last(thirdPageItems.artifacts)?.identifier).to.eql('my_service-1.0.0'); - expect(thirdPageItems.artifacts.length).to.be(5); - expect(thirdPageItems.total).to.be(15); + const res = await listSourcemaps({ page: 3, perPage: 2 }); + expect(first(res.artifacts)?.identifier).to.eql('list-test-1.0.1'); + expect(last(res.artifacts)?.identifier).to.eql('list-test-1.0.0'); + expect(res.artifacts.length).to.be(2); + expect(res.total).to.be(6); }); }); - it('can list source maps', async () => { + it('can list source maps without specifying pagination options', async () => { const sourcemaps = await listSourcemaps(); - expect(sourcemaps.artifacts.length).to.be(15); - expect(sourcemaps.total).to.be(15); + expect(sourcemaps.artifacts.length).to.be(6); + expect(sourcemaps.total).to.be(6); }); it('returns newest source maps first', async () => { @@ -144,10 +247,14 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); }); + function getRandomString() { + return Math.random().toString(36).substring(7); + } + describe('delete source maps', () => { - it('can delete a source map', async () => { + before(async () => { const sourcemap = await uploadSourcemap({ - serviceName: 'my_service', + serviceName: `delete-test_${getRandomString()}`, serviceVersion: '1.0.0', bundleFilePath: 'bar', sourcemap: { @@ -157,11 +264,63 @@ export default function ApiTest({ getService }: FtrProviderContext) { }, }); + // wait for the sourcemap to be indexed in .apm-source-map index + await waitForSourceMapCount(1); + + // delete sourcemap await deleteSourcemap(sourcemap.id); + + // wait for the sourcemap to be deleted from .apm-source-map index + await waitForSourceMapCount(0); + }); + + it('can delete a fleet source map artifact', async () => { const { artifacts, total } = await listSourcemaps(); expect(artifacts).to.be.empty(); expect(total).to.be(0); }); + + it('can delete an apm source map', async () => { + // check that the sourcemap is deleted from .apm-source-map index + const res = await esClient.search({ index: '.apm-source-map' }); + // @ts-expect-error + expect(res.hits.total.value).to.be(0); + }); + }); + + describe('source map migration from fleet artifacts to `.apm-source-map`', () => { + const totalCount = 100; + + before(async () => { + await Promise.all( + times(totalCount).map(async (i) => { + await uploadSourcemap({ + serviceName: `migration-test`, + serviceVersion: `1.0.${i}`, + bundleFilePath: 'bar', + sourcemap: { + version: 123, + sources: [''], + mappings: '', + }, + }); + }) + ); + + // wait for sourcemaps to be indexed in .apm-source-map index + await waitForSourceMapCount(totalCount); + }); + + it('it will migrate fleet artifacts to `.apm-source-map`', async () => { + await deleteAllApmSourceMaps(); + + // wait for source maps to be deleted before running migration + await waitForSourceMapCount(0); + + await runSourceMapMigration(); + + expect(await waitForSourceMapCount(totalCount)).to.be(totalCount); + }); }); }); } diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts b/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts index 91295ad7ceaf6..4351081703bd7 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts @@ -106,6 +106,7 @@ export default function ({ getService }: FtrProviderContext) { 'alerting_health_check', 'alerting_telemetry', 'alerts_invalidate_api_keys', + 'apm-source-map-migration-task', 'apm-telemetry-task', 'cases-telemetry-task', 'cleanup_failed_action_executions',