[APM] Migrate fleet sourcemaps to APM managed index

x-pack/plugins/apm/ftr_e2e/cypress/e2e/power_user/no_data_screen.cy.ts

@@ -10,7 +10,6 @@ describe('No data screen', () => {
     before(() => {
       // Change indices
       setApmIndices({
-        sourcemap: 'foo-*',
         error: 'foo-*',
         onboarding: 'foo-*',
         span: 'foo-*',
@@ -37,7 +36,6 @@ describe('No data screen', () => {
     after(() => {
       // reset to default indices
       setApmIndices({
-        sourcemap: '',
         error: '',
         onboarding: '',
         span: '',

x-pack/plugins/apm/public/components/app/settings/apm_indices/index.tsx

@@ -33,13 +33,6 @@ import {
 } from '../../../../services/rest/create_call_apm_api';
 
 const APM_INDEX_LABELS = [
-  {
-    configurationName: 'sourcemap',
-    label: i18n.translate(
-      'xpack.apm.settings.apmIndices.sourcemapIndicesLabel',
-      { defaultMessage: 'Sourcemap Indices' }
-    ),
-  },
   {
     configurationName: 'error',
     label: i18n.translate('xpack.apm.settings.apmIndices.errorIndicesLabel', {

x-pack/plugins/apm/scripts/shared/read_kibana_config.ts

@@ -43,7 +43,6 @@ export const readKibanaConfig = () => {
     'xpack.apm.indices.error': 'logs-apm*,apm-*',
     'xpack.apm.indices.span': 'traces-apm*,apm-*',
     'xpack.apm.indices.onboarding': 'apm-*',
-    'xpack.apm.indices.sourcemap': 'apm-*',
     'elasticsearch.hosts': 'http://localhost:9200',
     ...loadedKibanaConfig,
     ...cliEsCredentials,

x-pack/plugins/apm/server/index.ts

@@ -51,7 +51,6 @@ const configSchema = schema.object({
     span: schema.string({ defaultValue: 'traces-apm*,apm-*' }),
     error: schema.string({ defaultValue: 'logs-apm*,apm-*' }),
     metric: schema.string({ defaultValue: 'metrics-apm*,apm-*' }),
-    sourcemap: schema.string({ defaultValue: 'apm-*' }),
     onboarding: schema.string({ defaultValue: 'apm-*' }),
   }),
   forceSyntheticSource: schema.boolean({ defaultValue: false }),
@@ -61,10 +60,12 @@ const configSchema = schema.object({
 export const config: PluginConfigDescriptor<APMConfig> = {
   deprecations: ({
     rename,
+    unused,
     renameFromRoot,
     deprecateFromRoot,
     unusedFromRoot,
   }) => [
+    unused('indices.sourcemap', { level: 'warning' }),
     rename('autocreateApmIndexPattern', 'autoCreateApmDataView', {
       level: 'warning',
     }),

x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts

@@ -298,10 +298,6 @@ describe('data telemetry collection tasks', () => {
             '1d': 1,
             all: 1,
           },
-          sourcemap: {
-            '1d': 1,
-            all: 1,
-          },
           span: {
             '1d': 1,
             all: 1,
@@ -321,9 +317,6 @@ describe('data telemetry collection tasks', () => {
           onboarding: {
             ms: 0,
           },
-          sourcemap: {
-            ms: 0,
-          },
           span: {
             ms: 0,
           },

x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts

@@ -54,7 +54,10 @@ import { AgentName } from '../../../../typings/es_schemas/ui/fields/agent';
 import { Span } from '../../../../typings/es_schemas/ui/span';
 import { Transaction } from '../../../../typings/es_schemas/ui/transaction';
 import { APMTelemetry, APMPerService, APMDataTelemetry } from '../types';
-import { ApmIndicesConfig } from '../../../routes/settings/apm_indices/get_apm_indices';
+import {
+  ApmIndicesConfig,
+  APM_AGENT_CONFIGURATION_INDEX,
+} from '../../../routes/settings/apm_indices/get_apm_indices';
 import { TelemetryClient } from '../telemetry_client';
 
 type ISavedObjectsClient = Pick<SavedObjectsClient, 'find'>;
@@ -464,7 +467,6 @@ export const tasks: TelemetryTask[] = [
         span: indices.span,
         transaction: indices.transaction,
         onboarding: indices.onboarding,
-        sourcemap: indices.sourcemap,
       };
 
       type ProcessorEvent = keyof typeof indicesByProcessorEvent;
@@ -558,7 +560,7 @@ export const tasks: TelemetryTask[] = [
     name: 'agent_configuration',
     executor: async ({ indices, telemetryClient }) => {
       const agentConfigurationCount = await telemetryClient.search({
-        index: indices.apmAgentConfigurationIndex,
+        index: APM_AGENT_CONFIGURATION_INDEX,
         body: {
           size: 0,
           timeout,
@@ -1033,11 +1035,10 @@ export const tasks: TelemetryTask[] = [
     executor: async ({ indices, telemetryClient }) => {
       const response = await telemetryClient.indicesStats({
         index: [
-          indices.apmAgentConfigurationIndex,
+          APM_AGENT_CONFIGURATION_INDEX,
           indices.error,
           indices.metric,
           indices.onboarding,
-          indices.sourcemap,
           indices.span,
           indices.transaction,
         ],

x-pack/plugins/apm/server/lib/apm_telemetry/schema.ts

@@ -193,7 +193,6 @@ export const apmSchema: MakeSchemaFrom<APMUsage> = {
     span: timeframeMapSchema,
     error: timeframeMapSchema,
     metric: timeframeMapSchema,
-    sourcemap: timeframeMapSchema,
     onboarding: timeframeMapSchema,
     agent_configuration: timeframeMapAllSchema,
     max_transaction_groups_per_service: timeframeMapSchema,
@@ -221,7 +220,6 @@ export const apmSchema: MakeSchemaFrom<APMUsage> = {
     transaction: { ms: long },
     error: { ms: long },
     metric: { ms: long },
-    sourcemap: { ms: long },
     onboarding: { ms: long },
   },
   integrations: { ml: { all_jobs_count: long } },

x-pack/plugins/apm/server/lib/apm_telemetry/types.ts

@@ -101,7 +101,6 @@ export interface APMUsage {
     span: TimeframeMap;
     error: TimeframeMap;
     metric: TimeframeMap;
-    sourcemap: TimeframeMap;
     onboarding: TimeframeMap;
     agent_configuration: TimeframeMapAll;
     max_transaction_groups_per_service: TimeframeMap;
@@ -125,7 +124,7 @@ export interface APMUsage {
     };
   };
   retainment: Record<
-    'span' | 'transaction' | 'error' | 'metric' | 'sourcemap' | 'onboarding',
+    'span' | 'transaction' | 'error' | 'metric' | 'onboarding',
     { ms: number }
   >;
   integrations: {

x-pack/plugins/apm/server/lib/helpers/create_es_client/create_apm_event_client/unpack_processor_events.test.ts

@@ -27,7 +27,6 @@ describe('unpackProcessorEvents', () => {
       error: 'my-apm-*-error-*',
       span: 'my-apm-*-span-*',
       onboarding: 'my-apm-*-onboarding-*',
-      sourcemap: 'my-apm-*-sourcemap-*',
     } as ApmIndicesConfig;
 
     res = unpackProcessorEvents(request, indices);

x-pack/plugins/apm/server/plugin.ts

@@ -56,6 +56,8 @@ import {
 } from '../common/es_fields/apm';
 import { tutorialProvider } from './tutorial';
 import { migrateLegacyAPMIndicesToSpaceAware } from './saved_objects/migrations/migrate_legacy_apm_indices_to_space_aware';
+import { scheduleSourceMapMigration } from './routes/source_maps/schedule_source_map_migration';
+import { createApmSourceMapIndexTemplate } from './routes/source_maps/create_apm_source_map_index_template';
 
 export class APMPlugin
   implements
@@ -110,6 +112,9 @@ export class APMPlugin
     const getCoreStart = () =>
       core.getStartServices().then(([coreStart]) => coreStart);
 
+    const getPluginStart = () =>
+      core.getStartServices().then(([coreStart, pluginStart]) => pluginStart);
+
     const { ruleDataService } = plugins.ruleRegistry;
     const ruleDataClient = ruleDataService.initializeIndex({
       feature: APM_SERVER_FEATURE_ID,
@@ -220,6 +225,21 @@ export class APMPlugin
       kibanaVersion: this.initContext.env.packageInfo.version,
     });
 
+    const fleetStartPromise = resourcePlugins.fleet?.start();
+    const taskManager = plugins.taskManager;
+
+    // create source map index and run migrations
+    scheduleSourceMapMigration({
+      coreStartPromise: getCoreStart(),
+      pluginStartPromise: getPluginStart(),
+      fleetStartPromise,
+      taskManager,
+      logger: this.logger,
+    }).catch((e) => {
+      this.logger?.error('Failed to schedule APM source map migration');
+      this.logger?.error(e);
+    });
+
     return {
       config$,
       getApmIndices: boundGetApmIndices,
@@ -254,28 +274,39 @@ export class APMPlugin
     };
   }
 
-  public start(core: CoreStart) {
+  public start(core: CoreStart, plugins: APMPluginStartDependencies) {
     if (this.currentConfig == null || this.logger == null) {
       throw new Error('APMPlugin needs to be setup before calling start()');
     }
 
-    // create agent configuration index without blocking start lifecycle
-    createApmAgentConfigurationIndex({
-      client: core.elasticsearch.client.asInternalUser,
-      config: this.currentConfig,
-      logger: this.logger,
+    const logger = this.logger;
+    const client = core.elasticsearch.client.asInternalUser;
+
+    // create .apm-agent-configuration index without blocking start lifecycle
+    createApmAgentConfigurationIndex({ client, logger }).catch((e) => {
+      logger.error('Failed to create .apm-agent-configuration index');
+      logger.error(e);
     });
-    // create custom action index without blocking start lifecycle
-    createApmCustomLinkIndex({
-      client: core.elasticsearch.client.asInternalUser,
-      config: this.currentConfig,
-      logger: this.logger,
+
+    // create .apm-custom-link index without blocking start lifecycle
+    createApmCustomLinkIndex({ client, logger }).catch((e) => {
+      logger.error('Failed to create .apm-custom-link index');
+      logger.error(e);
     });
 
-    migrateLegacyAPMIndicesToSpaceAware({
-      coreStart: core,
-      logger: this.logger,
+    // create .apm-source-map index without blocking start lifecycle
+    createApmSourceMapIndexTemplate({ client, logger }).catch((e) => {
+      logger.error('Failed to create apm-source-map index template');
+      logger.error(e);
     });
+
+    // TODO: remove in 9.0
+    migrateLegacyAPMIndicesToSpaceAware({ coreStart: core, logger }).catch(
+      (e) => {
+        logger.error('Failed to run migration making APM indices space aware');
+        logger.error(e);
+      }
+    );
   }
 
   public stop() {}