From 8aa9241a8a3a990d374cee4dc3305ebe1eb9eeea Mon Sep 17 00:00:00 2001 From: Josh Dover <1813008+joshdover@users.noreply.github.com> Date: Fri, 6 May 2022 22:39:36 +0200 Subject: [PATCH] Optimize package installation performance, phase 2 (#131627) --- .../elasticsearch/ingest_pipeline/index.ts | 2 +- .../elasticsearch/ingest_pipeline/install.ts | 87 ++++---- .../elasticsearch/template/install.test.ts | 140 ++---------- .../epm/elasticsearch/template/install.ts | 204 ++++++++---------- .../fleet/server/services/epm/fields/field.ts | 6 +- .../services/epm/kibana/assets/install.ts | 1 + .../services/epm/packages/_install_package.ts | 52 +++-- .../server/services/epm/packages/assets.ts | 4 +- 8 files changed, 196 insertions(+), 300 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/index.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/index.ts index 574534290214a..5f093a19157f9 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/index.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/index.ts @@ -5,6 +5,6 @@ * 2.0. */ -export { installPipelines, isTopLevelPipeline } from './install'; +export { prepareToInstallPipelines, isTopLevelPipeline } from './install'; export { deletePreviousPipelines, deletePipeline } from './remove'; diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts index 49dae4d86b639..da035a44c9921 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts @@ -6,13 +6,12 @@ */ import type { TransportRequestOptions } from '@elastic/elasticsearch'; -import type { ElasticsearchClient, Logger, SavedObjectsClientContract } from '@kbn/core/server'; +import type { ElasticsearchClient, Logger } from '@kbn/core/server'; import { ElasticsearchAssetType } from '../../../../types'; import type { EsAssetReference, RegistryDataStream, InstallablePackage } from '../../../../types'; import { getAsset, getPathParts } from '../../archive'; import type { ArchiveEntry } from '../../archive'; -import { updateEsAssetReferences } from '../../packages/install'; import { FLEET_FINAL_PIPELINE_CONTENT, FLEET_FINAL_PIPELINE_ID, @@ -36,23 +35,23 @@ export const isTopLevelPipeline = (path: string) => { ); }; -export const installPipelines = async ( +export const prepareToInstallPipelines = ( installablePackage: InstallablePackage, - paths: string[], - esClient: ElasticsearchClient, - savedObjectsClient: SavedObjectsClientContract, - logger: Logger, - esReferences: EsAssetReference[] -) => { + paths: string[] +): { + assetsToAdd: EsAssetReference[]; + install: (esClient: ElasticsearchClient, logger: Logger) => Promise; +} => { // unlike other ES assets, pipeline names are versioned so after a template is updated // it can be created pointing to the new template, without removing the old one and effecting data // so do not remove the currently installed pipelines here const dataStreams = installablePackage.data_streams; - const { name: pkgName, version: pkgVersion } = installablePackage; + const { version: pkgVersion } = installablePackage; const pipelinePaths = paths.filter((path) => isPipeline(path)); const topLevelPipelinePaths = paths.filter((path) => isTopLevelPipeline(path)); - if (!dataStreams?.length && topLevelPipelinePaths.length === 0) return []; + if (!dataStreams?.length && topLevelPipelinePaths.length === 0) + return { assetsToAdd: [], install: () => Promise.resolve() }; // get and save pipeline refs before installing pipelines let pipelineRefs = dataStreams @@ -85,41 +84,41 @@ export const installPipelines = async ( pipelineRefs = [...pipelineRefs, ...topLevelPipelineRefs]; - esReferences = await updateEsAssetReferences(savedObjectsClient, pkgName, esReferences, { + return { assetsToAdd: pipelineRefs, - }); - - const pipelines = dataStreams - ? dataStreams.reduce>>((acc, dataStream) => { - if (dataStream.ingest_pipeline) { - acc.push( - installAllPipelines({ - dataStream, - esClient, - logger, - paths: pipelinePaths, - installablePackage, - }) - ); - } - return acc; - }, []) - : []; - - if (topLevelPipelinePaths) { - pipelines.push( - installAllPipelines({ - dataStream: undefined, - esClient, - logger, - paths: topLevelPipelinePaths, - installablePackage, - }) - ); - } + install: async (esClient, logger) => { + const pipelines = dataStreams + ? dataStreams.reduce>>((acc, dataStream) => { + if (dataStream.ingest_pipeline) { + acc.push( + installAllPipelines({ + dataStream, + esClient, + logger, + paths: pipelinePaths, + installablePackage, + }) + ); + } + return acc; + }, []) + : []; + + if (topLevelPipelinePaths) { + pipelines.push( + installAllPipelines({ + dataStream: undefined, + esClient, + logger, + paths: topLevelPipelinePaths, + installablePackage, + }) + ); + } - await Promise.all(pipelines); - return esReferences; + await Promise.all(pipelines); + }, + }; }; export function rewriteIngestPipeline( diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts index 998d0f9fb1ae5..3478da69bf721 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts @@ -4,30 +4,19 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { elasticsearchServiceMock } from '@kbn/core/server/mocks'; -import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { loggerMock } from '@kbn/logging-mocks'; - import { createAppContextStartContractMock } from '../../../../mocks'; import { appContextService } from '../../..'; import type { RegistryDataStream } from '../../../../types'; -import type { Field } from '../../fields/field'; -import { installTemplate } from './install'; +import { prepareTemplate } from './install'; -describe('EPM install', () => { +describe('EPM index template install', () => { beforeEach(async () => { appContextService.start(createAppContextStartContractMock()); }); - it('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix not set', async () => { - const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; - esClient.indices.getIndexTemplate.mockImplementation(() => - elasticsearchServiceMock.createSuccessTransportRequestPromise({ index_templates: [] }) - ); - - const fields: Field[] = []; + it('tests prepareTemplate to use correct priority and index_patterns for data stream with dataset_is_prefix not set', async () => { const dataStreamDatasetIsPrefixUnset = { type: 'metrics', dataset: 'package.dataset', @@ -43,29 +32,14 @@ describe('EPM install', () => { }; const templateIndexPatternDatasetIsPrefixUnset = 'metrics-package.dataset-*'; const templatePriorityDatasetIsPrefixUnset = 200; - await installTemplate({ - esClient, - logger: loggerMock.create(), - fields, - dataStream: dataStreamDatasetIsPrefixUnset, - packageVersion: pkg.version, - packageName: pkg.name, - }); - - const sentTemplate = ( - esClient.indices.putIndexTemplate.mock.calls[0][0] as estypes.IndicesPutIndexTemplateRequest - ).body; - - expect(sentTemplate).toBeDefined(); - expect(sentTemplate?.priority).toBe(templatePriorityDatasetIsPrefixUnset); - expect(sentTemplate?.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixUnset]); + const { + indexTemplate: { indexTemplate }, + } = prepareTemplate({ pkg, dataStream: dataStreamDatasetIsPrefixUnset }); + expect(indexTemplate.priority).toBe(templatePriorityDatasetIsPrefixUnset); + expect(indexTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixUnset]); }); - it('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix set to false', async () => { - const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; - esClient.indices.getIndexTemplate.mockResponse({ index_templates: [] }); - - const fields: Field[] = []; + it('tests prepareTemplate to use correct priority and index_patterns for data stream with dataset_is_prefix set to false', async () => { const dataStreamDatasetIsPrefixFalse = { type: 'metrics', dataset: 'package.dataset', @@ -82,29 +56,15 @@ describe('EPM install', () => { }; const templateIndexPatternDatasetIsPrefixFalse = 'metrics-package.dataset-*'; const templatePriorityDatasetIsPrefixFalse = 200; - await installTemplate({ - esClient, - logger: loggerMock.create(), - fields, - dataStream: dataStreamDatasetIsPrefixFalse, - packageVersion: pkg.version, - packageName: pkg.name, - }); - - const sentTemplate = ( - esClient.indices.putIndexTemplate.mock.calls[0][0] as estypes.IndicesPutIndexTemplateRequest - ).body; + const { + indexTemplate: { indexTemplate }, + } = prepareTemplate({ pkg, dataStream: dataStreamDatasetIsPrefixFalse }); - expect(sentTemplate).toBeDefined(); - expect(sentTemplate?.priority).toBe(templatePriorityDatasetIsPrefixFalse); - expect(sentTemplate?.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixFalse]); + expect(indexTemplate.priority).toBe(templatePriorityDatasetIsPrefixFalse); + expect(indexTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixFalse]); }); - it('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix set to true', async () => { - const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; - esClient.indices.getIndexTemplate.mockResponse({ index_templates: [] }); - - const fields: Field[] = []; + it('tests prepareTemplate to use correct priority and index_patterns for data stream with dataset_is_prefix set to true', async () => { const dataStreamDatasetIsPrefixTrue = { type: 'metrics', dataset: 'package.dataset', @@ -121,71 +81,11 @@ describe('EPM install', () => { }; const templateIndexPatternDatasetIsPrefixTrue = 'metrics-package.dataset.*-*'; const templatePriorityDatasetIsPrefixTrue = 150; - await installTemplate({ - esClient, - logger: loggerMock.create(), - fields, - dataStream: dataStreamDatasetIsPrefixTrue, - packageVersion: pkg.version, - packageName: pkg.name, - }); - - const sentTemplate = ( - esClient.indices.putIndexTemplate.mock.calls[0][0] as estypes.IndicesPutIndexTemplateRequest - ).body; - - expect(sentTemplate).toBeDefined(); - expect(sentTemplate?.priority).toBe(templatePriorityDatasetIsPrefixTrue); - expect(sentTemplate?.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixTrue]); - }); - - it('tests installPackage remove the aliases property if the property existed', async () => { - const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; - - esClient.indices.getIndexTemplate.mockResponse({ - index_templates: [ - { - name: 'metrics-package.dataset', - // @ts-expect-error not full interface - index_template: { - index_patterns: ['metrics-package.dataset-*'], - template: { aliases: {} }, - }, - }, - ], - }); - - const fields: Field[] = []; - const dataStreamDatasetIsPrefixUnset = { - type: 'metrics', - dataset: 'package.dataset', - title: 'test data stream', - release: 'experimental', - package: 'package', - path: 'path', - ingest_pipeline: 'default', - } as RegistryDataStream; - const pkg = { - name: 'package', - version: '0.0.1', - }; - const templateIndexPatternDatasetIsPrefixUnset = 'metrics-package.dataset-*'; - const templatePriorityDatasetIsPrefixUnset = 200; - await installTemplate({ - esClient, - logger: loggerMock.create(), - fields, - dataStream: dataStreamDatasetIsPrefixUnset, - packageVersion: pkg.version, - packageName: pkg.name, - }); + const { + indexTemplate: { indexTemplate }, + } = prepareTemplate({ pkg, dataStream: dataStreamDatasetIsPrefixTrue }); - const sentTemplate = ( - esClient.indices.putIndexTemplate.mock.calls[0][0] as estypes.IndicesPutIndexTemplateRequest - ).body; - expect(sentTemplate).toBeDefined(); - expect(sentTemplate?.template?.aliases).not.toBeDefined(); - expect(sentTemplate?.priority).toBe(templatePriorityDatasetIsPrefixUnset); - expect(sentTemplate?.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixUnset]); + expect(indexTemplate.priority).toBe(templatePriorityDatasetIsPrefixTrue); + expect(indexTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixTrue]); }); }); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index 2d2e5b2ffea2a..df6d9d84a08c5 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -7,7 +7,7 @@ import { merge } from 'lodash'; import Boom from '@hapi/boom'; -import type { ElasticsearchClient, Logger, SavedObjectsClientContract } from '@kbn/core/server'; +import type { ElasticsearchClient, Logger } from '@kbn/core/server'; import { ElasticsearchAssetType } from '../../../../types'; import type { @@ -20,13 +20,12 @@ import type { TemplateMapEntry, TemplateMap, EsAssetReference, + PackageInfo, } from '../../../../types'; import { loadFieldsFromYaml, processFields } from '../../fields/field'; -import type { Field } from '../../fields/field'; import { getPipelineNameForInstallation } from '../ingest_pipeline/install'; import { getAsset, getPathParts } from '../../archive'; -import { updateEsAssetReferences } from '../../packages/install'; import { FLEET_COMPONENT_TEMPLATES, PACKAGE_TEMPLATE_SUFFIX, @@ -47,65 +46,55 @@ import { buildDefaultSettings } from './default_settings'; const FLEET_COMPONENT_TEMPLATE_NAMES = FLEET_COMPONENT_TEMPLATES.map((tmpl) => tmpl.name); -export const installTemplates = async ( +export const prepareToInstallTemplates = ( installablePackage: InstallablePackage, - esClient: ElasticsearchClient, - logger: Logger, paths: string[], - savedObjectsClient: SavedObjectsClientContract, esReferences: EsAssetReference[] -): Promise<{ - installedTemplates: IndexTemplateEntry[]; - installedEsReferences: EsAssetReference[]; -}> => { - // install any pre-built index template assets, - // atm, this is only the base package's global index templates - // Install component templates first, as they are used by the index templates - await installPreBuiltComponentTemplates(paths, esClient, logger); - await installPreBuiltTemplates(paths, esClient, logger); - +): { + assetsToAdd: EsAssetReference[]; + assetsToRemove: EsAssetReference[]; + install: (esClient: ElasticsearchClient, logger: Logger) => Promise; +} => { // remove package installation's references to index templates - esReferences = await updateEsAssetReferences( - savedObjectsClient, - installablePackage.name, - esReferences, - { - assetsToRemove: esReferences.filter( - ({ type }) => - type === ElasticsearchAssetType.indexTemplate || - type === ElasticsearchAssetType.componentTemplate - ), - } + const assetsToRemove = esReferences.filter( + ({ type }) => + type === ElasticsearchAssetType.indexTemplate || + type === ElasticsearchAssetType.componentTemplate ); // build templates per data stream from yml files const dataStreams = installablePackage.data_streams; - if (!dataStreams) return { installedTemplates: [], installedEsReferences: esReferences }; - - const installedTemplatesNested = await Promise.all( - dataStreams.map((dataStream) => - installTemplateForDataStream({ - pkg: installablePackage, - esClient, - logger, - dataStream, - }) - ) - ); - const installedTemplates = installedTemplatesNested.flat(); + if (!dataStreams) return { assetsToAdd: [], assetsToRemove, install: () => Promise.resolve([]) }; - // get template refs to save - const installedIndexTemplateRefs = getAllTemplateRefs(installedTemplates); - - // add package installation's references to index templates - esReferences = await updateEsAssetReferences( - savedObjectsClient, - installablePackage.name, - esReferences, - { assetsToAdd: installedIndexTemplateRefs } + const templates = dataStreams.map((dataStream) => + prepareTemplate({ pkg: installablePackage, dataStream }) ); + const assetsToAdd = getAllTemplateRefs(templates.map((template) => template.indexTemplate)); + + return { + assetsToAdd, + assetsToRemove, + install: async (esClient, logger) => { + // install any pre-built index template assets, + // atm, this is only the base package's global index templates + // Install component templates first, as they are used by the index templates + await installPreBuiltComponentTemplates(paths, esClient, logger); + await installPreBuiltTemplates(paths, esClient, logger); + + await Promise.all( + templates.map((template) => + installComponentAndIndexTemplateForDataStream({ + esClient, + logger, + componentTemplates: template.componentTemplates, + indexTemplate: template.indexTemplate, + }) + ) + ); - return { installedTemplates, installedEsReferences: esReferences }; + return templates.map((template) => template.indexTemplate); + }, + }; }; const installPreBuiltTemplates = async ( @@ -187,31 +176,24 @@ const isComponentTemplate = (path: string) => { }; /** - * installTemplateForDataStream installs one template for each data stream + * installComponentAndIndexTemplateForDataStream installs one template for each data stream * * The template is currently loaded with the pkgkey-package-data_stream */ -export async function installTemplateForDataStream({ - pkg, +export async function installComponentAndIndexTemplateForDataStream({ esClient, logger, - dataStream, + componentTemplates, + indexTemplate, }: { - pkg: InstallablePackage; esClient: ElasticsearchClient; logger: Logger; - dataStream: RegistryDataStream; -}): Promise { - const fields = await loadFieldsFromYaml(pkg, dataStream.path); - return installTemplate({ - esClient, - logger, - fields, - dataStream, - packageVersion: pkg.version, - packageName: pkg.name, - }); + componentTemplates: TemplateMap; + indexTemplate: IndexTemplateEntry; +}) { + await installDataStreamComponentTemplates({ esClient, logger, componentTemplates }); + await installTemplate({ esClient, logger, template: indexTemplate }); } function putComponentTemplate( @@ -291,35 +273,18 @@ function buildComponentTemplates(params: { return templatesMap; } -async function installDataStreamComponentTemplates(params: { - mappings: IndexTemplateMappings; - templateName: string; - registryElasticsearch: RegistryElasticsearch | undefined; +async function installDataStreamComponentTemplates({ + esClient, + logger, + componentTemplates, +}: { esClient: ElasticsearchClient; logger: Logger; - packageName: string; - defaultSettings: IndexTemplate['template']['settings']; + componentTemplates: TemplateMap; }) { - const { - templateName, - registryElasticsearch, - esClient, - packageName, - defaultSettings, - logger, - mappings, - } = params; - const componentTemplates = buildComponentTemplates({ - mappings, - templateName, - registryElasticsearch, - packageName, - defaultSettings, - }); - const templateEntries = Object.entries(componentTemplates); // TODO: Check return values for errors await Promise.all( - templateEntries.map(async ([name, body]) => { + Object.entries(componentTemplates).map(async ([name, body]) => { if (isUserSettingsTemplate(name)) { try { // Attempt to create custom component templates, ignore if they already exist @@ -342,8 +307,6 @@ async function installDataStreamComponentTemplates(params: { } }) ); - - return { componentTemplateNames: Object.keys(componentTemplates) }; } export async function ensureDefaultComponentTemplates( @@ -387,21 +350,15 @@ export async function ensureComponentTemplate( return { isCreated: !existingTemplate }; } -export async function installTemplate({ - esClient, - logger, - fields, +export function prepareTemplate({ + pkg, dataStream, - packageVersion, - packageName, }: { - esClient: ElasticsearchClient; - logger: Logger; - fields: Field[]; + pkg: Pick; dataStream: RegistryDataStream; - packageVersion: string; - packageName: string; -}): Promise { +}): { componentTemplates: TemplateMap; indexTemplate: IndexTemplateEntry } { + const { name: packageName, version: packageVersion } = pkg; + const fields = loadFieldsFromYaml(pkg, dataStream.path); const validFields = processFields(fields); const mappings = generateMappings(validFields); const templateName = generateTemplateName(dataStream); @@ -425,40 +382,51 @@ export async function installTemplate({ ilmPolicy: dataStream.ilm_policy, }); - const { componentTemplateNames } = await installDataStreamComponentTemplates({ + const componentTemplates = buildComponentTemplates({ + defaultSettings, mappings, + packageName, templateName, registryElasticsearch: dataStream.elasticsearch, - esClient, - logger, - packageName, - defaultSettings, }); const template = getTemplate({ templateIndexPattern, pipelineName, packageName, - composedOfTemplates: componentTemplateNames, + composedOfTemplates: Object.keys(componentTemplates), templatePriority, hidden: dataStream.hidden, }); + return { + componentTemplates, + indexTemplate: { + templateName, + indexTemplate: template, + }, + }; +} + +async function installTemplate({ + esClient, + logger, + template, +}: { + esClient: ElasticsearchClient; + logger: Logger; + template: IndexTemplateEntry; +}) { // TODO: Check return values for errors const esClientParams = { - name: templateName, - body: template, + name: template.templateName, + body: template.indexTemplate, }; await retryTransientEsErrors( () => esClient.indices.putIndexTemplate(esClientParams, { ignore: [404] }), { logger } ); - - return { - templateName, - indexTemplate: template, - }; } export function getAllTemplateRefs(installedTemplates: IndexTemplateEntry[]) { diff --git a/x-pack/plugins/fleet/server/services/epm/fields/field.ts b/x-pack/plugins/fleet/server/services/epm/fields/field.ts index 3f1a8d8b2b7ba..0e00840b0c74e 100644 --- a/x-pack/plugins/fleet/server/services/epm/fields/field.ts +++ b/x-pack/plugins/fleet/server/services/epm/fields/field.ts @@ -261,12 +261,12 @@ const isFields = (path: string) => { * Gets all field files, optionally filtered by dataset, extracts .yml files, merges them together */ -export const loadFieldsFromYaml = async ( +export const loadFieldsFromYaml = ( pkg: Pick, datasetName?: string -): Promise => { +): Field[] => { // Fetch all field definition files - const fieldDefinitionFiles = await getAssetsData(pkg, isFields, datasetName); + const fieldDefinitionFiles = getAssetsData(pkg, isFields, datasetName); return fieldDefinitionFiles.reduce((acc, file) => { // Make sure it is defined as it is optional. Should never happen. if (file.buffer) { diff --git a/x-pack/plugins/fleet/server/services/epm/kibana/assets/install.ts b/x-pack/plugins/fleet/server/services/epm/kibana/assets/install.ts index 1462cd61c4bd3..b9582ce1cf148 100644 --- a/x-pack/plugins/fleet/server/services/epm/kibana/assets/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/kibana/assets/install.ts @@ -267,6 +267,7 @@ export async function installKibanaSavedObjects({ overwrite: true, readStream: createListStream(toBeSavedObjects), createNewCopies: false, + refresh: false, }) ); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index 24c324e6b7cd0..0124bff41736f 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -22,10 +22,10 @@ import { import type { InstallablePackage, InstallSource, PackageAssetReference } from '../../../../common'; import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants'; import type { AssetReference, Installation, InstallType } from '../../../types'; -import { installTemplates } from '../elasticsearch/template/install'; +import { prepareToInstallTemplates } from '../elasticsearch/template/install'; import { removeLegacyTemplates } from '../elasticsearch/template/remove_legacy'; import { - installPipelines, + prepareToInstallPipelines, isTopLevelPipeline, deletePreviousPipelines, } from '../elasticsearch/ingest_pipeline'; @@ -39,7 +39,7 @@ import { saveArchiveEntries } from '../archive/storage'; import { ConcurrentInstallOperationError } from '../../../errors'; import { packagePolicyService } from '../..'; -import { createInstallation } from './install'; +import { createInstallation, updateEsAssetReferences } from './install'; import { withPackageSpan } from './utils'; // this is only exported for testing @@ -146,17 +146,45 @@ export async function _installPackage({ installMlModel(packageInfo, paths, esClient, savedObjectsClient, logger, esReferences) ); - // installs versionized pipelines without removing currently installed ones - esReferences = await withPackageSpan('Install ingest pipelines', () => - installPipelines(packageInfo, paths, esClient, savedObjectsClient, logger, esReferences) + /** + * In order to install assets in parallel, we need to split the preparation step from the installation step. This + * allows us to know which asset references are going to be installed so that we can save them on the packages + * SO before installation begins. In the case of a failure during installing any individual asset, we'll have the + * references necessary to remove any assets in that were successfully installed during the rollback phase. + * + * This split of prepare/install could be extended to all asset types. Besides performance, it also allows us to + * more easily write unit tests against the asset generation code without needing to mock ES responses. + */ + const preparedIngestPipelines = prepareToInstallPipelines(packageInfo, paths); + const preparedIndexTemplates = prepareToInstallTemplates(packageInfo, paths, esReferences); + + // Update the references for the templates and ingest pipelines together. Need to be done togther to avoid race + // conditions on updating the installed_es field at the same time + // These must be saved before we actually attempt to install the templates or pipelines so that we know what to + // cleanup in the case that a single asset fails to install. + esReferences = await updateEsAssetReferences( + savedObjectsClient, + packageInfo.name, + esReferences, + { + assetsToRemove: preparedIndexTemplates.assetsToRemove, + assetsToAdd: [ + ...preparedIngestPipelines.assetsToAdd, + ...preparedIndexTemplates.assetsToAdd, + ], + } ); - // install or update the templates referencing the newly installed pipelines - const { installedTemplates, installedEsReferences: esReferencesAfterTemplates } = - await withPackageSpan('Install index templates', () => - installTemplates(packageInfo, esClient, logger, paths, savedObjectsClient, esReferences) - ); - esReferences = esReferencesAfterTemplates; + // Install index templates and ingest pipelines in parallel since they typically take the longest + const [installedTemplates] = await Promise.all([ + withPackageSpan('Install index templates', () => + preparedIndexTemplates.install(esClient, logger) + ), + // installs versionized pipelines without removing currently installed ones + withPackageSpan('Install ingest pipelines', () => + preparedIngestPipelines.install(esClient, logger) + ), + ]); try { await removeLegacyTemplates({ packageInfo, esClient, logger }); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/assets.ts b/x-pack/plugins/fleet/server/services/epm/packages/assets.ts index 0621d05d21497..d67e76f90e551 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/assets.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/assets.ts @@ -51,11 +51,11 @@ export function getAssets( // ASK: Does getAssetsData need an installSource now? // if so, should it be an Installation vs InstallablePackage or add another argument? -export async function getAssetsData( +export function getAssetsData( packageInfo: Pick, filter = (path: string): boolean => true, datasetName?: string -): Promise { +): ArchiveEntry[] { // Gather all asset data const assets = getAssets(packageInfo, filter, datasetName); const entries: ArchiveEntry[] = assets.map((path) => {