From c98ee2f6c10992f372ebea4ce7d91f02a68fce5c Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Fri, 23 Feb 2024 09:50:35 -0500 Subject: [PATCH] [Fleet] Fix loading fields for transform destination index template (#177608) --- .../elasticsearch/template/install.test.ts | 8 +- .../epm/elasticsearch/template/install.ts | 4 +- .../epm/elasticsearch/transform/install.ts | 28 +++--- .../elasticsearch/transform/mappings.test.ts | 91 +++++++++++++++++++ .../epm/elasticsearch/transform/mappings.ts | 18 ++++ .../fleet/server/services/epm/fields/field.ts | 37 +++++++- 6 files changed, 167 insertions(+), 19 deletions(-) create mode 100644 x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/mappings.test.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/mappings.ts diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts index 61edf78ec497e..b729e34fa8bb6 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts @@ -6,18 +6,18 @@ */ import { createAppContextStartContractMock } from '../../../../mocks'; import { appContextService } from '../../..'; -import { loadFieldsFromYaml } from '../../fields/field'; +import { loadDatastreamsFieldsFromYaml } from '../../fields/field'; import type { PackageInstallContext, RegistryDataStream } from '../../../../../common/types'; import { prepareTemplate, prepareToInstallTemplates } from './install'; jest.mock('../../fields/field', () => ({ ...jest.requireActual('../../fields/field'), - loadFieldsFromYaml: jest.fn(), + loadDatastreamsFieldsFromYaml: jest.fn(), })); -const mockedLoadFieldsFromYaml = loadFieldsFromYaml as jest.MockedFunction< - typeof loadFieldsFromYaml +const mockedLoadFieldsFromYaml = loadDatastreamsFieldsFromYaml as jest.MockedFunction< + typeof loadDatastreamsFieldsFromYaml >; const packageInstallContext = { packageInfo: { name: 'package', version: '0.0.1' }, diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index e1ff612076b3b..2dce7b7323567 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -30,7 +30,7 @@ import type { EsAssetReference, ExperimentalDataStreamFeature, } from '../../../../types'; -import { loadFieldsFromYaml, processFields } from '../../fields/field'; +import { loadDatastreamsFieldsFromYaml, processFields } from '../../fields/field'; import { getAssetFromAssetsMap, getPathParts } from '../../archive'; import { FLEET_COMPONENT_TEMPLATES, @@ -509,7 +509,7 @@ export function prepareTemplate({ experimentalDataStreamFeature?: ExperimentalDataStreamFeature; }): { componentTemplates: TemplateMap; indexTemplate: IndexTemplateEntry } { const { name: packageName, version: packageVersion } = packageInstallContext.packageInfo; - const fields = loadFieldsFromYaml(packageInstallContext, dataStream.path); + const fields = loadDatastreamsFieldsFromYaml(packageInstallContext, dataStream.path); const isIndexModeTimeSeries = dataStream.elasticsearch?.index_mode === 'time_series' || diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts index 84ea5fae04874..308a10f1bf94d 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts @@ -25,8 +25,7 @@ import { buildComponentTemplates, installComponentAndIndexTemplateForDataStream, } from '../template/install'; -import { isFields, processFields } from '../../fields/field'; -import { generateMappings } from '../template/template'; +import { isFields } from '../../fields/field'; import { getESAssetMetadata } from '../meta'; import { updateEsAssetReferences } from '../../packages/es_assets_reference'; import { getAssetFromAssetsMap, getPathParts } from '../../archive'; @@ -47,6 +46,7 @@ import { isUserSettingsTemplate } from '../template/utils'; import { deleteTransforms } from './remove'; import { getDestinationIndexAliases } from './transform_utils'; +import { loadMappingForTransform } from './mappings'; const DEFAULT_TRANSFORM_TEMPLATES_PRIORITY = 250; enum TRANSFORM_SPECS_TYPES { @@ -183,8 +183,6 @@ const processTransformAssetsPerModule = ( // Handling fields.yml and all other files within 'fields' folder if (fileName === TRANSFORM_SPECS_TYPES.FIELDS || isFields(path)) { - const validFields = processFields(content); - const mappings = generateMappings(validFields); const templateName = getTransformAssetNameForInstallation( installablePackage, transformModuleId, @@ -208,14 +206,6 @@ const processTransformAssetsPerModule = ( } else { destinationIndexTemplates[indexToModify] = template; } - - // If there's already mappings set previously, append it to new - const previousMappings = - transformsSpecifications.get(transformModuleId)?.get('mappings') ?? {}; - - transformsSpecifications.get(transformModuleId)?.set('mappings', { - properties: { ...previousMappings.properties, ...mappings.properties }, - }); } if (fileName === TRANSFORM_SPECS_TYPES.TRANSFORM) { @@ -394,6 +384,20 @@ const processTransformAssetsPerModule = ( version: t.transformVersion, })); + // Load and generate mappings + for (const destinationIndexTemplate of destinationIndexTemplates) { + if (!destinationIndexTemplate.transformModuleId) { + continue; + } + + transformsSpecifications + .get(destinationIndexTemplate.transformModuleId) + ?.set( + 'mappings', + loadMappingForTransform(packageInstallContext, destinationIndexTemplate.transformModuleId) + ); + } + return { indicesToAddRefs, indexTemplatesRefs, diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/mappings.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/mappings.test.ts new file mode 100644 index 0000000000000..f34015bf77697 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/mappings.test.ts @@ -0,0 +1,91 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loadMappingForTransform } from './mappings'; + +describe('loadMappingForTransform', () => { + it('should return a mappings without properties if there is no fields resource', () => { + const fields = loadMappingForTransform( + { + packageInfo: {} as any, + assetsMap: new Map(), + paths: [], + }, + 'test' + ); + + expect(fields).toEqual({ properties: {} }); + }); + + it('should merge shallow mapping without properties if there is no fields resource', () => { + const fields = loadMappingForTransform( + { + packageInfo: {} as any, + assetsMap: new Map([ + [ + '/package/ti_opencti/2.1.0/elasticsearch/transform/latest_ioc/fields/ecs.yml', + Buffer.from( + ` +- description: Description of the threat feed in a UI friendly format. + name: threat.feed.description + type: keyword +- description: The name of the threat feed in UI friendly format. + name: threat.feed.name + type: keyword` + ), + ], + [ + '/package/ti_opencti/2.1.0/elasticsearch/transform/latest_ioc/fields/ecs-extra.yml', + Buffer.from( + ` +- description: The display name indicator in an UI friendly format + level: extended + name: threat.indicator.name + type: keyword` + ), + ], + ]), + paths: [ + '/package/ti_opencti/2.1.0/elasticsearch/transform/latest_ioc/fields/ecs.yml', + '/package/ti_opencti/2.1.0/elasticsearch/transform/latest_ioc/fields/ecs-extra.yml', + ], + }, + 'latest_ioc' + ); + + expect(fields).toMatchInlineSnapshot(` + Object { + "properties": Object { + "threat": Object { + "properties": Object { + "feed": Object { + "properties": Object { + "description": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "indicator": Object { + "properties": Object { + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + }, + }, + }, + } + `); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/mappings.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/mappings.ts new file mode 100644 index 0000000000000..130dae0ecca51 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/mappings.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { type PackageInstallContext } from '../../../../../common/types/models'; +import { loadTransformFieldsFromYaml, processFields } from '../../fields/field'; +import { generateMappings } from '../template/template'; + +export function loadMappingForTransform( + packageInstallContext: PackageInstallContext, + transformModuleId: string +) { + const fields = loadTransformFieldsFromYaml(packageInstallContext, transformModuleId); + const validFields = processFields(fields); + return generateMappings(validFields); +} diff --git a/x-pack/plugins/fleet/server/services/epm/fields/field.ts b/x-pack/plugins/fleet/server/services/epm/fields/field.ts index 859752d5dead1..b8ca555c95a9b 100644 --- a/x-pack/plugins/fleet/server/services/epm/fields/field.ts +++ b/x-pack/plugins/fleet/server/services/epm/fields/field.ts @@ -289,13 +289,25 @@ export const isFields = (path: string) => { return path.includes('/fields/'); }; +export const filterForTransformAssets = (transformName: string) => { + return function isTransformAssets(path: string) { + return path.includes(`/transform/${transformName}`); + }; +}; + +function combineFilter(...filters: Array<(path: string) => boolean>) { + return function filterAsset(path: string) { + return filters.every((filter) => filter(path)); + }; +} + /** * loadFieldsFromYaml * * Gets all field files, optionally filtered by dataset, extracts .yml files, merges them together */ -export const loadFieldsFromYaml = ( +export const loadDatastreamsFieldsFromYaml = ( packageInstallContext: PackageInstallContext, datasetName?: string ): Field[] => { @@ -318,3 +330,26 @@ export const loadFieldsFromYaml = ( return acc; }, []); }; + +export const loadTransformFieldsFromYaml = ( + packageInstallContext: PackageInstallContext, + transformName: string +): Field[] => { + // Fetch all field definition files + const fieldDefinitionFiles = getAssetsDataFromAssetsMap( + packageInstallContext.packageInfo, + packageInstallContext.assetsMap, + combineFilter(isFields, filterForTransformAssets(transformName)) + ); + return fieldDefinitionFiles.reduce((acc, file) => { + // Make sure it is defined as it is optional. Should never happen. + if (file.buffer) { + const tmpFields = safeLoad(file.buffer.toString()); + // safeLoad() returns undefined for empty files, we don't want that + if (tmpFields) { + acc = acc.concat(tmpFields); + } + } + return acc; + }, []); +};