[Fleet] Fix loading fields for transform destination index template (#…

…177608)

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts

@@ -6,18 +6,18 @@
  */
 import { createAppContextStartContractMock } from '../../../../mocks';
 import { appContextService } from '../../..';
-import { loadFieldsFromYaml } from '../../fields/field';
+import { loadDatastreamsFieldsFromYaml } from '../../fields/field';
 import type { PackageInstallContext, RegistryDataStream } from '../../../../../common/types';
 
 import { prepareTemplate, prepareToInstallTemplates } from './install';
 
 jest.mock('../../fields/field', () => ({
   ...jest.requireActual('../../fields/field'),
-  loadFieldsFromYaml: jest.fn(),
+  loadDatastreamsFieldsFromYaml: jest.fn(),
 }));
 
-const mockedLoadFieldsFromYaml = loadFieldsFromYaml as jest.MockedFunction<
-  typeof loadFieldsFromYaml
+const mockedLoadFieldsFromYaml = loadDatastreamsFieldsFromYaml as jest.MockedFunction<
+  typeof loadDatastreamsFieldsFromYaml
 >;
 const packageInstallContext = {
   packageInfo: { name: 'package', version: '0.0.1' },

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts

@@ -30,7 +30,7 @@ import type {
   EsAssetReference,
   ExperimentalDataStreamFeature,
 } from '../../../../types';
-import { loadFieldsFromYaml, processFields } from '../../fields/field';
+import { loadDatastreamsFieldsFromYaml, processFields } from '../../fields/field';
 import { getAssetFromAssetsMap, getPathParts } from '../../archive';
 import {
   FLEET_COMPONENT_TEMPLATES,
@@ -509,7 +509,7 @@ export function prepareTemplate({
   experimentalDataStreamFeature?: ExperimentalDataStreamFeature;
 }): { componentTemplates: TemplateMap; indexTemplate: IndexTemplateEntry } {
   const { name: packageName, version: packageVersion } = packageInstallContext.packageInfo;
-  const fields = loadFieldsFromYaml(packageInstallContext, dataStream.path);
+  const fields = loadDatastreamsFieldsFromYaml(packageInstallContext, dataStream.path);
 
   const isIndexModeTimeSeries =
     dataStream.elasticsearch?.index_mode === 'time_series' ||

x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts

@@ -25,8 +25,7 @@ import {
   buildComponentTemplates,
   installComponentAndIndexTemplateForDataStream,
 } from '../template/install';
-import { isFields, processFields } from '../../fields/field';
-import { generateMappings } from '../template/template';
+import { isFields } from '../../fields/field';
 import { getESAssetMetadata } from '../meta';
 import { updateEsAssetReferences } from '../../packages/es_assets_reference';
 import { getAssetFromAssetsMap, getPathParts } from '../../archive';
@@ -47,6 +46,7 @@ import { isUserSettingsTemplate } from '../template/utils';
 
 import { deleteTransforms } from './remove';
 import { getDestinationIndexAliases } from './transform_utils';
+import { loadMappingForTransform } from './mappings';
 
 const DEFAULT_TRANSFORM_TEMPLATES_PRIORITY = 250;
 enum TRANSFORM_SPECS_TYPES {
@@ -183,8 +183,6 @@ const processTransformAssetsPerModule = (
 
     // Handling fields.yml and all other files within 'fields' folder
     if (fileName === TRANSFORM_SPECS_TYPES.FIELDS || isFields(path)) {
-      const validFields = processFields(content);
-      const mappings = generateMappings(validFields);
       const templateName = getTransformAssetNameForInstallation(
         installablePackage,
         transformModuleId,
@@ -208,14 +206,6 @@ const processTransformAssetsPerModule = (
       } else {
         destinationIndexTemplates[indexToModify] = template;
       }
-
-      // If there's already mappings set previously, append it to new
-      const previousMappings =
-        transformsSpecifications.get(transformModuleId)?.get('mappings') ?? {};
-
-      transformsSpecifications.get(transformModuleId)?.set('mappings', {
-        properties: { ...previousMappings.properties, ...mappings.properties },
-      });
     }
 
     if (fileName === TRANSFORM_SPECS_TYPES.TRANSFORM) {
@@ -394,6 +384,20 @@ const processTransformAssetsPerModule = (
     version: t.transformVersion,
   }));
 
+  // Load and generate mappings
+  for (const destinationIndexTemplate of destinationIndexTemplates) {
+    if (!destinationIndexTemplate.transformModuleId) {
+      continue;
+    }
+
+    transformsSpecifications
+      .get(destinationIndexTemplate.transformModuleId)
+      ?.set(
+        'mappings',
+        loadMappingForTransform(packageInstallContext, destinationIndexTemplate.transformModuleId)
+      );
+  }
+
   return {
     indicesToAddRefs,
     indexTemplatesRefs,

x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/mappings.test.ts

@@ -0,0 +1,91 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { loadMappingForTransform } from './mappings';
+
+describe('loadMappingForTransform', () => {
+  it('should return a mappings without properties if there is no fields resource', () => {
+    const fields = loadMappingForTransform(
+      {
+        packageInfo: {} as any,
+        assetsMap: new Map(),
+        paths: [],
+      },
+      'test'
+    );
+
+    expect(fields).toEqual({ properties: {} });
+  });
+
+  it('should merge shallow mapping without properties if there is no fields resource', () => {
+    const fields = loadMappingForTransform(
+      {
+        packageInfo: {} as any,
+        assetsMap: new Map([
+          [
+            '/package/ti_opencti/2.1.0/elasticsearch/transform/latest_ioc/fields/ecs.yml',
+            Buffer.from(
+              `
+- description: Description of the threat feed in a UI friendly format.
+  name: threat.feed.description
+  type: keyword
+- description: The name of the threat feed in UI friendly format.
+  name: threat.feed.name
+  type: keyword`
+            ),
+          ],
+          [
+            '/package/ti_opencti/2.1.0/elasticsearch/transform/latest_ioc/fields/ecs-extra.yml',
+            Buffer.from(
+              `
+- description: The display name indicator in an UI friendly format
+  level: extended
+  name: threat.indicator.name
+  type: keyword`
+            ),
+          ],
+        ]),
+        paths: [
+          '/package/ti_opencti/2.1.0/elasticsearch/transform/latest_ioc/fields/ecs.yml',
+          '/package/ti_opencti/2.1.0/elasticsearch/transform/latest_ioc/fields/ecs-extra.yml',
+        ],
+      },
+      'latest_ioc'
+    );
+
+    expect(fields).toMatchInlineSnapshot(`
+      Object {
+        "properties": Object {
+          "threat": Object {
+            "properties": Object {
+              "feed": Object {
+                "properties": Object {
+                  "description": Object {
+                    "ignore_above": 1024,
+                    "type": "keyword",
+                  },
+                  "name": Object {
+                    "ignore_above": 1024,
+                    "type": "keyword",
+                  },
+                },
+              },
+              "indicator": Object {
+                "properties": Object {
+                  "name": Object {
+                    "ignore_above": 1024,
+                    "type": "keyword",
+                  },
+                },
+              },
+            },
+          },
+        },
+      }
+    `);
+  });
+});

x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/mappings.ts

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { type PackageInstallContext } from '../../../../../common/types/models';
+import { loadTransformFieldsFromYaml, processFields } from '../../fields/field';
+import { generateMappings } from '../template/template';
+
+export function loadMappingForTransform(
+  packageInstallContext: PackageInstallContext,
+  transformModuleId: string
+) {
+  const fields = loadTransformFieldsFromYaml(packageInstallContext, transformModuleId);
+  const validFields = processFields(fields);
+  return generateMappings(validFields);
+}

x-pack/plugins/fleet/server/services/epm/fields/field.ts

@@ -289,13 +289,25 @@ export const isFields = (path: string) => {
   return path.includes('/fields/');
 };
 
+export const filterForTransformAssets = (transformName: string) => {
+  return function isTransformAssets(path: string) {
+    return path.includes(`/transform/${transformName}`);
+  };
+};
+
+function combineFilter(...filters: Array<(path: string) => boolean>) {
+  return function filterAsset(path: string) {
+    return filters.every((filter) => filter(path));
+  };
+}
+
 /**
  * loadFieldsFromYaml
  *
  * Gets all field files, optionally filtered by dataset, extracts .yml files, merges them together
  */
 
-export const loadFieldsFromYaml = (
+export const loadDatastreamsFieldsFromYaml = (
   packageInstallContext: PackageInstallContext,
   datasetName?: string
 ): Field[] => {
@@ -318,3 +330,26 @@ export const loadFieldsFromYaml = (
     return acc;
   }, []);
 };
+
+export const loadTransformFieldsFromYaml = (
+  packageInstallContext: PackageInstallContext,
+  transformName: string
+): Field[] => {
+  // Fetch all field definition files
+  const fieldDefinitionFiles = getAssetsDataFromAssetsMap(
+    packageInstallContext.packageInfo,
+    packageInstallContext.assetsMap,
+    combineFilter(isFields, filterForTransformAssets(transformName))
+  );
+  return fieldDefinitionFiles.reduce<Field[]>((acc, file) => {
+    // Make sure it is defined as it is optional. Should never happen.
+    if (file.buffer) {
+      const tmpFields = safeLoad(file.buffer.toString());
+      // safeLoad() returns undefined for empty files, we don't want that
+      if (tmpFields) {
+        acc = acc.concat(tmpFields);
+      }
+    }
+    return acc;
+  }, []);
+};