[Fleet] Add global component template to all fleet index templates (e…

…lastic#102225)

x-pack/plugins/fleet/common/types/index.ts

@@ -25,6 +25,7 @@ export interface FleetConfigType {
   };
   agentPolicies?: PreconfiguredAgentPolicy[];
   packages?: PreconfiguredPackage[];
+  agentIdVerificationEnabled?: boolean;
 }
 
 // Calling Object.entries(PackagesGroupedByStatus) gave `status: string`

x-pack/plugins/fleet/public/mock/plugin_configuration.ts

@@ -12,6 +12,7 @@ export const createConfigurationMock = (): FleetConfigType => {
     enabled: true,
     registryUrl: '',
     registryProxyUrl: '',
+    agentIdVerificationEnabled: true,
     agents: {
       enabled: true,
       elasticsearch: {

x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/final_pipeline.ts → x-pack/plugins/fleet/server/constants/fleet_es_assets.ts

@@ -5,9 +5,37 @@
  * 2.0.
  */
 
-export const FINAL_PIPELINE_ID = '.fleet_final_pipeline';
+export const FLEET_FINAL_PIPELINE_ID = '.fleet_final_pipeline-1';
 
-export const FINAL_PIPELINE = `---
+export const FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME = '.fleet_component_template-1';
+
+export const FLEET_GLOBAL_COMPONENT_TEMPLATE_CONTENT = {
+  _meta: {},
+  template: {
+    settings: {
+      index: {
+        final_pipeline: FLEET_FINAL_PIPELINE_ID,
+      },
+    },
+    mappings: {
+      properties: {
+        event: {
+          properties: {
+            ingested: {
+              type: 'date',
+            },
+            agent_id_status: {
+              ignore_above: 1024,
+              type: 'keyword',
+            },
+          },
+        },
+      },
+    },
+  },
+};
+
+export const FLEET_FINAL_PIPELINE_CONTENT = `---
 description: >
   Final pipeline for processing all incoming Fleet Agent documents.
 processors:

x-pack/plugins/fleet/server/constants/index.ts

@@ -57,3 +57,10 @@ export {
   PRECONFIGURATION_DELETION_RECORD_SAVED_OBJECT_TYPE,
   PRECONFIGURATION_LATEST_KEYWORD,
 } from '../../common';
+
+export {
+  FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME,
+  FLEET_GLOBAL_COMPONENT_TEMPLATE_CONTENT,
+  FLEET_FINAL_PIPELINE_ID,
+  FLEET_FINAL_PIPELINE_CONTENT,
+} from './fleet_es_assets';

x-pack/plugins/fleet/server/index.ts

@@ -77,6 +77,7 @@ export const config: PluginConfigDescriptor = {
     }),
     packages: PreconfiguredPackagesSchema,
     agentPolicies: PreconfiguredAgentPoliciesSchema,
+    agentIdVerificationEnabled: schema.boolean({ defaultValue: true }),
   }),
 };
 

x-pack/plugins/fleet/server/mocks/index.ts

@@ -4,6 +4,8 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
+import { of } from 'rxjs';
+
 import {
   elasticsearchServiceMock,
   loggingSystemMock,
@@ -22,6 +24,14 @@ import type { FleetAppContext } from '../plugin';
 export * from '../services/artifacts/mocks';
 
 export const createAppContextStartContractMock = (): FleetAppContext => {
+  const config = {
+    agents: { enabled: true, elasticsearch: {} },
+    enabled: true,
+    agentIdVerificationEnabled: true,
+  };
+
+  const config$ = of(config);
+
   return {
     elasticsearch: elasticsearchServiceMock.createStart(),
     data: dataPluginMock.createStartContract(),
@@ -33,7 +43,9 @@ export const createAppContextStartContractMock = (): FleetAppContext => {
     configInitialValue: {
       agents: { enabled: true, elasticsearch: {} },
       enabled: true,
+      agentIdVerificationEnabled: true,
     },
+    config$,
     kibanaVersion: '8.0.0',
     kibanaBranch: 'master',
   };

x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts

@@ -14,9 +14,9 @@ import { getAsset, getPathParts } from '../../archive';
 import type { ArchiveEntry } from '../../archive';
 import { saveInstalledEsRefs } from '../../packages/install';
 import { getInstallationObject } from '../../packages';
+import { FLEET_FINAL_PIPELINE_CONTENT, FLEET_FINAL_PIPELINE_ID } from '../../../../constants';
 
 import { deletePipelineRefs } from './remove';
-import { FINAL_PIPELINE, FINAL_PIPELINE_ID } from './final_pipeline';
 
 interface RewriteSubstitution {
   source: string;
@@ -190,22 +190,24 @@ export async function ensureFleetFinalPipelineIsInstalled(esClient: Elasticsearc
   const esClientRequestOptions: TransportRequestOptions = {
     ignore: [404],
   };
-  const res = await esClient.ingest.getPipeline({ id: FINAL_PIPELINE_ID }, esClientRequestOptions);
+  const res = await esClient.ingest.getPipeline(
+    { id: FLEET_FINAL_PIPELINE_ID },
+    esClientRequestOptions
+  );
 
   if (res.statusCode === 404) {
-    await esClient.ingest.putPipeline(
-      // @ts-ignore pipeline is define in yaml
-      { id: FINAL_PIPELINE_ID, body: FINAL_PIPELINE },
-      {
-        headers: {
-          // pipeline is YAML
-          'Content-Type': 'application/yaml',
-          // but we want JSON responses (to extract error messages, status code, or other metadata)
-          Accept: 'application/json',
-        },
-      }
-    );
+    await installPipeline({
+      esClient,
+      pipeline: {
+        nameForInstallation: FLEET_FINAL_PIPELINE_ID,
+        contentForInstallation: FLEET_FINAL_PIPELINE_CONTENT,
+        extension: 'yml',
+      },
+    });
+    return { isCreated: true };
   }
+
+  return { isCreated: false };
 }
 
 const isDirectory = ({ path }: ArchiveEntry) => path.endsWith('/');

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts

@@ -20,6 +20,10 @@ import type { Field } from '../../fields/field';
 import { getPipelineNameForInstallation } from '../ingest_pipeline/install';
 import { getAsset, getPathParts } from '../../archive';
 import { removeAssetTypesFromInstalledEs, saveInstalledEsRefs } from '../../packages/install';
+import {
+  FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME,
+  FLEET_GLOBAL_COMPONENT_TEMPLATE_CONTENT,
+} from '../../../../constants';
 
 import {
   generateMappings,
@@ -164,7 +168,7 @@ export async function installTemplateForDataStream({
 }
 
 interface TemplateMapEntry {
-  _meta: { package: { name: string } };
+  _meta: { package?: { name: string } };
   template:
     | {
         mappings: NonNullable<RegistryElasticsearch['index_template.mappings']>;
@@ -277,6 +281,28 @@ async function installDataStreamComponentTemplates(params: {
   return templateNames;
 }
 
+export async function ensureDefaultComponentTemplate(esClient: ElasticsearchClient) {
+  const { body: getTemplateRes } = await esClient.cluster.getComponentTemplate(
+    {
+      name: FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME,
+    },
+    {
+      ignore: [404],
+    }
+  );
+
+  const existingTemplate = getTemplateRes?.component_templates?.[0];
+  if (!existingTemplate) {
+    await putComponentTemplate(esClient, {
+      name: FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME,
+      body: FLEET_GLOBAL_COMPONENT_TEMPLATE_CONTENT,
+      create: true,
+    });
+  }
+
+  return { isCreated: !existingTemplate };
+}
+
 export async function installTemplate({
   esClient,
   fields,
@@ -378,12 +404,13 @@ export function getAllTemplateRefs(installedTemplates: IndexTemplateEntry[]) {
         type: ElasticsearchAssetType.indexTemplate,
       },
     ];
-    const componentTemplates = installedTemplate.indexTemplate.composed_of.map(
-      (componentTemplateId) => ({
+    const componentTemplates = installedTemplate.indexTemplate.composed_of
+      // Filter global component template shared between integrations
+      .filter((componentTemplateId) => componentTemplateId !== FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME)
+      .map((componentTemplateId) => ({
         id: componentTemplateId,
         type: ElasticsearchAssetType.componentTemplate,
-      })
-    );
+      }));
     return indexTemplates.concat(componentTemplates);
   });
 }

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts

@@ -24,6 +24,8 @@ import {
   generateTemplateIndexPattern,
 } from './template';
 
+const FLEET_COMPONENT_TEMPLATE = '.fleet_component_template-1';
+
 // Add our own serialiser to just do JSON.stringify
 expect.addSnapshotSerializer({
   print(val) {
@@ -67,7 +69,7 @@ describe('EPM template', () => {
       composedOfTemplates,
       templatePriority: 200,
     });
-    expect(template.composed_of).toStrictEqual(composedOfTemplates);
+    expect(template.composed_of).toStrictEqual([...composedOfTemplates, FLEET_COMPONENT_TEMPLATE]);
   });
 
   it('adds empty composed_of correctly', () => {
@@ -82,7 +84,7 @@ describe('EPM template', () => {
       composedOfTemplates,
       templatePriority: 200,
     });
-    expect(template.composed_of).toStrictEqual(composedOfTemplates);
+    expect(template.composed_of).toStrictEqual([FLEET_COMPONENT_TEMPLATE]);
   });
 
   it('adds hidden field correctly', () => {

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts

@@ -16,7 +16,7 @@ import type {
 } from '../../../../types';
 import { appContextService } from '../../../';
 import { getRegistryDataStreamAssetBaseName } from '../index';
-import { FINAL_PIPELINE_ID } from '../ingest_pipeline/final_pipeline';
+import { FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME } from '../../../../constants';
 
 interface Properties {
   [key: string]: any;
@@ -90,7 +90,11 @@ export function getTemplate({
   if (template.template.settings.index.final_pipeline) {
     throw new Error(`Error template for ${templateIndexPattern} contains a final_pipeline`);
   }
-  template.template.settings.index.final_pipeline = FINAL_PIPELINE_ID;
+
+  if (appContextService.getConfig()?.agentIdVerificationEnabled) {
+    // Add fleet global assets
+    template.composed_of = [...(template.composed_of || []), FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME];
+  }
 
   return template;
 }

x-pack/plugins/fleet/server/services/epm/packages/get.ts

@@ -101,6 +101,8 @@ export async function getPackageSavedObjects(
   });
 }
 
+export const getInstallations = getPackageSavedObjects;
+
 export async function getPackageInfo(options: {
   savedObjectsClient: SavedObjectsClientContract;
   pkgName: string;

x-pack/plugins/fleet/server/services/epm/packages/index.ts

@@ -17,6 +17,7 @@ export {
   getFile,
   getInstallationObject,
   getInstallation,
+  getInstallations,
   getPackageInfo,
   getPackages,
   getLimitedPackages,