From 3864fe1559281b4a2731e3a9439c501125e65669 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Wed, 23 Jun 2021 13:18:37 -0400 Subject: [PATCH] [Fleet] Add global component template to all fleet index templates (#102225) --- x-pack/plugins/fleet/common/types/index.ts | 1 + .../fleet/public/mock/plugin_configuration.ts | 1 + .../fleet_es_assets.ts} | 32 +++++++++++- .../plugins/fleet/server/constants/index.ts | 7 +++ x-pack/plugins/fleet/server/index.ts | 1 + x-pack/plugins/fleet/server/mocks/index.ts | 12 +++++ .../elasticsearch/ingest_pipeline/install.ts | 30 ++++++----- .../__snapshots__/template.test.ts.snap | 21 ++++---- .../epm/elasticsearch/template/install.ts | 37 ++++++++++++-- .../elasticsearch/template/template.test.ts | 6 ++- .../epm/elasticsearch/template/template.ts | 8 ++- .../fleet/server/services/epm/packages/get.ts | 2 + .../server/services/epm/packages/index.ts | 1 + x-pack/plugins/fleet/server/services/setup.ts | 51 ++++++++++++++++++- .../api_integration/apis/ml/modules/index.ts | 3 ++ .../apis/epm/final_pipeline.ts | 10 ++-- .../apis/epm/install_overrides.ts | 1 + 17 files changed, 182 insertions(+), 42 deletions(-) rename x-pack/plugins/fleet/server/{services/epm/elasticsearch/ingest_pipeline/final_pipeline.ts => constants/fleet_es_assets.ts} (82%) diff --git a/x-pack/plugins/fleet/common/types/index.ts b/x-pack/plugins/fleet/common/types/index.ts index 95f91165aaf94..59691bf32d099 100644 --- a/x-pack/plugins/fleet/common/types/index.ts +++ b/x-pack/plugins/fleet/common/types/index.ts @@ -25,6 +25,7 @@ export interface FleetConfigType { }; agentPolicies?: PreconfiguredAgentPolicy[]; packages?: PreconfiguredPackage[]; + agentIdVerificationEnabled?: boolean; } // Calling Object.entries(PackagesGroupedByStatus) gave `status: string` diff --git a/x-pack/plugins/fleet/public/mock/plugin_configuration.ts b/x-pack/plugins/fleet/public/mock/plugin_configuration.ts index 097b6aa98c067..5dad8ad504979 100644 --- a/x-pack/plugins/fleet/public/mock/plugin_configuration.ts +++ b/x-pack/plugins/fleet/public/mock/plugin_configuration.ts @@ -12,6 +12,7 @@ export const createConfigurationMock = (): FleetConfigType => { enabled: true, registryUrl: '', registryProxyUrl: '', + agentIdVerificationEnabled: true, agents: { enabled: true, elasticsearch: { diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/final_pipeline.ts b/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts similarity index 82% rename from x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/final_pipeline.ts rename to x-pack/plugins/fleet/server/constants/fleet_es_assets.ts index f929a4f139981..8e9dac11db799 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/final_pipeline.ts +++ b/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts @@ -5,9 +5,37 @@ * 2.0. */ -export const FINAL_PIPELINE_ID = '.fleet_final_pipeline'; +export const FLEET_FINAL_PIPELINE_ID = '.fleet_final_pipeline-1'; -export const FINAL_PIPELINE = `--- +export const FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME = '.fleet_component_template-1'; + +export const FLEET_GLOBAL_COMPONENT_TEMPLATE_CONTENT = { + _meta: {}, + template: { + settings: { + index: { + final_pipeline: FLEET_FINAL_PIPELINE_ID, + }, + }, + mappings: { + properties: { + event: { + properties: { + ingested: { + type: 'date', + }, + agent_id_status: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + }, + }, + }, +}; + +export const FLEET_FINAL_PIPELINE_CONTENT = `--- description: > Final pipeline for processing all incoming Fleet Agent documents. processors: diff --git a/x-pack/plugins/fleet/server/constants/index.ts b/x-pack/plugins/fleet/server/constants/index.ts index 16a92a2ffa1aa..3aca5e8800dc5 100644 --- a/x-pack/plugins/fleet/server/constants/index.ts +++ b/x-pack/plugins/fleet/server/constants/index.ts @@ -57,3 +57,10 @@ export { PRECONFIGURATION_DELETION_RECORD_SAVED_OBJECT_TYPE, PRECONFIGURATION_LATEST_KEYWORD, } from '../../common'; + +export { + FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME, + FLEET_GLOBAL_COMPONENT_TEMPLATE_CONTENT, + FLEET_FINAL_PIPELINE_ID, + FLEET_FINAL_PIPELINE_CONTENT, +} from './fleet_es_assets'; diff --git a/x-pack/plugins/fleet/server/index.ts b/x-pack/plugins/fleet/server/index.ts index 0a886ffedbd6c..ab1cd9002d04a 100644 --- a/x-pack/plugins/fleet/server/index.ts +++ b/x-pack/plugins/fleet/server/index.ts @@ -77,6 +77,7 @@ export const config: PluginConfigDescriptor = { }), packages: PreconfiguredPackagesSchema, agentPolicies: PreconfiguredAgentPoliciesSchema, + agentIdVerificationEnabled: schema.boolean({ defaultValue: true }), }), }; diff --git a/x-pack/plugins/fleet/server/mocks/index.ts b/x-pack/plugins/fleet/server/mocks/index.ts index a94f274b202ad..43a5a14b425b5 100644 --- a/x-pack/plugins/fleet/server/mocks/index.ts +++ b/x-pack/plugins/fleet/server/mocks/index.ts @@ -4,6 +4,8 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ +import { of } from 'rxjs'; + import { elasticsearchServiceMock, loggingSystemMock, @@ -22,6 +24,14 @@ import type { FleetAppContext } from '../plugin'; export * from '../services/artifacts/mocks'; export const createAppContextStartContractMock = (): FleetAppContext => { + const config = { + agents: { enabled: true, elasticsearch: {} }, + enabled: true, + agentIdVerificationEnabled: true, + }; + + const config$ = of(config); + return { elasticsearch: elasticsearchServiceMock.createStart(), data: dataPluginMock.createStartContract(), @@ -33,7 +43,9 @@ export const createAppContextStartContractMock = (): FleetAppContext => { configInitialValue: { agents: { enabled: true, elasticsearch: {} }, enabled: true, + agentIdVerificationEnabled: true, }, + config$, kibanaVersion: '8.0.0', kibanaBranch: 'master', }; diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts index 1d212f188120f..a6aa87c5ed0f5 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts @@ -14,9 +14,9 @@ import { getAsset, getPathParts } from '../../archive'; import type { ArchiveEntry } from '../../archive'; import { saveInstalledEsRefs } from '../../packages/install'; import { getInstallationObject } from '../../packages'; +import { FLEET_FINAL_PIPELINE_CONTENT, FLEET_FINAL_PIPELINE_ID } from '../../../../constants'; import { deletePipelineRefs } from './remove'; -import { FINAL_PIPELINE, FINAL_PIPELINE_ID } from './final_pipeline'; interface RewriteSubstitution { source: string; @@ -190,22 +190,24 @@ export async function ensureFleetFinalPipelineIsInstalled(esClient: Elasticsearc const esClientRequestOptions: TransportRequestOptions = { ignore: [404], }; - const res = await esClient.ingest.getPipeline({ id: FINAL_PIPELINE_ID }, esClientRequestOptions); + const res = await esClient.ingest.getPipeline( + { id: FLEET_FINAL_PIPELINE_ID }, + esClientRequestOptions + ); if (res.statusCode === 404) { - await esClient.ingest.putPipeline( - // @ts-ignore pipeline is define in yaml - { id: FINAL_PIPELINE_ID, body: FINAL_PIPELINE }, - { - headers: { - // pipeline is YAML - 'Content-Type': 'application/yaml', - // but we want JSON responses (to extract error messages, status code, or other metadata) - Accept: 'application/json', - }, - } - ); + await installPipeline({ + esClient, + pipeline: { + nameForInstallation: FLEET_FINAL_PIPELINE_ID, + contentForInstallation: FLEET_FINAL_PIPELINE_CONTENT, + extension: 'yml', + }, + }); + return { isCreated: true }; } + + return { isCreated: false }; } const isDirectory = ({ path }: ArchiveEntry) => path.endsWith('/'); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap index acf8ae742bf8f..6a4476316bfa5 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap @@ -25,8 +25,7 @@ exports[`EPM template tests loading base.yml: base.yml 1`] = ` "default_field": [ "long.nested.foo" ] - }, - "final_pipeline": ".fleet_final_pipeline" + } } }, "mappings": { @@ -99,7 +98,9 @@ exports[`EPM template tests loading base.yml: base.yml 1`] = ` } }, "data_stream": {}, - "composed_of": [], + "composed_of": [ + ".fleet_component_template-1" + ], "_meta": { "package": { "name": "nginx" @@ -140,8 +141,7 @@ exports[`EPM template tests loading coredns.logs.yml: coredns.logs.yml 1`] = ` "coredns.response.code", "coredns.response.flags" ] - }, - "final_pipeline": ".fleet_final_pipeline" + } } }, "mappings": { @@ -214,7 +214,9 @@ exports[`EPM template tests loading coredns.logs.yml: coredns.logs.yml 1`] = ` } }, "data_stream": {}, - "composed_of": [], + "composed_of": [ + ".fleet_component_template-1" + ], "_meta": { "package": { "name": "coredns" @@ -283,8 +285,7 @@ exports[`EPM template tests loading system.yml: system.yml 1`] = ` "system.users.scope", "system.users.remote_host" ] - }, - "final_pipeline": ".fleet_final_pipeline" + } } }, "mappings": { @@ -1741,7 +1742,9 @@ exports[`EPM template tests loading system.yml: system.yml 1`] = ` } }, "data_stream": {}, - "composed_of": [], + "composed_of": [ + ".fleet_component_template-1" + ], "_meta": { "package": { "name": "system" diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index db1fba1eedccd..e8dac60ddba1a 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -20,6 +20,10 @@ import type { Field } from '../../fields/field'; import { getPipelineNameForInstallation } from '../ingest_pipeline/install'; import { getAsset, getPathParts } from '../../archive'; import { removeAssetTypesFromInstalledEs, saveInstalledEsRefs } from '../../packages/install'; +import { + FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME, + FLEET_GLOBAL_COMPONENT_TEMPLATE_CONTENT, +} from '../../../../constants'; import { generateMappings, @@ -164,7 +168,7 @@ export async function installTemplateForDataStream({ } interface TemplateMapEntry { - _meta: { package: { name: string } }; + _meta: { package?: { name: string } }; template: | { mappings: NonNullable; @@ -277,6 +281,28 @@ async function installDataStreamComponentTemplates(params: { return templateNames; } +export async function ensureDefaultComponentTemplate(esClient: ElasticsearchClient) { + const { body: getTemplateRes } = await esClient.cluster.getComponentTemplate( + { + name: FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME, + }, + { + ignore: [404], + } + ); + + const existingTemplate = getTemplateRes?.component_templates?.[0]; + if (!existingTemplate) { + await putComponentTemplate(esClient, { + name: FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME, + body: FLEET_GLOBAL_COMPONENT_TEMPLATE_CONTENT, + create: true, + }); + } + + return { isCreated: !existingTemplate }; +} + export async function installTemplate({ esClient, fields, @@ -378,12 +404,13 @@ export function getAllTemplateRefs(installedTemplates: IndexTemplateEntry[]) { type: ElasticsearchAssetType.indexTemplate, }, ]; - const componentTemplates = installedTemplate.indexTemplate.composed_of.map( - (componentTemplateId) => ({ + const componentTemplates = installedTemplate.indexTemplate.composed_of + // Filter global component template shared between integrations + .filter((componentTemplateId) => componentTemplateId !== FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME) + .map((componentTemplateId) => ({ id: componentTemplateId, type: ElasticsearchAssetType.componentTemplate, - }) - ); + })); return indexTemplates.concat(componentTemplates); }); } diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts index ae7bff618dba2..d1f806f67ca5c 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts @@ -24,6 +24,8 @@ import { generateTemplateIndexPattern, } from './template'; +const FLEET_COMPONENT_TEMPLATE = '.fleet_component_template-1'; + // Add our own serialiser to just do JSON.stringify expect.addSnapshotSerializer({ print(val) { @@ -67,7 +69,7 @@ describe('EPM template', () => { composedOfTemplates, templatePriority: 200, }); - expect(template.composed_of).toStrictEqual(composedOfTemplates); + expect(template.composed_of).toStrictEqual([...composedOfTemplates, FLEET_COMPONENT_TEMPLATE]); }); it('adds empty composed_of correctly', () => { @@ -82,7 +84,7 @@ describe('EPM template', () => { composedOfTemplates, templatePriority: 200, }); - expect(template.composed_of).toStrictEqual(composedOfTemplates); + expect(template.composed_of).toStrictEqual([FLEET_COMPONENT_TEMPLATE]); }); it('adds hidden field correctly', () => { diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index 158996cc574d7..6aa7680395bed 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -16,7 +16,7 @@ import type { } from '../../../../types'; import { appContextService } from '../../../'; import { getRegistryDataStreamAssetBaseName } from '../index'; -import { FINAL_PIPELINE_ID } from '../ingest_pipeline/final_pipeline'; +import { FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME } from '../../../../constants'; interface Properties { [key: string]: any; @@ -90,7 +90,11 @@ export function getTemplate({ if (template.template.settings.index.final_pipeline) { throw new Error(`Error template for ${templateIndexPattern} contains a final_pipeline`); } - template.template.settings.index.final_pipeline = FINAL_PIPELINE_ID; + + if (appContextService.getConfig()?.agentIdVerificationEnabled) { + // Add fleet global assets + template.composed_of = [...(template.composed_of || []), FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME]; + } return template; } diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get.ts b/x-pack/plugins/fleet/server/services/epm/packages/get.ts index 28af2b563da79..6a5968441e634 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get.ts @@ -101,6 +101,8 @@ export async function getPackageSavedObjects( }); } +export const getInstallations = getPackageSavedObjects; + export async function getPackageInfo(options: { savedObjectsClient: SavedObjectsClientContract; pkgName: string; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/index.ts b/x-pack/plugins/fleet/server/services/epm/packages/index.ts index 608e157017e9b..1f9113590f0f7 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/index.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/index.ts @@ -17,6 +17,7 @@ export { getFile, getInstallationObject, getInstallation, + getInstallations, getPackageInfo, getPackages, getLimitedPackages, diff --git a/x-pack/plugins/fleet/server/services/setup.ts b/x-pack/plugins/fleet/server/services/setup.ts index 45805bb066c3b..cfef04846d92e 100644 --- a/x-pack/plugins/fleet/server/services/setup.ts +++ b/x-pack/plugins/fleet/server/services/setup.ts @@ -24,7 +24,10 @@ import { awaitIfPending } from './setup_utils'; import { ensureAgentActionPolicyChangeExists } from './agents'; import { awaitIfFleetServerSetupPending } from './fleet_server'; import { ensureFleetFinalPipelineIsInstalled } from './epm/elasticsearch/ingest_pipeline/install'; +import { ensureDefaultComponentTemplate } from './epm/elasticsearch/template/install'; +import { getInstallations, installPackage } from './epm/packages'; import { isPackageInstalled } from './epm/packages/install'; +import { pkgToPkgKey } from './epm/registry'; export interface SetupStatus { isInitialized: boolean; @@ -47,9 +50,10 @@ async function createSetupSideEffects( settingsService.settingsSetup(soClient), ]); - await ensureFleetFinalPipelineIsInstalled(esClient); - await awaitIfFleetServerSetupPending(); + if (appContextService.getConfig()?.agentIdVerificationEnabled) { + await ensureFleetGlobalEsAssets(soClient, esClient); + } const { agentPolicies: policiesOrUndefined, packages: packagesOrUndefined } = appContextService.getConfig() ?? {}; @@ -95,6 +99,49 @@ async function createSetupSideEffects( }; } +/** + * Ensure ES assets shared by all Fleet index template are installed + */ +export async function ensureFleetGlobalEsAssets( + soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient +) { + const logger = appContextService.getLogger(); + // Ensure Global Fleet ES assets are installed + const globalAssetsRes = await Promise.all([ + ensureDefaultComponentTemplate(esClient), + ensureFleetFinalPipelineIsInstalled(esClient), + ]); + + if (globalAssetsRes.some((asset) => asset.isCreated)) { + // Update existing index template + const packages = await getInstallations(soClient); + + await Promise.all( + packages.saved_objects.map(async ({ attributes: installation }) => { + if (installation.install_source !== 'registry') { + logger.error( + `Package needs to be manually reinstalled ${installation.name} after installing Fleet global assets` + ); + return; + } + await installPackage({ + installSource: installation.install_source, + savedObjectsClient: soClient, + pkgkey: pkgToPkgKey({ name: installation.name, version: installation.version }), + esClient, + // Force install the pacakge will update the index template and the datastream write indices + force: true, + }).catch((err) => { + logger.error( + `Package needs to be manually reinstalled ${installation.name} after installing Fleet global assets: ${err.message}` + ); + }); + }) + ); + } +} + export async function ensureDefaultEnrollmentAPIKeysExists( soClient: SavedObjectsClientContract, esClient: ElasticsearchClient, diff --git a/x-pack/test/api_integration/apis/ml/modules/index.ts b/x-pack/test/api_integration/apis/ml/modules/index.ts index 1a0c532dc36fa..3cf1c7f787840 100644 --- a/x-pack/test/api_integration/apis/ml/modules/index.ts +++ b/x-pack/test/api_integration/apis/ml/modules/index.ts @@ -9,11 +9,14 @@ import { FtrProviderContext } from '../../../ftr_provider_context'; export default function ({ getService, loadTestFile }: FtrProviderContext) { const ml = getService('ml'); + const supertest = getService('supertest'); const fleetPackages = ['apache', 'nginx']; describe('modules', function () { before(async () => { + // Fleet need to be setup to be able to setup packages + await supertest.post(`/api/fleet/setup`).set({ 'kbn-xsrf': 'some-xsrf-token' }).expect(200); for (const fleetPackage of fleetPackages) { await ml.testResources.installFleetPackage(fleetPackage); } diff --git a/x-pack/test/fleet_api_integration/apis/epm/final_pipeline.ts b/x-pack/test/fleet_api_integration/apis/epm/final_pipeline.ts index 81f712e095c78..68a78dd842c4b 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/final_pipeline.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/final_pipeline.ts @@ -12,7 +12,7 @@ import { skipIfNoDockerRegistry } from '../../helpers'; const TEST_INDEX = 'logs-log.log-test'; -const FINAL_PIPELINE_ID = '.fleet_final_pipeline'; +const FINAL_PIPELINE_ID = '.fleet_final_pipeline-1'; let pkgKey: string; @@ -43,7 +43,6 @@ export default function (providerContext: FtrProviderContext) { const { body: getPackagesRes } = await supertest.get( `/api/fleet/epm/packages?experimental=true` ); - const logPackage = getPackagesRes.response.find((p: any) => p.name === 'log'); if (!logPackage) { throw new Error('No log package'); @@ -85,12 +84,11 @@ export default function (providerContext: FtrProviderContext) { it('should correctly setup the final pipeline and apply to fleet managed index template', async () => { const pipelineRes = await es.ingest.getPipeline({ id: FINAL_PIPELINE_ID }); expect(pipelineRes.body).to.have.property(FINAL_PIPELINE_ID); - const res = await es.indices.getIndexTemplate({ name: 'logs-log.log' }); expect(res.body.index_templates.length).to.be(1); - expect( - res.body.index_templates[0]?.index_template?.template?.settings?.index?.final_pipeline - ).to.be(FINAL_PIPELINE_ID); + expect(res.body.index_templates[0]?.index_template?.composed_of).to.contain( + '.fleet_component_template-1' + ); }); it('For a doc written without api key should write the correct api key status', async () => { diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_overrides.ts b/x-pack/test/fleet_api_integration/apis/epm/install_overrides.ts index 204ee8508f468..770502db49dae 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_overrides.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_overrides.ts @@ -49,6 +49,7 @@ export default function (providerContext: FtrProviderContext) { `${templateName}@mappings`, `${templateName}@settings`, `${templateName}@custom`, + '.fleet_component_template-1', ]); ({ body } = await es.transport.request({