diff --git a/x-pack/plugins/fleet/common/constants/epm.ts b/x-pack/plugins/fleet/common/constants/epm.ts index 287b7ccdb88e..297b15790b52 100644 --- a/x-pack/plugins/fleet/common/constants/epm.ts +++ b/x-pack/plugins/fleet/common/constants/epm.ts @@ -3,8 +3,8 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ - export const PACKAGES_SAVED_OBJECT_TYPE = 'epm-packages'; +export const ASSETS_SAVED_OBJECT_TYPE = 'epm-packages-assets'; export const INDEX_PATTERN_SAVED_OBJECT_TYPE = 'index-pattern'; export const INDEX_PATTERN_PLACEHOLDER_SUFFIX = '-index_pattern_placeholder'; export const MAX_TIME_COMPLETE_INSTALL = 60000; diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index 0169c0c50f65..96218906c9a1 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -8,6 +8,7 @@ // TODO: Update when https://github.com/elastic/kibana/issues/53021 is closed import { SavedObject, SavedObjectAttributes, SavedObjectReference } from 'src/core/public'; import { + ASSETS_SAVED_OBJECT_TYPE, agentAssetTypes, dataTypes, defaultPackages, @@ -270,6 +271,7 @@ export type PackageInfo = export interface Installation extends SavedObjectAttributes { installed_kibana: KibanaAssetReference[]; installed_es: EsAssetReference[]; + package_assets: PackageAssetReference[]; es_index_patterns: Record; name: string; version: string; @@ -299,6 +301,10 @@ export type EsAssetReference = Pick & { type: ElasticsearchAssetType; }; +export type PackageAssetReference = Pick & { + type: typeof ASSETS_SAVED_OBJECT_TYPE; +}; + export type RequiredPackage = typeof requiredPackages; export type DefaultPackages = typeof defaultPackages; diff --git a/x-pack/plugins/fleet/server/constants/index.ts b/x-pack/plugins/fleet/server/constants/index.ts index b1d7318ff510..f380608e6817 100644 --- a/x-pack/plugins/fleet/server/constants/index.ts +++ b/x-pack/plugins/fleet/server/constants/index.ts @@ -41,6 +41,7 @@ export { PACKAGE_POLICY_SAVED_OBJECT_TYPE, OUTPUT_SAVED_OBJECT_TYPE, PACKAGES_SAVED_OBJECT_TYPE, + ASSETS_SAVED_OBJECT_TYPE, INDEX_PATTERN_SAVED_OBJECT_TYPE, ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE, GLOBAL_SETTINGS_SAVED_OBJECT_TYPE, diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index 201ca1c7a97b..20bbee2b1c79 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -12,6 +12,7 @@ import { AGENT_POLICY_SAVED_OBJECT_TYPE, PACKAGE_POLICY_SAVED_OBJECT_TYPE, PACKAGES_SAVED_OBJECT_TYPE, + ASSETS_SAVED_OBJECT_TYPE, AGENT_SAVED_OBJECT_TYPE, AGENT_EVENT_SAVED_OBJECT_TYPE, AGENT_ACTION_SAVED_OBJECT_TYPE, @@ -304,6 +305,13 @@ const getSavedObjectTypes = ( type: { type: 'keyword' }, }, }, + package_assets: { + type: 'nested', + properties: { + id: { type: 'keyword' }, + type: { type: 'keyword' }, + }, + }, install_started_at: { type: 'date' }, install_version: { type: 'keyword' }, install_status: { type: 'keyword' }, @@ -311,6 +319,25 @@ const getSavedObjectTypes = ( }, }, }, + [ASSETS_SAVED_OBJECT_TYPE]: { + name: ASSETS_SAVED_OBJECT_TYPE, + hidden: false, + namespaceType: 'agnostic', + management: { + importableAndExportable: false, + }, + mappings: { + properties: { + package_name: { type: 'keyword' }, + package_version: { type: 'keyword' }, + install_source: { type: 'keyword' }, + asset_path: { type: 'keyword' }, + media_type: { type: 'keyword' }, + data_utf8: { type: 'text', index: false }, + data_base64: { type: 'binary' }, + }, + }, + }, }); export function registerSavedObjects( diff --git a/x-pack/plugins/fleet/server/services/epm/archive/save_to_es.ts b/x-pack/plugins/fleet/server/services/epm/archive/save_to_es.ts new file mode 100644 index 000000000000..a29ae2112f01 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/archive/save_to_es.ts @@ -0,0 +1,121 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { extname } from 'path'; +import { isBinaryFile } from 'isbinaryfile'; +import mime from 'mime-types'; +import uuidv5 from 'uuid/v5'; +import { SavedObjectsClientContract, SavedObjectsBulkCreateObject } from 'src/core/server'; +import { + ASSETS_SAVED_OBJECT_TYPE, + InstallablePackage, + InstallSource, + PackageAssetReference, +} from '../../../../common'; +import { getArchiveEntry } from './index'; + +// uuid v5 requires a SHA-1 UUID as a namespace +// used to ensure same input produces the same id +const ID_NAMESPACE = '71403015-cdd5-404b-a5da-6c43f35cad84'; + +// could be anything, picked this from https://github.com/elastic/elastic-agent-client/issues/17 +const MAX_ES_ASSET_BYTES = 4 * 1024 * 1024; + +export interface PackageAsset { + package_name: string; + package_version: string; + install_source: string; + asset_path: string; + media_type: string; + data_utf8: string; + data_base64: string; +} + +export async function archiveEntryToESDocument(opts: { + path: string; + buffer: Buffer; + name: string; + version: string; + installSource: InstallSource; +}): Promise { + const { path, buffer, name, version, installSource } = opts; + const fileExt = extname(path); + const contentType = mime.lookup(fileExt); + const mediaType = mime.contentType(contentType || fileExt); + // can use to create a data URL like `data:${mediaType};base64,${base64Data}` + + const bufferIsBinary = await isBinaryFile(buffer); + const dataUtf8 = bufferIsBinary ? '' : buffer.toString('utf8'); + const dataBase64 = bufferIsBinary ? buffer.toString('base64') : ''; + + // validation: filesize? asset type? anything else + if (dataUtf8.length > MAX_ES_ASSET_BYTES) { + throw new Error(`File at ${path} is larger than maximum allowed size of ${MAX_ES_ASSET_BYTES}`); + } + + if (dataBase64.length > MAX_ES_ASSET_BYTES) { + throw new Error( + `After base64 encoding file at ${path} is larger than maximum allowed size of ${MAX_ES_ASSET_BYTES}` + ); + } + + return { + package_name: name, + package_version: version, + install_source: installSource, + asset_path: path, + media_type: mediaType || '', + data_utf8: dataUtf8, + data_base64: dataBase64, + }; +} + +export async function removeArchiveEntries(opts: { + savedObjectsClient: SavedObjectsClientContract; + refs: PackageAssetReference[]; +}) { + const { savedObjectsClient, refs } = opts; + const results = await Promise.all( + refs.map((ref) => savedObjectsClient.delete(ASSETS_SAVED_OBJECT_TYPE, ref.id)) + ); + return results; +} + +export async function saveArchiveEntries(opts: { + savedObjectsClient: SavedObjectsClientContract; + paths: string[]; + packageInfo: InstallablePackage; + installSource: InstallSource; +}) { + const { savedObjectsClient, paths, packageInfo, installSource } = opts; + const bulkBody = await Promise.all( + paths.map((path) => { + const buffer = getArchiveEntry(path); + if (!buffer) throw new Error(`Could not find ArchiveEntry at ${path}`); + const { name, version } = packageInfo; + return archiveEntryToBulkCreateObject({ path, buffer, name, version, installSource }); + }) + ); + + const results = await savedObjectsClient.bulkCreate(bulkBody); + return results; +} + +export async function archiveEntryToBulkCreateObject(opts: { + path: string; + buffer: Buffer; + name: string; + version: string; + installSource: InstallSource; +}): Promise> { + const { path, buffer, name, version, installSource } = opts; + const doc = await archiveEntryToESDocument({ path, buffer, name, version, installSource }); + return { + id: uuidv5(doc.asset_path, ID_NAMESPACE), + type: ASSETS_SAVED_OBJECT_TYPE, + attributes: doc, + }; +} diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index 1af7ce149dfc..7b84ecc259a5 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -5,7 +5,13 @@ */ import { SavedObject, SavedObjectsClientContract } from 'src/core/server'; -import { InstallablePackage, InstallSource, MAX_TIME_COMPLETE_INSTALL } from '../../../../common'; +import { + InstallablePackage, + InstallSource, + PackageAssetReference, + MAX_TIME_COMPLETE_INSTALL, + ASSETS_SAVED_OBJECT_TYPE, +} from '../../../../common'; import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants'; import { AssetReference, @@ -23,6 +29,7 @@ import { updateCurrentWriteIndices } from '../elasticsearch/template/template'; import { deleteKibanaSavedObjectsAssets } from './remove'; import { installTransform } from '../elasticsearch/transform/install'; import { createInstallation, saveKibanaAssetsRefs, updateVersion } from './install'; +import { saveArchiveEntries } from '../archive/save_to_es'; // this is only exported for testing // use a leading underscore to indicate it's not the supported path @@ -177,12 +184,28 @@ export async function _installPackage({ if (installKibanaAssetsError) throw installKibanaAssetsError; await Promise.all([installKibanaAssetsPromise, installIndexPatternPromise]); + const packageAssetResults = await saveArchiveEntries({ + savedObjectsClient, + paths, + packageInfo, + installSource, + }); + const packageAssetRefs: PackageAssetReference[] = packageAssetResults.saved_objects.map( + (result) => ({ + id: result.id, + type: ASSETS_SAVED_OBJECT_TYPE, + }) + ); + // update to newly installed version when all assets are successfully installed if (installedPkg) await updateVersion(savedObjectsClient, pkgName, pkgVersion); + await savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, { install_version: pkgVersion, install_status: 'installed', + package_assets: packageAssetRefs, }); + return [ ...installedKibanaAssetsRefs, ...installedPipelines, diff --git a/x-pack/plugins/fleet/server/services/epm/packages/ensure_installed_default_packages.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/ensure_installed_default_packages.test.ts index 4ad6fc96218d..fe7b8be23b03 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/ensure_installed_default_packages.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/ensure_installed_default_packages.test.ts @@ -43,6 +43,7 @@ const mockInstallation: SavedObject = { id: 'test-pkg', installed_kibana: [{ type: KibanaSavedObjectType.dashboard, id: 'dashboard-1' }], installed_es: [{ type: ElasticsearchAssetType.ingestPipeline, id: 'pipeline' }], + package_assets: [], es_index_patterns: { pattern: 'pattern-name' }, name: 'test package', version: '1.0.0', diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get_install_type.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/get_install_type.test.ts index a41511260c6e..2dcfc7949d5e 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get_install_type.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get_install_type.test.ts @@ -15,6 +15,7 @@ const mockInstallation: SavedObject = { id: 'test-pkg', installed_kibana: [{ type: KibanaSavedObjectType.dashboard, id: 'dashboard-1' }], installed_es: [{ type: ElasticsearchAssetType.ingestPipeline, id: 'pipeline' }], + package_assets: [], es_index_patterns: { pattern: 'pattern-name' }, name: 'test packagek', version: '1.0.0', @@ -32,6 +33,7 @@ const mockInstallationUpdateFail: SavedObject = { id: 'test-pkg', installed_kibana: [{ type: KibanaSavedObjectType.dashboard, id: 'dashboard-1' }], installed_es: [{ type: ElasticsearchAssetType.ingestPipeline, id: 'pipeline' }], + package_assets: [], es_index_patterns: { pattern: 'pattern-name' }, name: 'test packagek', version: '1.0.0', diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.ts index 29300818288b..d641c4945e68 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.ts @@ -379,6 +379,7 @@ export async function createInstallation(options: { { installed_kibana: [], installed_es: [], + package_assets: [], es_index_patterns: toSaveESIndexPatterns, name: pkgName, version: pkgVersion, diff --git a/x-pack/plugins/fleet/server/services/epm/packages/remove.ts b/x-pack/plugins/fleet/server/services/epm/packages/remove.ts index 2e879be20c18..6e0d574d311c 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/remove.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/remove.ts @@ -23,6 +23,7 @@ import { deleteTransforms } from '../elasticsearch/transform/remove'; import { packagePolicyService, appContextService } from '../..'; import { splitPkgKey } from '../registry'; import { deletePackageCache } from '../archive'; +import { removeArchiveEntries } from '../archive/save_to_es'; export async function removeInstallation(options: { savedObjectsClient: SavedObjectsClientContract; @@ -48,7 +49,7 @@ export async function removeInstallation(options: { `unable to remove package with existing package policy(s) in use by agent(s)` ); - // Delete the installed assets + // Delete the installed assets. Don't include installation.package_assets. Those are irrelevant to users const installedAssets = [...installation.installed_kibana, ...installation.installed_es]; await deleteAssets(installation, savedObjectsClient, callCluster); @@ -68,6 +69,8 @@ export async function removeInstallation(options: { version: pkgVersion, }); + await removeArchiveEntries({ savedObjectsClient, refs: installation.package_assets }); + // successful delete's in SO client return {}. return something more useful return installedAssets; } diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts index 3c508bed5b2f..c6d081da696c 100644 --- a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts @@ -1336,6 +1336,7 @@ export class EndpointDocGenerator { { id: 'logs-endpoint.events.security', type: 'index_template' }, { id: 'metrics-endpoint.telemetry', type: 'index_template' }, ] as EsAssetReference[], + package_assets: [], es_index_patterns: { alerts: 'logs-endpoint.alerts-*', events: 'events-endpoint-*', diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts b/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts index 2ae4273bfa7e..a7d46b9c6677 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts @@ -433,6 +433,7 @@ const expectAssetsInstalled = ({ ...res.attributes, installed_kibana: sortBy(res.attributes.installed_kibana, (o: AssetReference) => o.type), installed_es: sortBy(res.attributes.installed_es, (o: AssetReference) => o.type), + package_assets: sortBy(res.attributes.package_assets, (o: AssetReference) => o.type), }; expect(sortedRes).eql({ installed_kibana: [ @@ -487,6 +488,28 @@ const expectAssetsInstalled = ({ test_logs: 'logs-all_assets.test_logs-*', test_metrics: 'metrics-all_assets.test_metrics-*', }, + package_assets: [ + { id: '333a22a1-e639-5af5-ae62-907ffc83d603', type: 'epm-packages-assets' }, + { id: '256f3dad-6870-56c3-80a1-8dfa11e2d568', type: 'epm-packages-assets' }, + { id: '3fa0512f-bc01-5c2e-9df1-bc2f2a8259c8', type: 'epm-packages-assets' }, + { id: 'ea334ad8-80c2-5acd-934b-2a377290bf97', type: 'epm-packages-assets' }, + { id: '96c6eb85-fe2e-56c6-84be-5fda976796db', type: 'epm-packages-assets' }, + { id: '2d73a161-fa69-52d0-aa09-1bdc691b95bb', type: 'epm-packages-assets' }, + { id: '0a00c2d2-ce63-5b9c-9aa0-0cf1938f7362', type: 'epm-packages-assets' }, + { id: 'b36e6dd0-58f7-5dd0-a286-8187e4019274', type: 'epm-packages-assets' }, + { id: 'f839c76e-d194-555a-90a1-3265a45789e4', type: 'epm-packages-assets' }, + { id: '9af7bbb3-7d8a-50fa-acc9-9dde6f5efca2', type: 'epm-packages-assets' }, + { id: '1e97a20f-9d1c-529b-8ff2-da4e8ba8bb71', type: 'epm-packages-assets' }, + { id: '8cfe0a2b-7016-5522-87e4-6d352360d1fc', type: 'epm-packages-assets' }, + { id: 'bd5ff3c5-655e-5385-9918-b60ff3040aad', type: 'epm-packages-assets' }, + { id: '0954ce3b-3165-5c1f-a4c0-56eb5f2fa487', type: 'epm-packages-assets' }, + { id: '60d6d054-57e4-590f-a580-52bf3f5e7cca', type: 'epm-packages-assets' }, + { id: '47758dc2-979d-5fbe-a2bd-9eded68a5a43', type: 'epm-packages-assets' }, + { id: '318959c9-997b-5a14-b328-9fc7355b4b74', type: 'epm-packages-assets' }, + { id: 'e786cbd9-0f3b-5a0b-82a6-db25145ebf58', type: 'epm-packages-assets' }, + { id: '53c94591-aa33-591d-8200-cd524c2a0561', type: 'epm-packages-assets' }, + { id: 'b658d2d4-752e-54b8-afc2-4c76155c1466', type: 'epm-packages-assets' }, + ], name: 'all_assets', version: '0.1.0', internal: false, diff --git a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts index b16cf039f0da..37aa94beec8b 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts @@ -318,6 +318,26 @@ export default function (providerContext: FtrProviderContext) { test_logs: 'logs-all_assets.test_logs-*', test_metrics: 'metrics-all_assets.test_metrics-*', }, + package_assets: [ + { id: '3eb4c54a-638f-51b6-84e2-d53f5a666e37', type: 'epm-packages-assets' }, + { id: '4acfbf69-7a27-5c58-9c99-7c86843d958f', type: 'epm-packages-assets' }, + { id: '938655df-b339-523c-a9e4-123c89c0e1e1', type: 'epm-packages-assets' }, + { id: 'eec4606c-dbfa-565b-8e9c-fce1e641f3fc', type: 'epm-packages-assets' }, + { id: 'ef67e7e0-dca3-5a62-a42a-745db5ad7c1f', type: 'epm-packages-assets' }, + { id: '64239d25-be40-5e10-94b5-f6b74b8c5474', type: 'epm-packages-assets' }, + { id: '071b5113-4c9f-5ee9-aafe-d098a4c066f6', type: 'epm-packages-assets' }, + { id: '498d8215-2613-5399-9a13-fa4f0bf513e2', type: 'epm-packages-assets' }, + { id: 'd2f87071-c866-503a-8fcb-7b23a8c7afbf', type: 'epm-packages-assets' }, + { id: '5a080eba-f482-545c-8695-6ccbd426b2a2', type: 'epm-packages-assets' }, + { id: '28523a82-1328-578d-84cb-800970560200', type: 'epm-packages-assets' }, + { id: 'cc1e3e1d-f27b-5d05-86f6-6e4b9a47c7dc', type: 'epm-packages-assets' }, + { id: '5c3aa147-089c-5084-beca-53c00e72ac80', type: 'epm-packages-assets' }, + { id: '48e582df-b1d2-5f88-b6ea-ba1fafd3a569', type: 'epm-packages-assets' }, + { id: 'bf3b0b65-9fdc-53c6-a9ca-e76140e56490', type: 'epm-packages-assets' }, + { id: '2e56f08b-1d06-55ed-abee-4708e1ccf0aa', type: 'epm-packages-assets' }, + { id: 'c7bf1a39-e057-58a0-afde-fb4b48751d8c', type: 'epm-packages-assets' }, + { id: '8c665f28-a439-5f43-b5fd-8fda7b576735', type: 'epm-packages-assets' }, + ], name: 'all_assets', version: '0.2.0', internal: false,