diff --git a/x-pack/plugins/fleet/common/constants/epm.ts b/x-pack/plugins/fleet/common/constants/epm.ts index 297b15790b528..287b7ccdb88e0 100644 --- a/x-pack/plugins/fleet/common/constants/epm.ts +++ b/x-pack/plugins/fleet/common/constants/epm.ts @@ -3,8 +3,8 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ + export const PACKAGES_SAVED_OBJECT_TYPE = 'epm-packages'; -export const ASSETS_SAVED_OBJECT_TYPE = 'epm-packages-assets'; export const INDEX_PATTERN_SAVED_OBJECT_TYPE = 'index-pattern'; export const INDEX_PATTERN_PLACEHOLDER_SUFFIX = '-index_pattern_placeholder'; export const MAX_TIME_COMPLETE_INSTALL = 60000; diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index 77625e48dbc96..a9893b170492f 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -8,7 +8,6 @@ // TODO: Update when https://github.com/elastic/kibana/issues/53021 is closed import { SavedObject, SavedObjectAttributes, SavedObjectReference } from 'src/core/public'; import { - ASSETS_SAVED_OBJECT_TYPE, agentAssetTypes, dataTypes, defaultPackages, @@ -269,7 +268,6 @@ export type PackageInfo = export interface Installation extends SavedObjectAttributes { installed_kibana: KibanaAssetReference[]; installed_es: EsAssetReference[]; - package_assets: PackageAssetReference[]; es_index_patterns: Record; name: string; version: string; @@ -299,10 +297,6 @@ export type EsAssetReference = Pick & { type: ElasticsearchAssetType; }; -export type PackageAssetReference = Pick & { - type: typeof ASSETS_SAVED_OBJECT_TYPE; -}; - export type RequiredPackage = typeof requiredPackages; export type DefaultPackages = typeof defaultPackages; diff --git a/x-pack/plugins/fleet/server/constants/index.ts b/x-pack/plugins/fleet/server/constants/index.ts index f380608e6817e..b1d7318ff5107 100644 --- a/x-pack/plugins/fleet/server/constants/index.ts +++ b/x-pack/plugins/fleet/server/constants/index.ts @@ -41,7 +41,6 @@ export { PACKAGE_POLICY_SAVED_OBJECT_TYPE, OUTPUT_SAVED_OBJECT_TYPE, PACKAGES_SAVED_OBJECT_TYPE, - ASSETS_SAVED_OBJECT_TYPE, INDEX_PATTERN_SAVED_OBJECT_TYPE, ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE, GLOBAL_SETTINGS_SAVED_OBJECT_TYPE, diff --git a/x-pack/plugins/fleet/server/routes/epm/handlers.ts b/x-pack/plugins/fleet/server/routes/epm/handlers.ts index 9ccf60dc80a5f..cfcde99541f13 100644 --- a/x-pack/plugins/fleet/server/routes/epm/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/epm/handlers.ts @@ -41,13 +41,12 @@ import { removeInstallation, getLimitedPackages, getInstallationObject, - getInstallation, } from '../../services/epm/packages'; import { defaultIngestErrorHandler, ingestErrorToResponseOptions } from '../../errors'; import { splitPkgKey } from '../../services/epm/registry'; import { licenseService } from '../../services'; import { getArchiveEntry } from '../../services/epm/archive/cache'; -import { getAsset } from '../../services/epm/archive/storage'; +import { bufferToStream } from '../../services/epm/streams'; export const getCategoriesHandler: RequestHandler< undefined, @@ -108,51 +107,32 @@ export const getFileHandler: RequestHandler { - const { path, buffer, name, version, installSource } = opts; - const fileExt = extname(path); - const contentType = mime.lookup(fileExt); - const mediaType = mime.contentType(contentType || fileExt); - // can use to create a data URL like `data:${mediaType};base64,${base64Data}` - - const bufferIsBinary = await isBinaryFile(buffer); - const dataUtf8 = bufferIsBinary ? '' : buffer.toString('utf8'); - const dataBase64 = bufferIsBinary ? buffer.toString('base64') : ''; - - // validation: filesize? asset type? anything else - if (dataUtf8.length > MAX_ES_ASSET_BYTES) { - throw new Error(`File at ${path} is larger than maximum allowed size of ${MAX_ES_ASSET_BYTES}`); - } - - if (dataBase64.length > MAX_ES_ASSET_BYTES) { - throw new Error( - `After base64 encoding file at ${path} is larger than maximum allowed size of ${MAX_ES_ASSET_BYTES}` - ); - } - - return { - package_name: name, - package_version: version, - install_source: installSource, - asset_path: path, - media_type: mediaType || '', - data_utf8: dataUtf8, - data_base64: dataBase64, - }; -} - -export async function removeArchiveEntries(opts: { - savedObjectsClient: SavedObjectsClientContract; - refs: PackageAssetReference[]; -}) { - const { savedObjectsClient, refs } = opts; - const results = await Promise.all( - refs.map((ref) => savedObjectsClient.delete(ASSETS_SAVED_OBJECT_TYPE, ref.id)) - ); - return results; -} - -export async function saveArchiveEntries(opts: { - savedObjectsClient: SavedObjectsClientContract; - paths: string[]; - packageInfo: InstallablePackage; - installSource: InstallSource; -}) { - const { savedObjectsClient, paths, packageInfo, installSource } = opts; - const bulkBody = await Promise.all( - paths.map((path) => { - const buffer = getArchiveEntry(path); - if (!buffer) throw new Error(`Could not find ArchiveEntry at ${path}`); - const { name, version } = packageInfo; - return archiveEntryToBulkCreateObject({ path, buffer, name, version, installSource }); - }) - ); - - const results = await savedObjectsClient.bulkCreate(bulkBody); - return results; -} - -export async function archiveEntryToBulkCreateObject(opts: { - path: string; - buffer: Buffer; - name: string; - version: string; - installSource: InstallSource; -}): Promise> { - const { path, buffer, name, version, installSource } = opts; - const doc = await archiveEntryToESDocument({ path, buffer, name, version, installSource }); - return { - id: assetPathToObjectId(doc.asset_path), - type: ASSETS_SAVED_OBJECT_TYPE, - attributes: doc, - }; -} - -export async function getAsset(opts: { - savedObjectsClient: SavedObjectsClientContract; - path: string; -}) { - const { savedObjectsClient, path } = opts; - const assetSavedObject = await savedObjectsClient.get( - ASSETS_SAVED_OBJECT_TYPE, - assetPathToObjectId(path) - ); - const storedAsset = assetSavedObject?.attributes; - if (!storedAsset) { - return; - } - - return storedAsset; -} diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index c0e2fcb12bcf8..c7500a9cfeaf6 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -5,13 +5,7 @@ */ import { SavedObject, SavedObjectsClientContract } from 'src/core/server'; -import { - InstallablePackage, - InstallSource, - PackageAssetReference, - MAX_TIME_COMPLETE_INSTALL, - ASSETS_SAVED_OBJECT_TYPE, -} from '../../../../common'; +import { InstallablePackage, InstallSource, MAX_TIME_COMPLETE_INSTALL } from '../../../../common'; import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants'; import { AssetReference, @@ -30,7 +24,6 @@ import { deleteKibanaSavedObjectsAssets } from './remove'; import { installTransform } from '../elasticsearch/transform/install'; import { createInstallation, saveKibanaAssetsRefs, updateVersion } from './install'; import { installIlmForDataStream } from '../elasticsearch/datastream_ilm/install'; -import { saveArchiveEntries } from '../archive/storage'; import { ConcurrentInstallOperationError } from '../../../errors'; // this is only exported for testing @@ -195,26 +188,11 @@ export async function _installPackage({ if (installKibanaAssetsError) throw installKibanaAssetsError; await Promise.all([installKibanaAssetsPromise, installIndexPatternPromise]); - const packageAssetResults = await saveArchiveEntries({ - savedObjectsClient, - paths, - packageInfo, - installSource, - }); - const packageAssetRefs: PackageAssetReference[] = packageAssetResults.saved_objects.map( - (result) => ({ - id: result.id, - type: ASSETS_SAVED_OBJECT_TYPE, - }) - ); - // update to newly installed version when all assets are successfully installed if (installedPkg) await updateVersion(savedObjectsClient, pkgName, pkgVersion); - await savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, { install_version: pkgVersion, install_status: 'installed', - package_assets: packageAssetRefs, }); return [ diff --git a/x-pack/plugins/fleet/server/services/epm/packages/ensure_installed_default_packages.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/ensure_installed_default_packages.test.ts index fe7b8be23b03b..4ad6fc96218de 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/ensure_installed_default_packages.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/ensure_installed_default_packages.test.ts @@ -43,7 +43,6 @@ const mockInstallation: SavedObject = { id: 'test-pkg', installed_kibana: [{ type: KibanaSavedObjectType.dashboard, id: 'dashboard-1' }], installed_es: [{ type: ElasticsearchAssetType.ingestPipeline, id: 'pipeline' }], - package_assets: [], es_index_patterns: { pattern: 'pattern-name' }, name: 'test package', version: '1.0.0', diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get_install_type.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/get_install_type.test.ts index 2dcfc7949d5e5..a41511260c6e7 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get_install_type.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get_install_type.test.ts @@ -15,7 +15,6 @@ const mockInstallation: SavedObject = { id: 'test-pkg', installed_kibana: [{ type: KibanaSavedObjectType.dashboard, id: 'dashboard-1' }], installed_es: [{ type: ElasticsearchAssetType.ingestPipeline, id: 'pipeline' }], - package_assets: [], es_index_patterns: { pattern: 'pattern-name' }, name: 'test packagek', version: '1.0.0', @@ -33,7 +32,6 @@ const mockInstallationUpdateFail: SavedObject = { id: 'test-pkg', installed_kibana: [{ type: KibanaSavedObjectType.dashboard, id: 'dashboard-1' }], installed_es: [{ type: ElasticsearchAssetType.ingestPipeline, id: 'pipeline' }], - package_assets: [], es_index_patterns: { pattern: 'pattern-name' }, name: 'test packagek', version: '1.0.0', diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.ts index d641c4945e681..29300818288b4 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.ts @@ -379,7 +379,6 @@ export async function createInstallation(options: { { installed_kibana: [], installed_es: [], - package_assets: [], es_index_patterns: toSaveESIndexPatterns, name: pkgName, version: pkgVersion, diff --git a/x-pack/plugins/fleet/server/services/epm/packages/remove.ts b/x-pack/plugins/fleet/server/services/epm/packages/remove.ts index 331b6bfa882da..0b4a0faddf0cc 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/remove.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/remove.ts @@ -24,7 +24,6 @@ import { packagePolicyService, appContextService } from '../..'; import { splitPkgKey } from '../registry'; import { deletePackageCache } from '../archive'; import { deleteIlms } from '../elasticsearch/datastream_ilm/remove'; -import { removeArchiveEntries } from '../archive/storage'; export async function removeInstallation(options: { savedObjectsClient: SavedObjectsClientContract; @@ -50,7 +49,7 @@ export async function removeInstallation(options: { `unable to remove package with existing package policy(s) in use by agent(s)` ); - // Delete the installed assets. Don't include installation.package_assets. Those are irrelevant to users + // Delete the installed assets const installedAssets = [...installation.installed_kibana, ...installation.installed_es]; await deleteAssets(installation, savedObjectsClient, callCluster); @@ -70,8 +69,6 @@ export async function removeInstallation(options: { version: pkgVersion, }); - await removeArchiveEntries({ savedObjectsClient, refs: installation.package_assets }); - // successful delete's in SO client return {}. return something more useful return installedAssets; } diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts index 61c3d3ab192b6..1f9f1a2c6d098 100644 --- a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts @@ -1329,7 +1329,6 @@ export class EndpointDocGenerator { { id: 'logs-endpoint.events.security', type: 'index_template' }, { id: 'metrics-endpoint.telemetry', type: 'index_template' }, ] as EsAssetReference[], - package_assets: [], es_index_patterns: { alerts: 'logs-endpoint.alerts-*', events: 'events-endpoint-*', diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts b/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts index 1d5f864c27eea..9dcd3ea100589 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts @@ -433,7 +433,6 @@ const expectAssetsInstalled = ({ ...res.attributes, installed_kibana: sortBy(res.attributes.installed_kibana, (o: AssetReference) => o.type), installed_es: sortBy(res.attributes.installed_es, (o: AssetReference) => o.type), - package_assets: sortBy(res.attributes.package_assets, (o: AssetReference) => o.type), }; expect(sortedRes).eql({ installed_kibana: [ @@ -496,29 +495,6 @@ const expectAssetsInstalled = ({ test_logs: 'logs-all_assets.test_logs-*', test_metrics: 'metrics-all_assets.test_metrics-*', }, - package_assets: [ - { id: '333a22a1-e639-5af5-ae62-907ffc83d603', type: 'epm-packages-assets' }, - { id: '256f3dad-6870-56c3-80a1-8dfa11e2d568', type: 'epm-packages-assets' }, - { id: '3fa0512f-bc01-5c2e-9df1-bc2f2a8259c8', type: 'epm-packages-assets' }, - { id: 'ea334ad8-80c2-5acd-934b-2a377290bf97', type: 'epm-packages-assets' }, - { id: '96c6eb85-fe2e-56c6-84be-5fda976796db', type: 'epm-packages-assets' }, - { id: '2d73a161-fa69-52d0-aa09-1bdc691b95bb', type: 'epm-packages-assets' }, - { id: '0a00c2d2-ce63-5b9c-9aa0-0cf1938f7362', type: 'epm-packages-assets' }, - { id: '691f0505-18c5-57a6-9f40-06e8affbdf7a', type: 'epm-packages-assets' }, - { id: 'b36e6dd0-58f7-5dd0-a286-8187e4019274', type: 'epm-packages-assets' }, - { id: 'f839c76e-d194-555a-90a1-3265a45789e4', type: 'epm-packages-assets' }, - { id: '9af7bbb3-7d8a-50fa-acc9-9dde6f5efca2', type: 'epm-packages-assets' }, - { id: '1e97a20f-9d1c-529b-8ff2-da4e8ba8bb71', type: 'epm-packages-assets' }, - { id: '8cfe0a2b-7016-5522-87e4-6d352360d1fc', type: 'epm-packages-assets' }, - { id: 'bd5ff3c5-655e-5385-9918-b60ff3040aad', type: 'epm-packages-assets' }, - { id: '0954ce3b-3165-5c1f-a4c0-56eb5f2fa487', type: 'epm-packages-assets' }, - { id: '60d6d054-57e4-590f-a580-52bf3f5e7cca', type: 'epm-packages-assets' }, - { id: '47758dc2-979d-5fbe-a2bd-9eded68a5a43', type: 'epm-packages-assets' }, - { id: '318959c9-997b-5a14-b328-9fc7355b4b74', type: 'epm-packages-assets' }, - { id: 'e786cbd9-0f3b-5a0b-82a6-db25145ebf58', type: 'epm-packages-assets' }, - { id: '53c94591-aa33-591d-8200-cd524c2a0561', type: 'epm-packages-assets' }, - { id: 'b658d2d4-752e-54b8-afc2-4c76155c1466', type: 'epm-packages-assets' }, - ], name: 'all_assets', version: '0.1.0', internal: false, diff --git a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts index 7b264f949532e..d72209edf6501 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts @@ -322,26 +322,6 @@ export default function (providerContext: FtrProviderContext) { test_logs: 'logs-all_assets.test_logs-*', test_metrics: 'metrics-all_assets.test_metrics-*', }, - package_assets: [ - { id: '3eb4c54a-638f-51b6-84e2-d53f5a666e37', type: 'epm-packages-assets' }, - { id: '4acfbf69-7a27-5c58-9c99-7c86843d958f', type: 'epm-packages-assets' }, - { id: '938655df-b339-523c-a9e4-123c89c0e1e1', type: 'epm-packages-assets' }, - { id: 'eec4606c-dbfa-565b-8e9c-fce1e641f3fc', type: 'epm-packages-assets' }, - { id: 'ef67e7e0-dca3-5a62-a42a-745db5ad7c1f', type: 'epm-packages-assets' }, - { id: '64239d25-be40-5e10-94b5-f6b74b8c5474', type: 'epm-packages-assets' }, - { id: '071b5113-4c9f-5ee9-aafe-d098a4c066f6', type: 'epm-packages-assets' }, - { id: '498d8215-2613-5399-9a13-fa4f0bf513e2', type: 'epm-packages-assets' }, - { id: 'd2f87071-c866-503a-8fcb-7b23a8c7afbf', type: 'epm-packages-assets' }, - { id: '5a080eba-f482-545c-8695-6ccbd426b2a2', type: 'epm-packages-assets' }, - { id: '28523a82-1328-578d-84cb-800970560200', type: 'epm-packages-assets' }, - { id: 'cc1e3e1d-f27b-5d05-86f6-6e4b9a47c7dc', type: 'epm-packages-assets' }, - { id: '5c3aa147-089c-5084-beca-53c00e72ac80', type: 'epm-packages-assets' }, - { id: '48e582df-b1d2-5f88-b6ea-ba1fafd3a569', type: 'epm-packages-assets' }, - { id: 'bf3b0b65-9fdc-53c6-a9ca-e76140e56490', type: 'epm-packages-assets' }, - { id: '2e56f08b-1d06-55ed-abee-4708e1ccf0aa', type: 'epm-packages-assets' }, - { id: 'c7bf1a39-e057-58a0-afde-fb4b48751d8c', type: 'epm-packages-assets' }, - { id: '8c665f28-a439-5f43-b5fd-8fda7b576735', type: 'epm-packages-assets' }, - ], name: 'all_assets', version: '0.2.0', internal: false,