[Fleet][EPM] Save installed package assets in ES #83391
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -12,6 +12,7 @@ import { | |
| AGENT_POLICY_SAVED_OBJECT_TYPE, | ||
| PACKAGE_POLICY_SAVED_OBJECT_TYPE, | ||
| PACKAGES_SAVED_OBJECT_TYPE, | ||
| ASSETS_SAVED_OBJECT_TYPE, | ||
| AGENT_SAVED_OBJECT_TYPE, | ||
| AGENT_EVENT_SAVED_OBJECT_TYPE, | ||
| AGENT_ACTION_SAVED_OBJECT_TYPE, | ||
|
|
@@ -304,13 +305,39 @@ const getSavedObjectTypes = ( | |
| type: { type: 'keyword' }, | ||
| }, | ||
| }, | ||
| package_assets: { | ||
| type: 'nested', | ||
| properties: { | ||
| id: { type: 'keyword' }, | ||
| type: { type: 'keyword' }, | ||
| }, | ||
| }, | ||
| install_started_at: { type: 'date' }, | ||
| install_version: { type: 'keyword' }, | ||
| install_status: { type: 'keyword' }, | ||
| install_source: { type: 'keyword' }, | ||
| }, | ||
| }, | ||
| }, | ||
| [ASSETS_SAVED_OBJECT_TYPE]: { | ||
| name: ASSETS_SAVED_OBJECT_TYPE, | ||
| hidden: false, | ||
| namespaceType: 'agnostic', | ||
| management: { | ||
| importableAndExportable: false, | ||
| }, | ||
| mappings: { | ||
| properties: { | ||
| package_name: { type: 'keyword' }, | ||
| package_version: { type: 'keyword' }, | ||
| install_source: { type: 'keyword' }, | ||
| asset_path: { type: 'keyword' }, | ||
| media_type: { type: 'keyword' }, | ||
|
There was a problem hiding this comment. Also considered |
||
| data_utf8: { type: 'text', index: false }, | ||
| data_base64: { type: 'binary' }, | ||
|
There was a problem hiding this comment. Went with two fields because I thought we wanted |
||
| }, | ||
| }, | ||
| }, | ||
| }); | ||
|
|
||
| export function registerSavedObjects( | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,121 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the Elastic License; | ||
| * you may not use this file except in compliance with the Elastic License. | ||
| */ | ||
|
|
||
| import { extname } from 'path'; | ||
| import { isBinaryFile } from 'isbinaryfile'; | ||
| import mime from 'mime-types'; | ||
| import uuidv5 from 'uuid/v5'; | ||
| import { SavedObjectsClientContract, SavedObjectsBulkCreateObject } from 'src/core/server'; | ||
| import { | ||
| ASSETS_SAVED_OBJECT_TYPE, | ||
| InstallablePackage, | ||
| InstallSource, | ||
| PackageAssetReference, | ||
| } from '../../../../common'; | ||
| import { getArchiveEntry } from './index'; | ||
|
|
||
| // uuid v5 requires a SHA-1 UUID as a namespace | ||
| // used to ensure same input produces the same id | ||
| const ID_NAMESPACE = '71403015-cdd5-404b-a5da-6c43f35cad84'; | ||
|
|
||
| // could be anything, picked this from https://github.com/elastic/elastic-agent-client/issues/17 | ||
| const MAX_ES_ASSET_BYTES = 4 * 1024 * 1024; | ||
|
|
||
| export interface PackageAsset { | ||
| package_name: string; | ||
| package_version: string; | ||
| install_source: string; | ||
| asset_path: string; | ||
| media_type: string; | ||
| data_utf8: string; | ||
| data_base64: string; | ||
| } | ||
|
|
||
| export async function archiveEntryToESDocument(opts: { | ||
| path: string; | ||
| buffer: Buffer; | ||
| name: string; | ||
| version: string; | ||
| installSource: InstallSource; | ||
| }): Promise<PackageAsset> { | ||
| const { path, buffer, name, version, installSource } = opts; | ||
| const fileExt = extname(path); | ||
| const contentType = mime.lookup(fileExt); | ||
| const mediaType = mime.contentType(contentType || fileExt); | ||
| // can use to create a data URL like `data:${mediaType};base64,${base64Data}` | ||
|
|
||
| const bufferIsBinary = await isBinaryFile(buffer); | ||
| const dataUtf8 = bufferIsBinary ? '' : buffer.toString('utf8'); | ||
| const dataBase64 = bufferIsBinary ? buffer.toString('base64') : ''; | ||
|
|
||
| // validation: filesize? asset type? anything else | ||
| if (dataUtf8.length > MAX_ES_ASSET_BYTES) { | ||
| throw new Error(`File at ${path} is larger than maximum allowed size of ${MAX_ES_ASSET_BYTES}`); | ||
| } | ||
|
|
||
| if (dataBase64.length > MAX_ES_ASSET_BYTES) { | ||
| throw new Error( | ||
| `After base64 encoding file at ${path} is larger than maximum allowed size of ${MAX_ES_ASSET_BYTES}` | ||
| ); | ||
| } | ||
|
|
||
| return { | ||
| package_name: name, | ||
| package_version: version, | ||
| install_source: installSource, | ||
| asset_path: path, | ||
| media_type: mediaType || '', | ||
| data_utf8: dataUtf8, | ||
| data_base64: dataBase64, | ||
| }; | ||
| } | ||
|
|
||
| export async function removeArchiveEntries(opts: { | ||
| savedObjectsClient: SavedObjectsClientContract; | ||
| refs: PackageAssetReference[]; | ||
| }) { | ||
| const { savedObjectsClient, refs } = opts; | ||
| const results = await Promise.all( | ||
| refs.map((ref) => savedObjectsClient.delete(ASSETS_SAVED_OBJECT_TYPE, ref.id)) | ||
| ); | ||
| return results; | ||
| } | ||
|
|
||
| export async function saveArchiveEntries(opts: { | ||
| savedObjectsClient: SavedObjectsClientContract; | ||
| paths: string[]; | ||
| packageInfo: InstallablePackage; | ||
| installSource: InstallSource; | ||
| }) { | ||
| const { savedObjectsClient, paths, packageInfo, installSource } = opts; | ||
| const bulkBody = await Promise.all( | ||
| paths.map((path) => { | ||
| const buffer = getArchiveEntry(path); | ||
| if (!buffer) throw new Error(`Could not find ArchiveEntry at ${path}`); | ||
| const { name, version } = packageInfo; | ||
| return archiveEntryToBulkCreateObject({ path, buffer, name, version, installSource }); | ||
| }) | ||
| ); | ||
|
|
||
| const results = await savedObjectsClient.bulkCreate<PackageAsset>(bulkBody); | ||
| return results; | ||
| } | ||
|
|
||
| export async function archiveEntryToBulkCreateObject(opts: { | ||
| path: string; | ||
| buffer: Buffer; | ||
| name: string; | ||
| version: string; | ||
| installSource: InstallSource; | ||
| }): Promise<SavedObjectsBulkCreateObject<PackageAsset>> { | ||
| const { path, buffer, name, version, installSource } = opts; | ||
| const doc = await archiveEntryToESDocument({ path, buffer, name, version, installSource }); | ||
| return { | ||
| id: uuidv5(doc.asset_path, ID_NAMESPACE), | ||
| type: ASSETS_SAVED_OBJECT_TYPE, | ||
| attributes: doc, | ||
| }; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Do you need to add this new type to
allSavedObjectTypesso that your users are granted access to these new saved objects? It's probably not strictly necessary since you still requiresuperuserprivileges for Fleet, and superusers can do everything regardless of what you specify herekibana/x-pack/plugins/fleet/server/plugin.ts
Lines 104 to 112 in aa07f5c