[Fleet][EPM] Restore es-storage-related commits

x-pack/plugins/fleet/common/constants/epm.ts

@@ -3,8 +3,8 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-
 export const PACKAGES_SAVED_OBJECT_TYPE = 'epm-packages';
+export const ASSETS_SAVED_OBJECT_TYPE = 'epm-packages-assets';
 export const INDEX_PATTERN_SAVED_OBJECT_TYPE = 'index-pattern';
 export const MAX_TIME_COMPLETE_INSTALL = 60000;
 

x-pack/plugins/fleet/common/types/models/epm.ts

@@ -8,6 +8,7 @@
 // TODO: Update when https://github.com/elastic/kibana/issues/53021 is closed
 import { SavedObject, SavedObjectAttributes, SavedObjectReference } from 'src/core/public';
 import {
+  ASSETS_SAVED_OBJECT_TYPE,
   agentAssetTypes,
   dataTypes,
   defaultPackages,
@@ -268,6 +269,7 @@ export type PackageInfo =
 export interface Installation extends SavedObjectAttributes {
   installed_kibana: KibanaAssetReference[];
   installed_es: EsAssetReference[];
+  package_assets: PackageAssetReference[];
   es_index_patterns: Record<string, string>;
   name: string;
   version: string;
@@ -297,6 +299,10 @@ export type EsAssetReference = Pick<SavedObjectReference, 'id'> & {
   type: ElasticsearchAssetType;
 };
 
+export type PackageAssetReference = Pick<SavedObjectReference, 'id'> & {
+  type: typeof ASSETS_SAVED_OBJECT_TYPE;
+};
+
 export type RequiredPackage = typeof requiredPackages;
 
 export type DefaultPackages = typeof defaultPackages;

x-pack/plugins/fleet/server/constants/index.ts

@@ -40,6 +40,7 @@ export {
   PACKAGE_POLICY_SAVED_OBJECT_TYPE,
   OUTPUT_SAVED_OBJECT_TYPE,
   PACKAGES_SAVED_OBJECT_TYPE,
+  ASSETS_SAVED_OBJECT_TYPE,
   INDEX_PATTERN_SAVED_OBJECT_TYPE,
   ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE,
   GLOBAL_SETTINGS_SAVED_OBJECT_TYPE,

x-pack/plugins/fleet/server/routes/epm/handlers.ts

@@ -41,12 +41,13 @@ import {
   removeInstallation,
   getLimitedPackages,
   getInstallationObject,
+  getInstallation,
 } from '../../services/epm/packages';
 import { defaultIngestErrorHandler, ingestErrorToResponseOptions } from '../../errors';
 import { splitPkgKey } from '../../services/epm/registry';
 import { licenseService } from '../../services';
 import { getArchiveEntry } from '../../services/epm/archive/cache';
-import { bufferToStream } from '../../services/epm/streams';
+import { getAsset } from '../../services/epm/archive/storage';
 
 export const getCategoriesHandler: RequestHandler<
   undefined,
@@ -107,32 +108,51 @@ export const getFileHandler: RequestHandler<TypeOf<typeof GetFileRequestSchema.p
   try {
     const { pkgName, pkgVersion, filePath } = request.params;
     const savedObjectsClient = context.core.savedObjects.client;
-    const savedObject = await getInstallationObject({ savedObjectsClient, pkgName });
-    const pkgInstallSource = savedObject?.attributes.install_source;
-    // TODO: when package storage is available, remove installSource check and check cache and storage, remove registry call
-    if (pkgInstallSource === 'upload' && pkgVersion === savedObject?.attributes.version) {
-      const headerContentType = mime.contentType(path.extname(filePath));
-      if (!headerContentType) {
+    const installation = await getInstallation({ savedObjectsClient, pkgName });
+    const useLocalFile = pkgVersion === installation?.version;
+
+    if (useLocalFile) {
+      const assetPath = `${pkgName}-${pkgVersion}/${filePath}`;
+      const fileBuffer = getArchiveEntry(assetPath);
+      // only pull local installation if we don't have it cached
+      const storedAsset = !fileBuffer && (await getAsset({ savedObjectsClient, path: assetPath }));
+
+      // error, if neither is available
+      if (!fileBuffer && !storedAsset) {
         return response.custom({
-          body: `unknown content type for file: ${filePath}`,
-          statusCode: 400,
+          body: `installed package file not found: ${filePath}`,
+          statusCode: 404,
         });
       }
-      const archiveFile = getArchiveEntry(`${pkgName}-${pkgVersion}/${filePath}`);
-      if (!archiveFile) {
+
+      // if storedAsset is not available, fileBuffer *must* be
+      // b/c we error if we don't have at least one, and storedAsset is the least likely
+      const { buffer, contentType } = storedAsset
+        ? {
+            contentType: storedAsset.media_type,
+            buffer: storedAsset.data_utf8
+              ? Buffer.from(storedAsset.data_utf8, 'utf8')
+              : Buffer.from(storedAsset.data_base64, 'base64'),
+          }
+        : {
+            contentType: mime.contentType(path.extname(assetPath)),
+            buffer: fileBuffer,
+          };
+
+      if (!contentType) {
         return response.custom({
-          body: `uploaded package file not found: ${filePath}`,
-          statusCode: 404,
+          body: `unknown content type for file: ${filePath}`,
+          statusCode: 400,
         });
       }
-      const headers: ResponseHeaders = {
-        'cache-control': 'max-age=10, public',
-        'content-type': headerContentType,
-      };
+
       return response.custom({
-        body: bufferToStream(archiveFile),
+        body: buffer,
         statusCode: 200,
-        headers,
+        headers: {
+          'cache-control': 'max-age=10, public',
+          'content-type': contentType,
+        },
       });
     } else {
       const registryResponse = await getFile(pkgName, pkgVersion, filePath);

x-pack/plugins/fleet/server/saved_objects/index.ts

@@ -12,6 +12,7 @@ import {
   AGENT_POLICY_SAVED_OBJECT_TYPE,
   PACKAGE_POLICY_SAVED_OBJECT_TYPE,
   PACKAGES_SAVED_OBJECT_TYPE,
+  ASSETS_SAVED_OBJECT_TYPE,
   AGENT_SAVED_OBJECT_TYPE,
   AGENT_EVENT_SAVED_OBJECT_TYPE,
   AGENT_ACTION_SAVED_OBJECT_TYPE,
@@ -304,13 +305,39 @@ const getSavedObjectTypes = (
             type: { type: 'keyword' },
           },
         },
+        package_assets: {
+          type: 'nested',
+          properties: {
+            id: { type: 'keyword' },
+            type: { type: 'keyword' },
+          },
+        },
         install_started_at: { type: 'date' },
         install_version: { type: 'keyword' },
         install_status: { type: 'keyword' },
         install_source: { type: 'keyword' },
       },
     },
   },
+  [ASSETS_SAVED_OBJECT_TYPE]: {
+    name: ASSETS_SAVED_OBJECT_TYPE,
+    hidden: false,
+    namespaceType: 'agnostic',
+    management: {
+      importableAndExportable: false,
+    },
+    mappings: {
+      properties: {
+        package_name: { type: 'keyword' },
+        package_version: { type: 'keyword' },
+        install_source: { type: 'keyword' },
+        asset_path: { type: 'keyword' },
+        media_type: { type: 'keyword' },
+        data_utf8: { type: 'text', index: false },
+        data_base64: { type: 'binary' },
+      },
+    },
+  },
 });
 
 export function registerSavedObjects(

x-pack/plugins/fleet/server/services/epm/archive/storage.ts

@@ -0,0 +1,140 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { extname } from 'path';
+import { isBinaryFile } from 'isbinaryfile';
+import mime from 'mime-types';
+import uuidv5 from 'uuid/v5';
+import { SavedObjectsClientContract, SavedObjectsBulkCreateObject } from 'src/core/server';
+import {
+  ASSETS_SAVED_OBJECT_TYPE,
+  InstallablePackage,
+  InstallSource,
+  PackageAssetReference,
+} from '../../../../common';
+import { getArchiveEntry } from './index';
+
+// could be anything, picked this from https://github.com/elastic/elastic-agent-client/issues/17
+const MAX_ES_ASSET_BYTES = 4 * 1024 * 1024;
+
+export interface PackageAsset {
+  package_name: string;
+  package_version: string;
+  install_source: string;
+  asset_path: string;
+  media_type: string;
+  data_utf8: string;
+  data_base64: string;
+}
+
+export function assetPathToObjectId(assetPath: string): string {
+  // uuid v5 requires a SHA-1 UUID as a namespace
+  // used to ensure same input produces the same id
+  return uuidv5(assetPath, '71403015-cdd5-404b-a5da-6c43f35cad84');
+}
+
+export async function archiveEntryToESDocument(opts: {
+  path: string;
+  buffer: Buffer;
+  name: string;
+  version: string;
+  installSource: InstallSource;
+}): Promise<PackageAsset> {
+  const { path, buffer, name, version, installSource } = opts;
+  const fileExt = extname(path);
+  const contentType = mime.lookup(fileExt);
+  const mediaType = mime.contentType(contentType || fileExt);
+  // can use to create a data URL like `data:${mediaType};base64,${base64Data}`
+
+  const bufferIsBinary = await isBinaryFile(buffer);
+  const dataUtf8 = bufferIsBinary ? '' : buffer.toString('utf8');
+  const dataBase64 = bufferIsBinary ? buffer.toString('base64') : '';
+
+  // validation: filesize? asset type? anything else
+  if (dataUtf8.length > MAX_ES_ASSET_BYTES) {
+    throw new Error(`File at ${path} is larger than maximum allowed size of ${MAX_ES_ASSET_BYTES}`);
+  }
+
+  if (dataBase64.length > MAX_ES_ASSET_BYTES) {
+    throw new Error(
+      `After base64 encoding file at ${path} is larger than maximum allowed size of ${MAX_ES_ASSET_BYTES}`
+    );
+  }
+
+  return {
+    package_name: name,
+    package_version: version,
+    install_source: installSource,
+    asset_path: path,
+    media_type: mediaType || '',
+    data_utf8: dataUtf8,
+    data_base64: dataBase64,
+  };
+}
+
+export async function removeArchiveEntries(opts: {
+  savedObjectsClient: SavedObjectsClientContract;
+  refs: PackageAssetReference[];
+}) {
+  const { savedObjectsClient, refs } = opts;
+  const results = await Promise.all(
+    refs.map((ref) => savedObjectsClient.delete(ASSETS_SAVED_OBJECT_TYPE, ref.id))
+  );
+  return results;
+}
+
+export async function saveArchiveEntries(opts: {
+  savedObjectsClient: SavedObjectsClientContract;
+  paths: string[];
+  packageInfo: InstallablePackage;
+  installSource: InstallSource;
+}) {
+  const { savedObjectsClient, paths, packageInfo, installSource } = opts;
+  const bulkBody = await Promise.all(
+    paths.map((path) => {
+      const buffer = getArchiveEntry(path);
+      if (!buffer) throw new Error(`Could not find ArchiveEntry at ${path}`);
+      const { name, version } = packageInfo;
+      return archiveEntryToBulkCreateObject({ path, buffer, name, version, installSource });
+    })
+  );
+
+  const results = await savedObjectsClient.bulkCreate<PackageAsset>(bulkBody);
+  return results;
+}
+
+export async function archiveEntryToBulkCreateObject(opts: {
+  path: string;
+  buffer: Buffer;
+  name: string;
+  version: string;
+  installSource: InstallSource;
+}): Promise<SavedObjectsBulkCreateObject<PackageAsset>> {
+  const { path, buffer, name, version, installSource } = opts;
+  const doc = await archiveEntryToESDocument({ path, buffer, name, version, installSource });
+  return {
+    id: assetPathToObjectId(doc.asset_path),
+    type: ASSETS_SAVED_OBJECT_TYPE,
+    attributes: doc,
+  };
+}
+
+export async function getAsset(opts: {
+  savedObjectsClient: SavedObjectsClientContract;
+  path: string;
+}) {
+  const { savedObjectsClient, path } = opts;
+  const assetSavedObject = await savedObjectsClient.get<PackageAsset>(
+    ASSETS_SAVED_OBJECT_TYPE,
+    assetPathToObjectId(path)
+  );
+  const storedAsset = assetSavedObject?.attributes;
+  if (!storedAsset) {
+    return;
+  }
+
+  return storedAsset;
+}

x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts

@@ -5,7 +5,13 @@
  */
 
 import { SavedObject, SavedObjectsClientContract } from 'src/core/server';
-import { InstallablePackage, InstallSource, MAX_TIME_COMPLETE_INSTALL } from '../../../../common';
+import {
+  InstallablePackage,
+  InstallSource,
+  PackageAssetReference,
+  MAX_TIME_COMPLETE_INSTALL,
+  ASSETS_SAVED_OBJECT_TYPE,
+} from '../../../../common';
 import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants';
 import {
   AssetReference,
@@ -24,6 +30,7 @@ import { deleteKibanaSavedObjectsAssets } from './remove';
 import { installTransform } from '../elasticsearch/transform/install';
 import { createInstallation, saveKibanaAssetsRefs, updateVersion } from './install';
 import { installIlmForDataStream } from '../elasticsearch/datastream_ilm/install';
+import { saveArchiveEntries } from '../archive/storage';
 import { ConcurrentInstallOperationError } from '../../../errors';
 
 // this is only exported for testing
@@ -188,11 +195,26 @@ export async function _installPackage({
     if (installKibanaAssetsError) throw installKibanaAssetsError;
     await Promise.all([installKibanaAssetsPromise, installIndexPatternPromise]);
 
+    const packageAssetResults = await saveArchiveEntries({
+      savedObjectsClient,
+      paths,
+      packageInfo,
+      installSource,
+    });
+    const packageAssetRefs: PackageAssetReference[] = packageAssetResults.saved_objects.map(
+      (result) => ({
+        id: result.id,
+        type: ASSETS_SAVED_OBJECT_TYPE,
+      })
+    );
+
     // update to newly installed version when all assets are successfully installed
     if (installedPkg) await updateVersion(savedObjectsClient, pkgName, pkgVersion);
+
     await savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, {
       install_version: pkgVersion,
       install_status: 'installed',
+      package_assets: packageAssetRefs,
     });
 
     return [

x-pack/plugins/fleet/server/services/epm/packages/ensure_installed_default_packages.test.ts

@@ -43,6 +43,7 @@ const mockInstallation: SavedObject<Installation> = {
     id: 'test-pkg',
     installed_kibana: [{ type: KibanaSavedObjectType.dashboard, id: 'dashboard-1' }],
     installed_es: [{ type: ElasticsearchAssetType.ingestPipeline, id: 'pipeline' }],
+    package_assets: [],
     es_index_patterns: { pattern: 'pattern-name' },
     name: 'test package',
     version: '1.0.0',

x-pack/plugins/fleet/server/services/epm/packages/get_install_type.test.ts

@@ -15,6 +15,7 @@ const mockInstallation: SavedObject<Installation> = {
     id: 'test-pkg',
     installed_kibana: [{ type: KibanaSavedObjectType.dashboard, id: 'dashboard-1' }],
     installed_es: [{ type: ElasticsearchAssetType.ingestPipeline, id: 'pipeline' }],
+    package_assets: [],
     es_index_patterns: { pattern: 'pattern-name' },
     name: 'test packagek',
     version: '1.0.0',
@@ -32,6 +33,7 @@ const mockInstallationUpdateFail: SavedObject<Installation> = {
     id: 'test-pkg',
     installed_kibana: [{ type: KibanaSavedObjectType.dashboard, id: 'dashboard-1' }],
     installed_es: [{ type: ElasticsearchAssetType.ingestPipeline, id: 'pipeline' }],
+    package_assets: [],
     es_index_patterns: { pattern: 'pattern-name' },
     name: 'test packagek',
     version: '1.0.0',