Change from plain ES on new index to SO on .kibana

x-pack/plugins/fleet/common/constants/epm.ts

@@ -3,8 +3,8 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-export const PACKAGE_ASSETS_INDEX_NAME = 'epm-packages-assets';
 export const PACKAGES_SAVED_OBJECT_TYPE = 'epm-packages';
+export const ASSETS_SAVED_OBJECT_TYPE = 'epm-packages-assets';
 export const INDEX_PATTERN_SAVED_OBJECT_TYPE = 'index-pattern';
 export const INDEX_PATTERN_PLACEHOLDER_SUFFIX = '-index_pattern_placeholder';
 export const MAX_TIME_COMPLETE_INSTALL = 60000;

x-pack/plugins/fleet/common/types/models/epm.ts

@@ -8,7 +8,7 @@
 // TODO: Update when https://github.com/elastic/kibana/issues/53021 is closed
 import { SavedObject, SavedObjectAttributes, SavedObjectReference } from 'src/core/public';
 import {
-  PACKAGE_ASSETS_INDEX_NAME,
+  ASSETS_SAVED_OBJECT_TYPE,
   agentAssetTypes,
   dataTypes,
   defaultPackages,
@@ -302,7 +302,7 @@ export type EsAssetReference = Pick<SavedObjectReference, 'id'> & {
 };
 
 export type PackageAssetReference = Pick<SavedObjectReference, 'id'> & {
-  type: typeof PACKAGE_ASSETS_INDEX_NAME;
+  type: typeof ASSETS_SAVED_OBJECT_TYPE;
 };
 
 export type RequiredPackage = typeof requiredPackages;

x-pack/plugins/fleet/server/constants/index.ts

@@ -41,6 +41,7 @@ export {
   PACKAGE_POLICY_SAVED_OBJECT_TYPE,
   OUTPUT_SAVED_OBJECT_TYPE,
   PACKAGES_SAVED_OBJECT_TYPE,
+  ASSETS_SAVED_OBJECT_TYPE,
   INDEX_PATTERN_SAVED_OBJECT_TYPE,
   ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE,
   GLOBAL_SETTINGS_SAVED_OBJECT_TYPE,

x-pack/plugins/fleet/server/saved_objects/index.ts

@@ -12,6 +12,7 @@ import {
   AGENT_POLICY_SAVED_OBJECT_TYPE,
   PACKAGE_POLICY_SAVED_OBJECT_TYPE,
   PACKAGES_SAVED_OBJECT_TYPE,
+  ASSETS_SAVED_OBJECT_TYPE,
   AGENT_SAVED_OBJECT_TYPE,
   AGENT_EVENT_SAVED_OBJECT_TYPE,
   AGENT_ACTION_SAVED_OBJECT_TYPE,
@@ -318,6 +319,25 @@ const getSavedObjectTypes = (
       },
     },
   },
+  [ASSETS_SAVED_OBJECT_TYPE]: {
+    name: ASSETS_SAVED_OBJECT_TYPE,
+    hidden: false,
+    namespaceType: 'agnostic',
+    management: {
+      importableAndExportable: false,
+    },
+    mappings: {
+      properties: {
+        package_name: { type: 'keyword' },
+        package_version: { type: 'keyword' },
+        install_source: { type: 'keyword' },
+        asset_path: { type: 'keyword' },
+        media_type: { type: 'keyword' },
+        data_utf8: { type: 'text', index: false },
+        data_base64: { type: 'binary' },
+      },
+    },
+  },
 });
 
 export function registerSavedObjects(

x-pack/plugins/fleet/server/services/epm/archive/save_to_es.ts

@@ -8,33 +8,22 @@ import { extname } from 'path';
 import { isBinaryFile } from 'isbinaryfile';
 import mime from 'mime-types';
 import uuidv5 from 'uuid/v5';
+import { SavedObjectsClientContract, SavedObjectsBulkCreateObject } from 'src/core/server';
 import {
-  PACKAGE_ASSETS_INDEX_NAME,
+  ASSETS_SAVED_OBJECT_TYPE,
   InstallablePackage,
   InstallSource,
   PackageAssetReference,
 } from '../../../../common';
-import { CallESAsCurrentUser } from '../../../types';
-import { appContextService } from '../../../services';
 import { getArchiveEntry } from './index';
 
 // uuid v5 requires a SHA-1 UUID as a namespace
-// used to ensure consistent ids when given the same inputs
+// used to ensure same input produces the same id
 const ID_NAMESPACE = '71403015-cdd5-404b-a5da-6c43f35cad84';
 
 // could be anything, picked this from https://github.com/elastic/elastic-agent-client/issues/17
 const MAX_ES_ASSET_BYTES = 4 * 1024 * 1024;
 
-const packageAssetMappings = {
-  package_name: { type: 'keyword' },
-  package_version: { type: 'keyword' },
-  install_source: { type: 'keyword' },
-  asset_path: { type: 'keyword' },
-  media_type: { type: 'keyword' },
-  data_utf8: { type: 'text', index: false },
-  data_base64: { type: 'binary' },
-};
-
 export interface PackageAsset {
   package_name: string;
   package_version: string;
@@ -45,66 +34,6 @@ export interface PackageAsset {
   data_base64: string;
 }
 
-export const ensurePackagesIndex = async (opts: { callCluster: CallESAsCurrentUser }) => {
-  const { callCluster } = opts;
-  const logger = appContextService.getLogger();
-  const indexExists = await callCluster('indices.exists', { index: PACKAGE_ASSETS_INDEX_NAME });
-  if (!indexExists) {
-    try {
-      const clientParams = {
-        index: PACKAGE_ASSETS_INDEX_NAME,
-        body: {
-          mappings: {
-            properties: packageAssetMappings,
-          },
-        },
-      };
-      await callCluster('indices.create', clientParams);
-    } catch (putErr) {
-      logger.error(`${PACKAGE_ASSETS_INDEX_NAME} could not be created`);
-    }
-  }
-};
-
-export const saveArchiveEntriesToES = async (opts: {
-  callCluster: CallESAsCurrentUser;
-  paths: string[];
-  packageInfo: InstallablePackage;
-  installSource: InstallSource;
-}) => {
-  const { callCluster, paths, packageInfo, installSource } = opts;
-  await ensurePackagesIndex({ callCluster });
-  const bulkBody = await createBulkBody({ paths, packageInfo, installSource });
-  const results: BulkResponse = await callCluster('bulk', { body: bulkBody });
-  return results;
-};
-
-export async function createBulkBody(opts: {
-  paths: string[];
-  packageInfo: InstallablePackage;
-  installSource: InstallSource;
-}) {
-  const { paths, packageInfo, installSource } = opts;
-  const bulkBody = await Promise.all(
-    paths.map(async (path) => {
-      const buffer = getArchiveEntry(path);
-      if (!buffer) throw new Error(`Could not find ArchiveEntry at ${path}`);
-      const { name, version } = packageInfo;
-      const doc = await archiveEntryToESDocument({ path, buffer, name, version, installSource });
-      const action = {
-        index: {
-          _index: PACKAGE_ASSETS_INDEX_NAME,
-          _id: uuidv5(doc.asset_path, ID_NAMESPACE),
-        },
-      };
-
-      return [action, doc];
-    })
-  );
-
-  return bulkBody.flat();
-}
-
 export async function archiveEntryToESDocument(opts: {
   path: string;
   buffer: Buffer;
@@ -144,48 +73,49 @@ export async function archiveEntryToESDocument(opts: {
   };
 }
 
-export async function removeArchiveEntriesFromES(opts: {
-  callCluster: CallESAsCurrentUser;
+export async function removeArchiveEntries(opts: {
+  savedObjectsClient: SavedObjectsClientContract;
   refs: PackageAssetReference[];
 }) {
-  const bulkBody = opts.refs.map(({ id }) => {
-    return {
-      delete: { _index: PACKAGE_ASSETS_INDEX_NAME, _id: id },
-    };
-  });
-  const results: BulkResponse = await opts.callCluster('bulk', { body: bulkBody });
+  const { savedObjectsClient, refs } = opts;
+  const results = await Promise.all(
+    refs.map((ref) => savedObjectsClient.delete(ASSETS_SAVED_OBJECT_TYPE, ref.id))
+  );
+  return results;
+}
+
+export async function saveArchiveEntries(opts: {
+  savedObjectsClient: SavedObjectsClientContract;
+  paths: string[];
+  packageInfo: InstallablePackage;
+  installSource: InstallSource;
+}) {
+  const { savedObjectsClient, paths, packageInfo, installSource } = opts;
+  const bulkBody = await Promise.all(
+    paths.map((path) => {
+      const buffer = getArchiveEntry(path);
+      if (!buffer) throw new Error(`Could not find ArchiveEntry at ${path}`);
+      const { name, version } = packageInfo;
+      return archiveEntryToBulkCreateObject({ path, buffer, name, version, installSource });
+    })
+  );
+
+  const results = await savedObjectsClient.bulkCreate<PackageAsset>(bulkBody);
   return results;
 }
 
-// based on plugins/security_solution/server/lib/detection_engine/signals/types.ts
-// ideally we use proper/official types
-type BulkItem = Record<
-  'create' | 'delete' | 'index' | 'update',
-  {
-    _index: string;
-    _type?: string;
-    _id: string;
-    _version: number;
-    result?: 'created' | 'deleted' | 'updated';
-    _shards?: {
-      total: number;
-      successful: number;
-      failed: number;
-    };
-    _seq_no?: number;
-    _primary_term?: number;
-    status: number;
-    error?: {
-      type: string;
-      reason: string;
-      index_uuid?: string;
-      shard: string;
-      index: string;
-    };
-  }
->;
-interface BulkResponse {
-  took: number;
-  errors: boolean;
-  items: BulkItem[];
+export async function archiveEntryToBulkCreateObject(opts: {
+  path: string;
+  buffer: Buffer;
+  name: string;
+  version: string;
+  installSource: InstallSource;
+}): Promise<SavedObjectsBulkCreateObject<PackageAsset>> {
+  const { path, buffer, name, version, installSource } = opts;
+  const doc = await archiveEntryToESDocument({ path, buffer, name, version, installSource });
+  return {
+    id: uuidv5(doc.asset_path, ID_NAMESPACE),
+    type: ASSETS_SAVED_OBJECT_TYPE,
+    attributes: doc,
+  };
 }

x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts

@@ -10,7 +10,7 @@ import {
   InstallSource,
   PackageAssetReference,
   MAX_TIME_COMPLETE_INSTALL,
-  PACKAGE_ASSETS_INDEX_NAME,
+  ASSETS_SAVED_OBJECT_TYPE,
 } from '../../../../common';
 import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants';
 import {
@@ -29,7 +29,7 @@ import { updateCurrentWriteIndices } from '../elasticsearch/template/template';
 import { deleteKibanaSavedObjectsAssets } from './remove';
 import { installTransform } from '../elasticsearch/transform/install';
 import { createInstallation, saveKibanaAssetsRefs, updateVersion } from './install';
-import { saveArchiveEntriesToES } from '../archive/save_to_es';
+import { saveArchiveEntries } from '../archive/save_to_es';
 
 // this is only exported for testing
 // use a leading underscore to indicate it's not the supported path
@@ -184,16 +184,18 @@ export async function _installPackage({
   if (installKibanaAssetsError) throw installKibanaAssetsError;
   await Promise.all([installKibanaAssetsPromise, installIndexPatternPromise]);
 
-  const packageAssetResults = await saveArchiveEntriesToES({
-    callCluster,
+  const packageAssetResults = await saveArchiveEntries({
+    savedObjectsClient,
     paths,
     packageInfo,
     installSource,
   });
-  const packageAssetRefs: PackageAssetReference[] = packageAssetResults.items.map((result) => ({
-    id: result.index._id,
-    type: PACKAGE_ASSETS_INDEX_NAME,
-  }));
+  const packageAssetRefs: PackageAssetReference[] = packageAssetResults.saved_objects.map(
+    (result) => ({
+      id: result.id,
+      type: ASSETS_SAVED_OBJECT_TYPE,
+    })
+  );
 
   // update to newly installed version when all assets are successfully installed
   if (installedPkg) await updateVersion(savedObjectsClient, pkgName, pkgVersion);

x-pack/plugins/fleet/server/services/epm/packages/remove.ts

@@ -23,7 +23,7 @@ import { deleteTransforms } from '../elasticsearch/transform/remove';
 import { packagePolicyService, appContextService } from '../..';
 import { splitPkgKey } from '../registry';
 import { deletePackageCache } from '../archive';
-import { removeArchiveEntriesFromES } from '../archive/save_to_es';
+import { removeArchiveEntries } from '../archive/save_to_es';
 
 export async function removeInstallation(options: {
   savedObjectsClient: SavedObjectsClientContract;
@@ -49,7 +49,7 @@ export async function removeInstallation(options: {
       `unable to remove package with existing package policy(s) in use by agent(s)`
     );
 
-  // Delete the installed assets
+  // Delete the installed assets. Don't include installation.package_assets. Those are irrelevant to users
   const installedAssets = [...installation.installed_kibana, ...installation.installed_es];
   await deleteAssets(installation, savedObjectsClient, callCluster);
 
@@ -69,7 +69,8 @@ export async function removeInstallation(options: {
     version: pkgVersion,
   });
 
-  await removeArchiveEntriesFromES({ callCluster, refs: installation.package_assets });
+  await removeArchiveEntries({ savedObjectsClient, refs: installation.package_assets });
+
   // successful delete's in SO client return {}. return something more useful
   return installedAssets;
 }