[Ingest Manager] Install uploaded package (#77986)

* Refactor: installPackage -> installPackageFromRegistry

* Refactor: factor out source-agnostic installation steps

* Unpack and cache uploaded zip and tgz files.

* Add basic archive verification and parse manifest.

* Catch error when zip archive is uploaded as gzip.

* Add API integration tests.

* Remove unnecessary use of "package key" concept.

* Add 'install_source' property to saved object epm-packages.

* Adjust tests.

* Add API integration test for manifest missing fields.

* Refactor loadArchive -> loadArchivePackage.

* Refactor caching of package archive content

* Get datasets and config templates from manifest files.

* Use file paths from archive instead of asset paths from registry.

* Correctly load registry packages into cache

* Use InstallablePackage instead of RegistryPackage where possible.

* Actually install uploaded package.

* Add missing field to saved objects in tests.

* Adjust unit test to pick buffer extractor.

* Adjust unit test.

* Fix and re-enable getAsset() test.

* Adjust integration tests.

* Make error message match test.

* Pick data_stream.dataset from manifest if set.

* dataset -> data_stream also in comments

* Remove unused variable.

* Use pkgToPkgKey() where appropriate.

* More dataset -> data stream renaming.

Co-authored-by: Kibana Machine <[email protected]>

x-pack/plugins/ingest_manager/common/types/models/epm.ts

@@ -20,6 +20,7 @@ export enum InstallStatus {
 }
 
 export type InstallType = 'reinstall' | 'reupdate' | 'rollback' | 'update' | 'install';
+export type InstallSource = 'registry' | 'upload';
 
 export type EpmPackageInstallStatus = 'installed' | 'installing';
 
@@ -49,10 +50,8 @@ export enum AgentAssetType {
 
 export type RegistryRelease = 'ga' | 'beta' | 'experimental';
 
-// from /package/{name}
-// type Package struct at https://github.com/elastic/package-registry/blob/master/util/package.go
-// https://github.com/elastic/package-registry/blob/master/docs/api/package.json
-export interface RegistryPackage {
+// Fields common to packages that come from direct upload and the registry
+export interface InstallablePackage {
   name: string;
   title?: string;
   version: string;
@@ -61,14 +60,24 @@ export interface RegistryPackage {
   description: string;
   type: string;
   categories: string[];
-  requirement: RequirementsByServiceName;
   screenshots?: RegistryImage[];
   icons?: RegistryImage[];
   assets?: string[];
   internal?: boolean;
   format_version: string;
   data_streams?: RegistryDataStream[];
   policy_templates?: RegistryPolicyTemplate[];
+}
+
+// Uploaded package archives don't have extra fields
+// Linter complaint disabled because this extra type is meant for better code readability
+// eslint-disable-next-line @typescript-eslint/no-empty-interface
+export interface ArchivePackage extends InstallablePackage {}
+
+// Registry packages do have extra fields.
+// cf. type Package struct at https://github.com/elastic/package-registry/blob/master/util/package.go
+export interface RegistryPackage extends InstallablePackage {
+  requirement: RequirementsByServiceName;
   download: string;
   path: string;
 }
@@ -240,6 +249,7 @@ export interface Installation extends SavedObjectAttributes {
   install_status: EpmPackageInstallStatus;
   install_version: string;
   install_started_at: string;
+  install_source: InstallSource;
 }
 
 export type Installable<T> = Installed<T> | NotInstalled<T>;

x-pack/plugins/ingest_manager/server/errors/handlers.test.ts

@@ -13,6 +13,7 @@ import {
   IngestManagerError,
   RegistryError,
   PackageNotFoundError,
+  PackageUnsupportedMediaTypeError,
   defaultIngestErrorHandler,
 } from './index';
 
@@ -101,6 +102,25 @@ describe('defaultIngestErrorHandler', () => {
       expect(mockContract.logger?.error).toHaveBeenCalledWith(error.message);
     });
 
+    it('415: PackageUnsupportedMediaType', async () => {
+      const error = new PackageUnsupportedMediaTypeError('123');
+      const response = httpServerMock.createResponseFactory();
+
+      await defaultIngestErrorHandler({ error, response });
+
+      // response
+      expect(response.ok).toHaveBeenCalledTimes(0);
+      expect(response.customError).toHaveBeenCalledTimes(1);
+      expect(response.customError).toHaveBeenCalledWith({
+        statusCode: 415,
+        body: { message: error.message },
+      });
+
+      // logging
+      expect(mockContract.logger?.error).toHaveBeenCalledTimes(1);
+      expect(mockContract.logger?.error).toHaveBeenCalledWith(error.message);
+    });
+
     it('404: PackageNotFoundError', async () => {
       const error = new PackageNotFoundError('123');
       const response = httpServerMock.createResponseFactory();

x-pack/plugins/ingest_manager/server/errors/handlers.ts

@@ -13,7 +13,12 @@ import {
 } from 'src/core/server';
 import { errors as LegacyESErrors } from 'elasticsearch';
 import { appContextService } from '../services';
-import { IngestManagerError, RegistryError, PackageNotFoundError } from './index';
+import {
+  IngestManagerError,
+  RegistryError,
+  PackageNotFoundError,
+  PackageUnsupportedMediaTypeError,
+} from './index';
 
 type IngestErrorHandler = (
   params: IngestErrorHandlerParams
@@ -52,6 +57,9 @@ const getHTTPResponseCode = (error: IngestManagerError): number => {
   if (error instanceof PackageNotFoundError) {
     return 404; // Not Found
   }
+  if (error instanceof PackageUnsupportedMediaTypeError) {
+    return 415; // Unsupported Media Type
+  }
 
   return 400; // Bad Request
 };

x-pack/plugins/ingest_manager/server/errors/index.ts

@@ -18,3 +18,7 @@ export class RegistryConnectionError extends RegistryError {}
 export class RegistryResponseError extends RegistryError {}
 export class PackageNotFoundError extends IngestManagerError {}
 export class PackageOutdatedError extends IngestManagerError {}
+export class PackageUnsupportedMediaTypeError extends IngestManagerError {}
+export class PackageInvalidArchiveError extends IngestManagerError {}
+export class PackageCacheError extends IngestManagerError {}
+export class PackageOperationNotSupportedError extends IngestManagerError {}

x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts

@@ -8,7 +8,6 @@ import { RequestHandler, CustomHttpResponseOptions } from 'src/core/server';
 import {
   GetInfoResponse,
   InstallPackageResponse,
-  MessageResponse,
   DeletePackageResponse,
   GetCategoriesResponse,
   GetPackagesResponse,
@@ -35,8 +34,9 @@ import {
   getFile,
   getPackageInfo,
   handleInstallPackageFailure,
-  installPackage,
   isBulkInstallError,
+  installPackageFromRegistry,
+  installPackageByUpload,
   removeInstallation,
   getLimitedPackages,
   getInstallationObject,
@@ -148,7 +148,7 @@ export const installPackageFromRegistryHandler: RequestHandler<
   const { pkgName, pkgVersion } = splitPkgKey(pkgkey);
   const installedPkg = await getInstallationObject({ savedObjectsClient, pkgName });
   try {
-    const res = await installPackage({
+    const res = await installPackageFromRegistry({
       savedObjectsClient,
       pkgkey,
       callCluster,
@@ -212,10 +212,24 @@ export const installPackageByUploadHandler: RequestHandler<
   undefined,
   TypeOf<typeof InstallPackageByUploadRequestSchema.body>
 > = async (context, request, response) => {
-  const body: MessageResponse = {
-    response: 'package upload was received ok, but not installed (not implemented yet)',
-  };
-  return response.ok({ body });
+  const savedObjectsClient = context.core.savedObjects.client;
+  const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser;
+  const contentType = request.headers['content-type'] as string; // from types it could also be string[] or undefined but this is checked later
+  const archiveBuffer = Buffer.from(request.body);
+  try {
+    const res = await installPackageByUpload({
+      savedObjectsClient,
+      callCluster,
+      archiveBuffer,
+      contentType,
+    });
+    const body: InstallPackageResponse = {
+      response: res,
+    };
+    return response.ok({ body });
+  } catch (error) {
+    return defaultIngestErrorHandler({ error, response });
+  }
 };
 
 export const deletePackageHandler: RequestHandler<TypeOf<

x-pack/plugins/ingest_manager/server/saved_objects/index.ts

@@ -303,6 +303,7 @@ const getSavedObjectTypes = (
         install_started_at: { type: 'date' },
         install_version: { type: 'keyword' },
         install_status: { type: 'keyword' },
+        install_source: { type: 'keyword' },
       },
     },
   },

x-pack/plugins/ingest_manager/server/saved_objects/migrations/to_v7_10_0.ts

@@ -14,6 +14,7 @@ import {
   EnrollmentAPIKey,
   Settings,
   AgentAction,
+  Installation,
 } from '../../types';
 
 export const migrateAgentToV7100: SavedObjectMigrationFn<
@@ -134,3 +135,12 @@ export const migrateAgentActionToV7100 = (
     }
   );
 };
+
+export const migrateInstallationToV7100: SavedObjectMigrationFn<
+  Exclude<Installation, 'install_source'>,
+  Installation
+> = (installationDoc) => {
+  installationDoc.attributes.install_source = 'registry';
+
+  return installationDoc;
+};