[Fleet] Make upload and registry package info consistent (#126915)

* Update docker image + set up initial validation test

* Get validation test passing

* Remove erroneous test load call

* Address PR review + improve comments + rename validation.ts -> parse.ts

* Replace packages in fleet_packages.json

* Add temp debug log to debug CI failures

* Use a non-colliding package in bundled package tests

* (debug) Add logging output

* (debug) More logging

* (debug) Log bundled package dir in module

* Use absolute path for bundled packages

* Remove debug logs + use KIBANA_BUILD_LOCATION if it exists

* Add support for developer.bundledPackageLocation config value

* (debug) Try some more logs

* (debug) Try some more logs

* Fix test hopefully 🤞

* Fix other failing tests

* Move default for bundled package dir to schema definition

* Fix schema default value for bundledPackageLocation

* Fix snapshot

* Fix regression in bundled packages fetch

Co-authored-by: Kibana Machine <[email protected]>

fleet_packages.json

@@ -12,4 +12,25 @@
   in order to verify package integrity.
 */
 
-[]
+[
+  {
+    "name": "apm",
+    "version": "8.1.0"
+  },
+  {
+    "name": "elastic_agent",
+    "version": "1.3.0"
+  },
+  {
+    "name": "endpoint",
+    "version": "1.5.0"
+  },
+  {
+    "name": "fleet_server",
+    "version": "1.1.0"
+  },
+  {
+    "name": "synthetics",
+    "version": "0.9.2"
+  }
+]

x-pack/plugins/fleet/common/types/index.ts

@@ -35,6 +35,7 @@ export interface FleetConfigType {
   developer?: {
     disableRegistryVersionCheck?: boolean;
     allowAgentUpgradeSourceUri?: boolean;
+    bundledPackageLocation?: string;
   };
 }
 

x-pack/plugins/fleet/common/types/models/epm.ts

@@ -305,6 +305,7 @@ export enum RegistryDataStreamKeys {
 }
 
 export interface RegistryDataStream {
+  [key: string]: any;
   [RegistryDataStreamKeys.type]: string;
   [RegistryDataStreamKeys.ilm_policy]?: string;
   [RegistryDataStreamKeys.hidden]?: boolean;
@@ -323,6 +324,7 @@ export interface RegistryElasticsearch {
   privileges?: RegistryDataStreamPrivileges;
   'index_template.settings'?: estypes.IndicesIndexSettings;
   'index_template.mappings'?: estypes.MappingTypeMapping;
+  'ingest_pipeline.name'?: string;
 }
 
 export interface RegistryDataStreamPrivileges {

x-pack/plugins/fleet/common/types/models/package_spec.ts

@@ -70,4 +70,5 @@ export interface PackageSpecScreenshot {
   title: string;
   size?: string;
   type?: string;
+  path?: string;
 }

x-pack/plugins/fleet/server/index.ts

@@ -5,6 +5,8 @@
  * 2.0.
  */
 
+import path from 'path';
+
 import { schema } from '@kbn/config-schema';
 import type { TypeOf } from '@kbn/config-schema';
 import type { PluginConfigDescriptor, PluginInitializerContext } from 'src/core/server';
@@ -40,6 +42,8 @@ export type {
 } from './types';
 export { AgentNotFoundError, FleetUnauthorizedError } from './errors';
 
+const DEFAULT_BUNDLED_PACKAGE_LOCATION = path.join(__dirname, '../target/bundled_packages');
+
 export const config: PluginConfigDescriptor = {
   exposeToBrowser: {
     epm: true,
@@ -130,6 +134,7 @@ export const config: PluginConfigDescriptor = {
     developer: schema.object({
       disableRegistryVersionCheck: schema.boolean({ defaultValue: false }),
       allowAgentUpgradeSourceUri: schema.boolean({ defaultValue: false }),
+      bundledPackageLocation: schema.string({ defaultValue: DEFAULT_BUNDLED_PACKAGE_LOCATION }),
     }),
   }),
 };

x-pack/plugins/fleet/server/integration_tests/helpers/docker_registry_helper.ts

@@ -24,7 +24,7 @@ export function useDockerRegistry() {
 
   let dockerProcess: ChildProcess | undefined;
   async function startDockerRegistryServer() {
-    const dockerImage = `docker.elastic.co/package-registry/distribution@sha256:8b4ce36ecdf86e6cfdf781d9df8d564a014add9afc9aec21cf2c5a68ff82d3ab`;
+    const dockerImage = `docker.elastic.co/package-registry/distribution@sha256:b3dfc6a11ff7dce82ba8689ea9eeb54e353c6b4bfd2d28127b20ef72fd8883e9`;
 
     const args = ['run', '--rm', '-p', `${packageRegistryPort}:8080`, dockerImage];
 

x-pack/plugins/fleet/server/integration_tests/validate_bundled_packages.test.ts

@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import path from 'path';
+import fs from 'fs/promises';
+
+import JSON5 from 'json5';
+import { REPO_ROOT } from '@kbn/utils';
+
+import * as Registry from '../services/epm/registry';
+import { generatePackageInfoFromArchiveBuffer } from '../services/epm/archive';
+
+import { createAppContextStartContractMock } from '../mocks';
+import { appContextService } from '../services';
+
+import { useDockerRegistry } from './helpers';
+
+describe('validate bundled packages', () => {
+  const registryUrl = useDockerRegistry();
+  let mockContract: ReturnType<typeof createAppContextStartContractMock>;
+
+  beforeEach(() => {
+    mockContract = createAppContextStartContractMock({ registryUrl });
+    appContextService.start(mockContract);
+  });
+
+  async function getBundledPackageEntries() {
+    const configFilePath = path.resolve(REPO_ROOT, 'fleet_packages.json');
+    const configFile = await fs.readFile(configFilePath, 'utf8');
+    const bundledPackages = JSON5.parse(configFile);
+
+    return bundledPackages as Array<{ name: string; version: string }>;
+  }
+
+  async function setupPackageObjects() {
+    const bundledPackages = await getBundledPackageEntries();
+
+    const packageObjects = await Promise.all(
+      bundledPackages.map(async (bundledPackage) => {
+        const registryPackage = await Registry.getRegistryPackage(
+          bundledPackage.name,
+          bundledPackage.version
+        );
+
+        const packageArchive = await Registry.fetchArchiveBuffer(
+          bundledPackage.name,
+          bundledPackage.version
+        );
+
+        return { registryPackage, packageArchive };
+      })
+    );
+
+    return packageObjects;
+  }
+
+  it('generates matching package info objects for uploaded and registry packages', async () => {
+    const packageObjects = await setupPackageObjects();
+
+    for (const packageObject of packageObjects) {
+      const { registryPackage, packageArchive } = packageObject;
+
+      const archivePackageInfo = await generatePackageInfoFromArchiveBuffer(
+        packageArchive.archiveBuffer,
+        'application/zip'
+      );
+
+      expect(archivePackageInfo.packageInfo.data_streams).toEqual(
+        registryPackage.packageInfo.data_streams
+      );
+    }
+  });
+});

x-pack/plugins/fleet/server/mocks/index.ts

@@ -21,6 +21,7 @@ import type { PackagePolicyServiceInterface } from '../services/package_policy';
 import type { AgentPolicyServiceInterface } from '../services';
 import type { FleetAppContext } from '../plugin';
 import { createMockTelemetryEventsSender } from '../telemetry/__mocks__';
+import type { FleetConfigType } from '../../common';
 import { createFleetAuthzMock } from '../../common';
 import { agentServiceMock } from '../services/agents/agent_service.mock';
 import type { FleetRequestHandlerContext } from '../types';
@@ -39,11 +40,14 @@ export interface MockedFleetAppContext extends FleetAppContext {
   logger: ReturnType<ReturnType<typeof loggingSystemMock.create>['get']>;
 }
 
-export const createAppContextStartContractMock = (): MockedFleetAppContext => {
+export const createAppContextStartContractMock = (
+  configOverrides: Partial<FleetConfigType> = {}
+): MockedFleetAppContext => {
   const config = {
     agents: { enabled: true, elasticsearch: {} },
     enabled: true,
     agentIdVerificationEnabled: true,
+    ...configOverrides,
   };
 
   const config$ = of(config);

x-pack/plugins/fleet/server/services/epm/archive/index.ts

@@ -23,7 +23,7 @@ import { getBufferExtractor } from './extract';
 
 export * from './cache';
 export { getBufferExtractor, untarBuffer, unzipBuffer } from './extract';
-export { parseAndVerifyArchiveBuffer as parseAndVerifyArchiveEntries } from './validation';
+export { generatePackageInfoFromArchiveBuffer } from './parse';
 
 export interface ArchiveEntry {
   path: string;