[Fleet] Make upload and registry package info consistent

fleet_packages.json

@@ -12,4 +12,25 @@
   in order to verify package integrity.
 */
 
-[]
+[
+  {
+    "name": "apm",
+    "version": "8.1.0"
+  },
+  {
+    "name": "elastic_agent",
+    "version": "1.3.0"
+  },
+  {
+    "name": "endpoint",
+    "version": "1.5.0"
+  },
+  {
+    "name": "fleet_server",
+    "version": "1.1.0"
+  },
+  {
+    "name": "synthetics",
+    "version": "0.9.2"
+  }
+]

x-pack/plugins/fleet/common/types/index.ts

@@ -35,6 +35,7 @@ export interface FleetConfigType {
   developer?: {
     disableRegistryVersionCheck?: boolean;
     allowAgentUpgradeSourceUri?: boolean;
+    bundledPackageLocation?: string;
   };
 }
 

x-pack/plugins/fleet/server/index.ts

@@ -130,6 +130,7 @@ export const config: PluginConfigDescriptor = {
     developer: schema.object({
       disableRegistryVersionCheck: schema.boolean({ defaultValue: false }),
       allowAgentUpgradeSourceUri: schema.boolean({ defaultValue: false }),
+      bundledPackageLocation: schema.maybe(schema.string()),
     }),
   }),
 };

x-pack/plugins/fleet/server/integration_tests/validate_bundled_packages.test.ts

@@ -12,7 +12,7 @@ import JSON5 from 'json5';
 import { REPO_ROOT } from '@kbn/utils';
 
 import * as Registry from '../services/epm/registry';
-import { parseAndVerifyArchiveEntries } from '../services/epm/archive';
+import { generatePackageInfoFromArchiveBuffer } from '../services/epm/archive';
 
 import { createAppContextStartContractMock } from '../mocks';
 import { appContextService } from '../services';
@@ -64,7 +64,7 @@ describe('validate bundled packages', () => {
     for (const packageObject of packageObjects) {
       const { registryPackage, packageArchive } = packageObject;
 
-      const archivePackageInfo = await parseAndVerifyArchiveEntries(
+      const archivePackageInfo = await generatePackageInfoFromArchiveBuffer(
         packageArchive.archiveBuffer,
         'application/zip'
       );

x-pack/plugins/fleet/server/services/epm/archive/index.ts

@@ -23,7 +23,7 @@ import { getBufferExtractor } from './extract';
 
 export * from './cache';
 export { getBufferExtractor, untarBuffer, unzipBuffer } from './extract';
-export { parseAndVerifyArchiveBuffer as parseAndVerifyArchiveEntries } from './validation';
+export { generatePackageInfoFromArchiveBuffer } from './parse';
 
 export interface ArchiveEntry {
   path: string;

x-pack/plugins/fleet/server/services/epm/archive/validation.ts → x-pack/plugins/fleet/server/services/epm/archive/parse.ts

@@ -35,12 +35,15 @@ const MANIFEST_NAME = 'manifest.yml';
 
 const DEFAULT_RELEASE_VALUE = 'ga';
 
+// Ingest pipelines are specified in a `data_stream/<name>/elasticsearch/ingest_pipeline/` directory where a `default`
+// ingest pipeline should be specified by one of these filenames.
 const DEFAULT_INGEST_PIPELINE_VALUE = 'default';
 const DEFAULT_INGEST_PIPELINE_FILE_NAME_YML = 'default.yml';
 const DEFAULT_INGEST_PIPELINE_FILE_NAME_JSON = 'default.json';
 
 // Borrowed from https://github.com/elastic/kibana/blob/main/x-pack/plugins/security_solution/common/utils/expand_dotted.ts
-// With some alterations around non-object values
+// with some alterations around non-object values. The package registry service expands some dotted fields from manifest files,
+// so we need to do the same here.
 const expandDottedField = (dottedFieldName: string, val: unknown): object => {
   const parts = dottedFieldName.split('.');
 
@@ -113,10 +116,19 @@ const registryPolicyTemplateProps = Object.values(RegistryPolicyTemplateKeys);
 const registryStreamProps = Object.values(RegistryStreamKeys);
 const registryDataStreamProps = Object.values(RegistryDataStreamKeys);
 
-// TODO: everything below performs verification of manifest.yml files, and hence duplicates functionality already implemented in the
-// package registry. At some point this should probably be replaced (or enhanced) with verification based on
-// https://github.com/elastic/package-spec/
-export async function parseAndVerifyArchiveBuffer(
+/*
+  This function generates a package info object (see type `ArchivePackage`) by parsing and verifying the `manifest.yml` file as well
+  as the directory structure for the given package archive and other files adhering to the package spec: https://github.com/elastic/package-spec.
+
+  Currently, this process is duplicative of logic that's already implemented in the Package Registry codebase,
+  e.g. https://github.com/elastic/package-registry/blob/main/packages/package.go. Because of this duplication, it's likely for our parsing/verification
+  logic to fall out of sync with the registry codebase's implementation.
+
+  This should be addressed in https://github.com/elastic/kibana/issues/115032
+  where we'll no longer use the package registry endpoint as a source of truth for package info objects, and instead Fleet will _always_ generate
+  them in the manner implemented below.
+*/
+export async function generatePackageInfoFromArchiveBuffer(
   archiveBuffer: Buffer,
   contentType: string
 ): Promise<{ paths: string[]; packageInfo: ArchivePackage }> {
@@ -363,7 +375,6 @@ export function parseAndVerifyStreams(
         streamObject.vars = vars;
       }
 
-      // default template path name see https://github.com/elastic/package-registry/blob/master/util/dataset.go#L143
       streams.push(
         Object.entries(restOfProps).reduce((validatedStream, [key, value]) => {
           if (registryStreamProps.includes(key as RegistryStreamKeys)) {

x-pack/plugins/fleet/server/services/epm/archive/storage.ts

@@ -27,7 +27,7 @@ import { appContextService } from '../../app_context';
 
 import { getArchiveEntry, setArchiveEntry, setArchiveFilelist, setPackageInfo } from './index';
 import type { ArchiveEntry } from './index';
-import { parseAndVerifyPolicyTemplates, parseAndVerifyStreams } from './validation';
+import { parseAndVerifyPolicyTemplates, parseAndVerifyStreams } from './parse';
 
 const ONE_BYTE = 1024 * 1024;
 // could be anything, picked this from https://github.com/elastic/elastic-agent-client/issues/17

x-pack/plugins/fleet/server/services/epm/packages/bundled_packages.ts

@@ -12,9 +12,13 @@ import type { BundledPackage } from '../../../types';
 import { appContextService } from '../../app_context';
 import { splitPkgKey } from '../registry';
 
-const BUNDLED_PACKAGE_DIRECTORY = path.join(__dirname, '../../../../target/bundled_packages');
-
 export async function getBundledPackages(): Promise<BundledPackage[]> {
+  const config = appContextService.getConfig();
+
+  const BUNDLED_PACKAGE_DIRECTORY = config?.developer?.bundledPackageLocation
+    ? config.developer.bundledPackageLocation
+    : path.join(__dirname, '../../../../target/bundled_packages');
+
   try {
     const dirContents = await fs.readdir(BUNDLED_PACKAGE_DIRECTORY);
     const zipFiles = dirContents.filter((file) => file.endsWith('.zip'));

x-pack/plugins/fleet/server/services/epm/packages/install.test.ts

@@ -54,7 +54,7 @@ jest.mock('../kibana/index_pattern/install', () => {
 });
 jest.mock('../archive', () => {
   return {
-    parseAndVerifyArchiveEntries: jest.fn(() =>
+    generatePackageInfoFromArchiveBuffer: jest.fn(() =>
       Promise.resolve({ packageInfo: { name: 'apache', version: '1.3.0' } })
     ),
     unpackBufferToCache: jest.fn(),

x-pack/plugins/fleet/server/services/epm/packages/install.ts

@@ -32,7 +32,11 @@ import type {
 } from '../../../types';
 import { appContextService } from '../../app_context';
 import * as Registry from '../registry';
-import { setPackageInfo, parseAndVerifyArchiveEntries, unpackBufferToCache } from '../archive';
+import {
+  setPackageInfo,
+  generatePackageInfoFromArchiveBuffer,
+  unpackBufferToCache,
+} from '../archive';
 import { toAssetReference } from '../kibana/assets/install';
 import type { ArchiveAsset } from '../kibana/assets/install';
 
@@ -391,7 +395,7 @@ async function installPackageByUpload({
   let installType: InstallType = 'unknown';
   const telemetryEvent: PackageUpdateEvent = getTelemetryEvent('', '');
   try {
-    const { packageInfo } = await parseAndVerifyArchiveEntries(archiveBuffer, contentType);
+    const { packageInfo } = await generatePackageInfoFromArchiveBuffer(archiveBuffer, contentType);
 
     const installedPkg = await getInstallationObject({
       savedObjectsClient,

x-pack/test/fleet_api_integration/apis/epm/install_bundled.ts

@@ -9,6 +9,7 @@ import expect from '@kbn/expect';
 import fs from 'fs/promises';
 import path from 'path';
 
+import { BUNDLED_PACKAGE_DIR } from '../../config';
 import { FtrProviderContext } from '../../../api_integration/ftr_provider_context';
 import { skipIfNoDockerRegistry } from '../../helpers';
 import { setupFleetAndAgents } from '../agents/services';
@@ -22,30 +23,31 @@ export default function (providerContext: FtrProviderContext) {
     path.dirname(__filename),
     '../fixtures/bundled_packages'
   );
-  const BUNDLED_PACKAGES_DIR = path.join(
-    path.dirname(__filename),
-    '../../../../plugins/fleet/target/bundled_packages'
-  );
 
   const bundlePackage = async (name: string) => {
     try {
-      await fs.access(BUNDLED_PACKAGES_DIR);
+      await fs.access(BUNDLED_PACKAGE_DIR);
     } catch (error) {
-      await fs.mkdir(BUNDLED_PACKAGES_DIR);
+      await fs.mkdir(BUNDLED_PACKAGE_DIR);
     }
 
     await fs.copyFile(
       path.join(BUNDLED_PACKAGE_FIXTURES_DIR, `${name}.zip`),
-      path.join(BUNDLED_PACKAGES_DIR, `${name}.zip`)
+      path.join(BUNDLED_PACKAGE_DIR, `${name}.zip`)
     );
   };
 
   const removeBundledPackages = async () => {
     try {
-      const files = await fs.readdir(BUNDLED_PACKAGES_DIR);
+      const files = await fs.readdir(BUNDLED_PACKAGE_DIR);
 
       for (const file of files) {
-        await fs.unlink(path.join(BUNDLED_PACKAGES_DIR, file));
+        const isFixtureFile = !!(await fs.readFile(path.join(BUNDLED_PACKAGE_FIXTURES_DIR, file)));
+
+        // Only remove fixture files - leave normal bundled packages in place
+        if (isFixtureFile) {
+          await fs.unlink(path.join(BUNDLED_PACKAGE_DIR, file));
+        }
       }
     } catch (error) {
       log.error('Error removing bundled packages');
@@ -63,10 +65,11 @@ export default function (providerContext: FtrProviderContext) {
 
     describe('without registry', () => {
       it('installs from bundled source via api', async () => {
-        await bundlePackage('elastic_agent-1.2.0');
+        // Need to bundle a package that doesn't conflict with those listed in `fleet_packages.json
+        await bundlePackage('nginx-1.2.1');
 
         const response = await supertest
-          .post(`/api/fleet/epm/packages/elastic_agent/1.2.0`)
+          .post(`/api/fleet/epm/packages/nginx/1.2.1`)
           .set('kbn-xsrf', 'xxxx')
           .type('application/json')
           .send({ force: true })
@@ -76,11 +79,11 @@ export default function (providerContext: FtrProviderContext) {
       });
 
       it('allows for upgrading from newer bundled source when outdated package was installed from bundled source', async () => {
-        await bundlePackage('elastic_agent-1.0.0');
-        await bundlePackage('elastic_agent-1.2.0');
+        await bundlePackage('nginx-1.1.0');
+        await bundlePackage('nginx-1.2.1');
 
         const installResponse = await supertest
-          .post(`/api/fleet/epm/packages/elastic_agent/1.0.0`)
+          .post(`/api/fleet/epm/packages/nginx/1.1.0`)
           .set('kbn-xsrf', 'xxxx')
           .type('application/json')
           .send({ force: true })
@@ -89,7 +92,7 @@ export default function (providerContext: FtrProviderContext) {
         expect(installResponse.body._meta.install_source).to.be('bundled');
 
         const updateResponse = await supertest
-          .post(`/api/fleet/epm/packages/elastic_agent/1.2.0`)
+          .post(`/api/fleet/epm/packages/nginx/1.2.1`)
           .set('kbn-xsrf', 'xxxx')
           .type('application/json')
           .send({ force: true })
@@ -101,19 +104,20 @@ export default function (providerContext: FtrProviderContext) {
 
     describe('with registry', () => {
       it('allows for updating from registry when outdated package is installed from bundled source', async () => {
-        await bundlePackage('elastic_agent-1.2.0');
+        await bundlePackage('nginx-1.1.0');
 
         const bundledInstallResponse = await supertest
-          .post(`/api/fleet/epm/packages/elastic_agent/1.2.0`)
+          .post(`/api/fleet/epm/packages/nginx/1.1.0`)
           .set('kbn-xsrf', 'xxxx')
           .type('application/json')
           .send({ force: true })
           .expect(200);
 
         expect(bundledInstallResponse.body._meta.install_source).to.be('bundled');
 
+        // Update to one version prior to the bundled version of nginx
         const registryUpdateResponse = await supertest
-          .post(`/api/fleet/epm/packages/elastic_agent/1.3.0`)
+          .post(`/api/fleet/epm/packages/elastic_agent/1.2.0`)
           .set('kbn-xsrf', 'xxxx')
           .type('application/json')
           .send({ force: true })

x-pack/test/fleet_api_integration/apis/epm/install_by_upload.ts

@@ -75,7 +75,7 @@ export default function (providerContext: FtrProviderContext) {
         .type('application/gzip')
         .send(buf)
         .expect(200);
-      expect(res.body.items.length).to.be(27);
+      expect(res.body.items.length).to.be(29);
     });
 
     it('should install a zip archive correctly and package info should return correctly after validation', async function () {
@@ -86,7 +86,7 @@ export default function (providerContext: FtrProviderContext) {
         .type('application/zip')
         .send(buf)
         .expect(200);
-      expect(res.body.items.length).to.be(27);
+      expect(res.body.items.length).to.be(29);
     });
 
     it('should throw an error if the archive is zip but content type is gzip', async function () {

x-pack/test/fleet_api_integration/config.ts

@@ -17,6 +17,8 @@ import { defineDockerServersConfig } from '@kbn/test';
 export const dockerImage =
   'docker.elastic.co/package-registry/distribution@sha256:b3dfc6a11ff7dce82ba8689ea9eeb54e353c6b4bfd2d28127b20ef72fd8883e9';
 
+export const BUNDLED_PACKAGE_DIR = '/tmp/fleet_bundled_packages';
+
 export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const xPackAPITestsConfig = await readConfigFile(require.resolve('../api_integration/config.ts'));
 
@@ -63,6 +65,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
         `--xpack.fleet.packages.0.name=endpoint`,
         `--xpack.fleet.packages.0.version=latest`,
         ...(registryPort ? [`--xpack.fleet.registryUrl=http://localhost:${registryPort}`] : []),
+        `--xpack.fleet.developer.bundledPackageLocation=${BUNDLED_PACKAGE_DIR}`,
       ],
     },
   };