Merge branch 'main' into use-locators-2

.gitignore

@@ -93,3 +93,5 @@ elastic-agent-*
 fleet-server-*
 elastic-agent.yml
 fleet-server.yml
+
+/x-pack/plugins/fleet/server/bundled_packages

fleet_packages.json

@@ -0,0 +1,36 @@
+/*
+
+  Packages in this file are considered "bundled" and are installed as part of Fleet's setup process. Each entry points to a valid version name
+  avaiable in the Elastic Package Registry service, and must include a sha-512 checksum of the `.zip` archive for the given package.
+
+  You may opt in to using the "snapshot" environment of the EPR service by passing the `--use-epr-snapshot-registry` flag to `yarn build`. This will
+  cause the package archive download to pull from the "spapshot" environment instead of the "production" environment. Be aware that not all packages
+  exist in the snapshot environment, so you may have errors when fetching package versions. It's recommended to alter this file to contain _only_ the
+  packages you're testing when using the snapshot environment.
+
+  These files don't include any kind of checksum, but they should eventually include a package signature as introduced in https://github.com/elastic/elastic-package/issues/583
+  in order to verify package integrity.
+*/
+
+[
+  {
+    "name": "apm",
+    "version": "8.0.0"
+  },
+  {
+    "name": "elastic_agent",
+    "version": "1.3.0"
+  },
+  {
+    "name": "endpoint",
+    "version": "1.4.1"
+  },
+  {
+    "name": "fleet_server",
+    "version": "1.1.0"
+  },
+  {
+    "name": "synthetics",
+    "version": "0.9.0"
+  }
+]

src/dev/build/args.test.ts

@@ -43,6 +43,7 @@ it('build default and oss dist for current platform, without packages, by defaul
         "initialize": true,
         "isRelease": false,
         "targetAllPlatforms": false,
+        "useSnapshotEpr": false,
         "versionQualifier": "",
       },
       "log": <ToolingLog>,
@@ -73,6 +74,7 @@ it('builds packages if --all-platforms is passed', () => {
         "initialize": true,
         "isRelease": false,
         "targetAllPlatforms": true,
+        "useSnapshotEpr": false,
         "versionQualifier": "",
       },
       "log": <ToolingLog>,
@@ -103,6 +105,7 @@ it('limits packages if --rpm passed with --all-platforms', () => {
         "initialize": true,
         "isRelease": false,
         "targetAllPlatforms": true,
+        "useSnapshotEpr": false,
         "versionQualifier": "",
       },
       "log": <ToolingLog>,
@@ -133,6 +136,7 @@ it('limits packages if --deb passed with --all-platforms', () => {
         "initialize": true,
         "isRelease": false,
         "targetAllPlatforms": true,
+        "useSnapshotEpr": false,
         "versionQualifier": "",
       },
       "log": <ToolingLog>,
@@ -164,6 +168,7 @@ it('limits packages if --docker passed with --all-platforms', () => {
         "initialize": true,
         "isRelease": false,
         "targetAllPlatforms": true,
+        "useSnapshotEpr": false,
         "versionQualifier": "",
       },
       "log": <ToolingLog>,
@@ -202,6 +207,7 @@ it('limits packages if --docker passed with --skip-docker-ubi and --all-platform
         "initialize": true,
         "isRelease": false,
         "targetAllPlatforms": true,
+        "useSnapshotEpr": false,
         "versionQualifier": "",
       },
       "log": <ToolingLog>,
@@ -233,6 +239,7 @@ it('limits packages if --all-platforms passed with --skip-docker-ubuntu', () =>
         "initialize": true,
         "isRelease": false,
         "targetAllPlatforms": true,
+        "useSnapshotEpr": false,
         "versionQualifier": "",
       },
       "log": <ToolingLog>,

src/dev/build/args.ts

@@ -40,6 +40,7 @@ export function readCliArgs(argv: string[]) {
       'silent',
       'debug',
       'help',
+      'use-snapshot-epr',
     ],
     alias: {
       v: 'verbose',
@@ -115,6 +116,7 @@ export function readCliArgs(argv: string[]) {
     createDockerUBI: isOsPackageDesired('docker-images') && !Boolean(flags['skip-docker-ubi']),
     createDockerContexts: !Boolean(flags['skip-docker-contexts']),
     targetAllPlatforms: Boolean(flags['all-platforms']),
+    useSnapshotEpr: Boolean(flags['use-snapshot-epr']),
   };
 
   return {

src/dev/build/build_distributables.ts

@@ -30,6 +30,7 @@ export interface BuildOptions {
   versionQualifier: string | undefined;
   targetAllPlatforms: boolean;
   createExamplePlugins: boolean;
+  useSnapshotEpr: boolean;
 }
 
 export async function buildDistributables(log: ToolingLog, options: BuildOptions): Promise<void> {
@@ -84,6 +85,7 @@ export async function buildDistributables(log: ToolingLog, options: BuildOptions
     await run(Tasks.CleanTypescript);
     await run(Tasks.CleanExtraFilesFromModules);
     await run(Tasks.CleanEmptyFolders);
+    await run(Tasks.BundleFleetPackages);
   }
 
   /**

src/dev/build/tasks/bundle_fleet_packages.ts

@@ -0,0 +1,104 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import axios from 'axios';
+import JSON5 from 'json5';
+
+// @ts-expect-error untyped internal module used to prevent axios from using xhr adapter in tests
+import AxiosHttpAdapter from 'axios/lib/adapters/http';
+
+import { ToolingLog } from '@kbn/dev-utils';
+import { closeSync, openSync, writeSync } from 'fs';
+import { dirname } from 'path';
+import { readCliArgs } from '../args';
+
+import { Task, read, mkdirp } from '../lib';
+
+const BUNDLED_PACKAGES_DIR = 'x-pack/plugins/fleet/server/bundled_packages';
+
+interface FleetPackage {
+  name: string;
+  version: string;
+}
+
+export const BundleFleetPackages: Task = {
+  description: 'Bundling fleet packages',
+
+  async run(config, log, build) {
+    log.info('Fetching fleet packages from package registry');
+    log.indent(4);
+
+    // Support the `--use-snapshot-epr` command line argument to fetch from the snapshot registry
+    // in development or test environments
+    const { buildOptions } = readCliArgs(process.argv);
+    const eprUrl = buildOptions?.useSnapshotEpr
+      ? 'https://epr-snapshot.elastic.co'
+      : 'https://epr.elastic.co';
+
+    const configFilePath = config.resolveFromRepo('fleet_packages.json');
+    const fleetPackages = (await read(configFilePath)) || '[]';
+
+    await Promise.all(
+      JSON5.parse(fleetPackages).map(async (fleetPackage: FleetPackage) => {
+        const archivePath = `${fleetPackage.name}-${fleetPackage.version}.zip`;
+        const archiveUrl = `${eprUrl}/epr/${fleetPackage.name}/${fleetPackage.name}-${fleetPackage.version}.zip`;
+
+        const destination = build.resolvePath(BUNDLED_PACKAGES_DIR, archivePath);
+
+        try {
+          await downloadPackageArchive({ log, url: archiveUrl, destination });
+        } catch (error) {
+          log.warning(`Failed to download bundled package archive ${archivePath}`);
+          log.warning(error);
+        }
+      })
+    );
+  },
+};
+
+/**
+ * We need to skip the checksum process on Fleet's bundled packages for now because we can't reliably generate
+ * a consistent checksum for the `.zip` file returned from the EPR service. This download process should be updated
+ * to verify packages using the proposed package signature field provided in https://github.com/elastic/elastic-package/issues/583
+ */
+async function downloadPackageArchive({
+  log,
+  url,
+  destination,
+}: {
+  log: ToolingLog;
+  url: string;
+  destination: string;
+}) {
+  log.info(`Downloading bundled package from ${url}`);
+
+  await mkdirp(dirname(destination));
+  const file = openSync(destination, 'w');
+
+  try {
+    const response = await axios.request({
+      url,
+      responseType: 'stream',
+      adapter: AxiosHttpAdapter,
+    });
+
+    await new Promise((resolve, reject) => {
+      response.data.on('data', (chunk: Buffer) => {
+        writeSync(file, chunk);
+      });
+
+      response.data.on('error', reject);
+      response.data.on('end', resolve);
+    });
+  } catch (error) {
+    log.warning(`Error downloading bundled package from ${url}`);
+    log.warning(error);
+  } finally {
+    closeSync(file);
+  }
+}

src/dev/build/tasks/index.ts

@@ -10,6 +10,7 @@ export * from './bin';
 export * from './build_kibana_platform_plugins';
 export * from './build_kibana_example_plugins';
 export * from './build_packages_task';
+export * from './bundle_fleet_packages';
 export * from './clean_tasks';
 export * from './copy_source_task';
 export * from './create_archives_sources_task';

src/dev/precommit_hook/casing_check_config.js

@@ -61,6 +61,9 @@ export const IGNORE_FILE_GLOBS = [
 
   'x-pack/plugins/maps/server/fonts/**/*',
 
+  // Bundled package names typically use a format like ${pkgName}-${pkgVersion}, so don't lint them
+  'x-pack/plugins/fleet/server/bundled_packages/**/*',
+
   // Bazel default files
   '**/WORKSPACE.bazel',
   '**/BUILD.bazel',

x-pack/plugins/fleet/common/constants/epm.ts

@@ -60,4 +60,5 @@ export const installationStatuses = {
   Installing: 'installing',
   InstallFailed: 'install_failed',
   NotInstalled: 'not_installed',
+  InstalledBundled: 'installed_bundled',
 } as const;

x-pack/plugins/fleet/common/types/models/epm.ts

@@ -45,7 +45,11 @@ export interface DefaultPackagesInstallationError {
 export type InstallType = 'reinstall' | 'reupdate' | 'rollback' | 'update' | 'install' | 'unknown';
 export type InstallSource = 'registry' | 'upload';
 
-export type EpmPackageInstallStatus = 'installed' | 'installing' | 'install_failed';
+export type EpmPackageInstallStatus =
+  | 'installed'
+  | 'installing'
+  | 'install_failed'
+  | 'installed_bundled';
 
 export type DetailViewPanelName = 'overview' | 'policies' | 'assets' | 'settings' | 'custom';
 export type ServiceName = 'kibana' | 'elasticsearch';
@@ -410,13 +414,22 @@ export interface PackageUsageStats {
   agent_policy_count: number;
 }
 
-export type Installable<T> = Installed<T> | Installing<T> | NotInstalled<T> | InstallFailed<T>;
+export type Installable<T> =
+  | InstalledRegistry<T>
+  | Installing<T>
+  | NotInstalled<T>
+  | InstallFailed<T>
+  | InstalledBundled<T>;
 
-export type Installed<T = {}> = T & {
+export type InstalledRegistry<T = {}> = T & {
   status: InstallationStatus['Installed'];
   savedObject: SavedObject<Installation>;
 };
 
+export type InstalledBundled<T = {}> = T & {
+  status: InstallationStatus['InstalledBundled'];
+};
+
 export type Installing<T = {}> = T & {
   status: InstallationStatus['Installing'];
   savedObject: SavedObject<Installation>;

x-pack/plugins/fleet/server/services/epm/archive/validation.ts

@@ -209,9 +209,9 @@ export function parseAndVerifyDataStreams(
       streams: manifestStreams,
       ...restOfProps
     } = manifest;
-    if (!(dataStreamTitle && release && type)) {
+    if (!(dataStreamTitle && type)) {
       throw new PackageInvalidArchiveError(
-        `Invalid manifest for data stream '${dataStreamPath}': one or more fields missing of 'title', 'release', 'type'`
+        `Invalid manifest for data stream '${dataStreamPath}': one or more fields missing of 'title', 'type'`
       );
     }
     const streams = parseAndVerifyStreams(manifestStreams, dataStreamPath);