From 1d75cc67d1b81c42d833dd2403abde0e1d6818b9 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Wed, 9 Feb 2022 13:06:17 -0500 Subject: [PATCH 01/32] Allow bundled installs to occur even if EPR is unreachable --- x-pack/plugins/fleet/server/errors/index.ts | 1 + .../epm/packages/bulk_install_packages.ts | 32 +++++++++++++++++-- .../epm/packages/get_bundled_packages.ts | 7 +++- .../server/services/epm/registry/index.ts | 2 +- .../server/services/epm/registry/requests.ts | 15 +++++---- 5 files changed, 46 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/fleet/server/errors/index.ts b/x-pack/plugins/fleet/server/errors/index.ts index 380862d36fe01..0d8627c13b3dc 100644 --- a/x-pack/plugins/fleet/server/errors/index.ts +++ b/x-pack/plugins/fleet/server/errors/index.ts @@ -43,6 +43,7 @@ export class ConcurrentInstallOperationError extends IngestManagerError {} export class AgentReassignmentError extends IngestManagerError {} export class PackagePolicyIneligibleForUpgradeError extends IngestManagerError {} export class PackagePolicyValidationError extends IngestManagerError {} +export class BundledPackageNotFoundError extends IngestManagerError {} export class HostedAgentPolicyRestrictionRelatedError extends IngestManagerError { constructor(message = 'Cannot perform that action') { super( diff --git a/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts b/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts index a32809672e1b4..cdebccb054c3b 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts @@ -12,6 +12,8 @@ import * as Registry from '../registry'; import type { InstallResult } from '../../../types'; +import { BundledPackageNotFoundError } from '../../../errors'; + import { installPackage, isPackageVersionOrLaterInstalled } from './install'; import type { BulkInstallResponse, IBulkInstallPackageError } from './install'; import { getBundledPackages } from './get_bundled_packages'; @@ -39,8 +41,32 @@ export async function bulkInstallPackages({ const packagesResults = await Promise.allSettled( packagesToInstall.map((pkg) => { - if (typeof pkg === 'string') return Registry.fetchFindLatestPackage(pkg); - return Promise.resolve(pkg); + if (typeof pkg !== 'string') { + return Promise.resolve(pkg); + } + + return Registry.fetchFindLatestPackage(pkg).catch((error) => { + // For registry preferred bulk installs, consider an inability to reach the configured + // registry a fotal error + if (preferredSource === 'registry') { + return Promise.reject(error); + } + + // For bundled preferred bulk installs, we want to allow the installation to continue, even + // if the registry is not reachable + const bundledPackage = bundledPackages.find((b) => b.name === pkg); + + if (!bundledPackage) { + return Promise.reject( + new BundledPackageNotFoundError(`No bundled package found with name ${pkg}`) + ); + } + + return Promise.resolve({ + name: bundledPackage.name, + version: bundledPackage.version, + }); + }); }) ); @@ -86,7 +112,7 @@ export async function bulkInstallPackages({ let installResult: InstallResult; const pkgkey = Registry.pkgToPkgKey(pkgKeyProps); - const bundledPackage = bundledPackages.find((pkg) => pkg.name === pkgkey); + const bundledPackage = bundledPackages.find((pkg) => `${pkg.name}-${pkg.version}` === pkgkey); // If preferred source is bundled packages on disk, attempt to install from disk first, then fall back to registry if (preferredSource === 'bundled') { diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get_bundled_packages.ts b/x-pack/plugins/fleet/server/services/epm/packages/get_bundled_packages.ts index a9f9b754640cb..a7bdda0d68c7e 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get_bundled_packages.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get_bundled_packages.ts @@ -9,11 +9,13 @@ import path from 'path'; import fs from 'fs/promises'; import { appContextService } from '../../app_context'; +import { splitPkgKey } from '../registry'; const BUNDLED_PACKAGE_DIRECTORY = path.join(__dirname, '../../../bundled_packages'); interface BundledPackage { name: string; + version: string; buffer: Buffer; } @@ -26,8 +28,11 @@ export async function getBundledPackages(): Promise { zipFiles.map(async (zipFile) => { const file = await fs.readFile(path.join(BUNDLED_PACKAGE_DIRECTORY, zipFile)); + const { pkgName, pkgVersion } = splitPkgKey(zipFile.replace(/\.zip$/, '')); + return { - name: zipFile.replace(/\.zip$/, ''), + name: pkgName, + version: pkgVersion, buffer: file, }; }) diff --git a/x-pack/plugins/fleet/server/services/epm/registry/index.ts b/x-pack/plugins/fleet/server/services/epm/registry/index.ts index 8cfb2844159bc..6564114f9cd3f 100644 --- a/x-pack/plugins/fleet/server/services/epm/registry/index.ts +++ b/x-pack/plugins/fleet/server/services/epm/registry/index.ts @@ -71,7 +71,7 @@ export async function fetchFindLatestPackage(packageName: string): Promise { +export async function getResponse(url: string, retries: number = 5): Promise { try { // we only want to retry certain failures like network issues // the rest should only try the one time then fail as they do now const response = await pRetry(() => registryFetch(url), { factor: 2, - retries: 5, + retries, onFailedAttempt: (error) => { // we only want to retry certain types of errors, like `ECONNREFUSED` and other operational errors // and let the others through without retrying @@ -67,13 +67,16 @@ export async function getResponse(url: string): Promise { } } -export async function getResponseStream(url: string): Promise { - const res = await getResponse(url); +export async function getResponseStream( + url: string, + retries?: number +): Promise { + const res = await getResponse(url, retries); return res.body; } -export async function fetchUrl(url: string): Promise { - return getResponseStream(url).then(streamToString); +export async function fetchUrl(url: string, retries?: number): Promise { + return getResponseStream(url, retries).then(streamToString); } // node-fetch throws a FetchError for those types of errors and From 68ad69a4076abbba7645d4d4b5d40163132b8686 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Wed, 9 Feb 2022 13:36:31 -0500 Subject: [PATCH 02/32] Fix type errors in test --- x-pack/plugins/fleet/server/services/preconfiguration.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts index 6d6d641381da2..129eac19ff7a6 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts @@ -743,11 +743,13 @@ describe('policy preconfiguration', () => { mockedGetBundledPackages.mockResolvedValue([ { name: 'test_package', + version: '1.0.0', buffer: Buffer.from('test_package'), }, { name: 'test_package_2', + version: '1.0.0', buffer: Buffer.from('test_package_2'), }, ]); @@ -784,6 +786,7 @@ describe('policy preconfiguration', () => { mockedGetBundledPackages.mockResolvedValue([ { name: 'test_package', + version: '1.0.0', buffer: Buffer.from('test_package'), }, ]); @@ -823,6 +826,7 @@ describe('policy preconfiguration', () => { mockedGetBundledPackages.mockResolvedValue([ { name: 'test_package', + version: '1.0.0', buffer: Buffer.from('test_package'), }, ]); From c69c92a0984916fccd78476303a9f9a972a6de51 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Wed, 9 Feb 2022 14:44:49 -0500 Subject: [PATCH 03/32] Fix failing test --- x-pack/plugins/fleet/server/services/preconfiguration.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts index 129eac19ff7a6..603fadc011b24 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts @@ -166,7 +166,7 @@ jest.mock('./epm/packages/install', () => ({ const installedPackage = mockInstalledPackages.get(pkgName); - if (installedPackage) { + if (installedPackage && installedPackage.version === '1.0.0') { return installedPackage; } From c09df4f085359d24b7ea1b3ada86a060b9c3fc67 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Wed, 9 Feb 2022 14:45:16 -0500 Subject: [PATCH 04/32] fixup! Fix failing test --- x-pack/plugins/fleet/server/services/preconfiguration.test.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts index 603fadc011b24..a192be59698eb 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts @@ -166,10 +166,6 @@ jest.mock('./epm/packages/install', () => ({ const installedPackage = mockInstalledPackages.get(pkgName); - if (installedPackage && installedPackage.version === '1.0.0') { - return installedPackage; - } - // Just install every bundled package at version '1.0.0' const packageInstallation = { name: pkgName, version: '1.0.0', title: pkgName }; mockInstalledPackages.set(pkgName, packageInstallation); From 8056bba398ea848cf5fcb48bae026f6c8e9a71f7 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Wed, 9 Feb 2022 14:45:47 -0500 Subject: [PATCH 05/32] Remove unused object in mock --- x-pack/plugins/fleet/server/services/preconfiguration.test.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts index a192be59698eb..e0ebec5fc8e82 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts @@ -164,8 +164,6 @@ jest.mock('./epm/packages/install', () => ({ // Treat the buffer value passed in tests as the package's name for simplicity const pkgName = archiveBuffer.toString('utf8'); - const installedPackage = mockInstalledPackages.get(pkgName); - // Just install every bundled package at version '1.0.0' const packageInstallation = { name: pkgName, version: '1.0.0', title: pkgName }; mockInstalledPackages.set(pkgName, packageInstallation); From aba45c70131110007ba82e01e2df63060a726a03 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Thu, 10 Feb 2022 09:04:38 -0500 Subject: [PATCH 06/32] Make creation of preconfigured agent policy functional --- .../plugins/fleet/common/types/models/epm.ts | 2 +- .../agent_policies/full_agent_policy.ts | 9 ++- .../fleet/server/services/agent_policy.ts | 17 +++-- .../fleet/server/services/epm/packages/get.ts | 32 +++++++++- .../services/managed_package_policies.ts | 4 +- .../package_policies_to_agent_permissions.ts | 4 +- .../fleet/server/services/package_policy.ts | 62 ++++++++++++++----- .../fleet/server/services/preconfiguration.ts | 10 ++- 8 files changed, 111 insertions(+), 29 deletions(-) diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index 983ee7fff3db1..5ea4f86f39a43 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -107,7 +107,7 @@ export type InstallablePackage = RegistryPackage | ArchivePackage; export type ArchivePackage = PackageSpecManifest & // should an uploaded package be able to specify `internal`? - Pick; + Pick; export type RegistryPackage = PackageSpecManifest & Partial & diff --git a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts index 0bb6b4a33f6a3..edd29fd5d1af0 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts @@ -31,10 +31,11 @@ import { getMonitoringPermissions } from './monitoring_permissions'; export async function getFullAgentPolicy( soClient: SavedObjectsClientContract, id: string, - options?: { standalone: boolean } + options: Partial<{ standalone: boolean; useBundledPackages: boolean }> = {} ): Promise { let agentPolicy; - const standalone = options?.standalone; + const standalone = options.standalone ?? false; + const useBundledPackages = options.useBundledPackages ?? false; try { agentPolicy = await agentPolicyService.get(soClient, id); @@ -111,7 +112,9 @@ export async function getFullAgentPolicy( }; const dataPermissions = - (await storedPackagePoliciesToAgentPermissions(soClient, agentPolicy.package_policies)) || {}; + (await storedPackagePoliciesToAgentPermissions(soClient, agentPolicy.package_policies, { + useBundledPackages, + })) || {}; dataPermissions._elastic_agent_checks = { cluster: DEFAULT_CLUSTER_PERMISSIONS, diff --git a/x-pack/plugins/fleet/server/services/agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policy.ts index d473504622bad..4f37b9672bb0a 100644 --- a/x-pack/plugins/fleet/server/services/agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policy.ts @@ -602,7 +602,11 @@ class AgentPolicyService { }; } - public async deployPolicy(soClient: SavedObjectsClientContract, agentPolicyId: string) { + public async deployPolicy( + soClient: SavedObjectsClientContract, + agentPolicyId: string, + useBundledPackages: boolean = false + ) { // Use internal ES client so we have permissions to write to .fleet* indices const esClient = appContextService.getInternalUserESClient(); const defaultOutputId = await outputService.getDefaultDataOutputId(soClient); @@ -612,7 +616,9 @@ class AgentPolicyService { } const policy = await agentPolicyService.get(soClient, agentPolicyId); - const fullPolicy = await agentPolicyService.getFullAgentPolicy(soClient, agentPolicyId); + const fullPolicy = await agentPolicyService.getFullAgentPolicy(soClient, agentPolicyId, { + useBundledPackages, + }); if (!policy || !fullPolicy || !fullPolicy.revision) { return; } @@ -715,7 +721,7 @@ class AgentPolicyService { public async getFullAgentPolicy( soClient: SavedObjectsClientContract, id: string, - options?: { standalone: boolean } + options: Partial<{ standalone: boolean; useBundledPackages: boolean }> = {} ): Promise { return getFullAgentPolicy(soClient, id, options); } @@ -733,12 +739,14 @@ export async function addPackageToAgentPolicy( packagePolicyId?: string | number, packagePolicyDescription?: string, transformPackagePolicy?: (p: NewPackagePolicy) => NewPackagePolicy, - bumpAgentPolicyRevison = false + bumpAgentPolicyRevison = false, + useBundledPackages = false ) { const packageInfo = await getPackageInfo({ savedObjectsClient: soClient, pkgName: packageToInstall.name, pkgVersion: packageToInstall.version, + useBundledPackages, }); const basePackagePolicy = packageToPackagePolicy( @@ -767,5 +775,6 @@ export async function addPackageToAgentPolicy( skipUniqueNameVerification: true, overwrite: true, force: true, // To add package to managed policy we need the force flag + useBundledPackages, }); } diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get.ts b/x-pack/plugins/fleet/server/services/epm/packages/get.ts index 755fafd8fbc7a..bab6ac249230e 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get.ts @@ -21,7 +21,7 @@ import type { GetCategoriesRequest, } from '../../../../common/types'; import type { Installation, PackageInfo } from '../../../types'; -import { IngestManagerError } from '../../../errors'; +import { BundledPackageNotFoundError, IngestManagerError } from '../../../errors'; import { appContextService } from '../../'; import * as Registry from '../registry'; import { getEsPackage } from '../archive/storage'; @@ -29,6 +29,7 @@ import { getArchivePackage } from '../archive'; import { normalizeKuery } from '../../saved_object'; import { createInstallableFrom } from './index'; +import { getBundledPackages } from './get_bundled_packages'; export type { SearchParams } from '../registry'; export { getFile } from '../registry'; @@ -102,11 +103,36 @@ export async function getPackageInfo(options: { savedObjectsClient: SavedObjectsClientContract; pkgName: string; pkgVersion: string; + useBundledPackages?: boolean; }): Promise { - const { savedObjectsClient, pkgName, pkgVersion } = options; + const { savedObjectsClient, pkgName, pkgVersion, useBundledPackages = false } = options; const [savedObject, latestPackage] = await Promise.all([ getInstallationObject({ savedObjectsClient, pkgName }), - Registry.fetchFindLatestPackage(pkgName), + new Promise<{ name: string; version: string }>((resolve, reject) => { + if (!useBundledPackages) { + resolve(Registry.fetchFindLatestPackage(pkgName)); + } + + return Registry.fetchFindLatestPackage(pkgName).catch(async (error) => { + const bundledPackages = await getBundledPackages(); + const bundledPackage = bundledPackages.find( + (b) => b.name === pkgName && b.version === pkgVersion + ); + + if (!bundledPackage) { + return reject( + new BundledPackageNotFoundError( + `No bundled package found with key ${pkgName}-${pkgVersion}` + ) + ); + } + + resolve({ + name: bundledPackage.name, + version: bundledPackage.version, + }); + }); + }), ]); // If no package version is provided, use the installed version in the response diff --git a/x-pack/plugins/fleet/server/services/managed_package_policies.ts b/x-pack/plugins/fleet/server/services/managed_package_policies.ts index c702cfe96d986..e2b3fb10a4b7c 100644 --- a/x-pack/plugins/fleet/server/services/managed_package_policies.ts +++ b/x-pack/plugins/fleet/server/services/managed_package_policies.ts @@ -31,7 +31,8 @@ export interface UpgradeManagedPackagePoliciesResult { export const upgradeManagedPackagePolicies = async ( soClient: SavedObjectsClientContract, esClient: ElasticsearchClient, - packagePolicyIds: string[] + packagePolicyIds: string[], + useBundledPackages = false ): Promise => { const results: UpgradeManagedPackagePoliciesResult[] = []; @@ -46,6 +47,7 @@ export const upgradeManagedPackagePolicies = async ( savedObjectsClient: soClient, pkgName: packagePolicy.package.name, pkgVersion: packagePolicy.package.version, + useBundledPackages, }); const installedPackage = await getInstallation({ diff --git a/x-pack/plugins/fleet/server/services/package_policies_to_agent_permissions.ts b/x-pack/plugins/fleet/server/services/package_policies_to_agent_permissions.ts index d361c583c8f6c..70bc87724336e 100644 --- a/x-pack/plugins/fleet/server/services/package_policies_to_agent_permissions.ts +++ b/x-pack/plugins/fleet/server/services/package_policies_to_agent_permissions.ts @@ -15,7 +15,8 @@ export const DEFAULT_CLUSTER_PERMISSIONS = ['monitor']; export async function storedPackagePoliciesToAgentPermissions( soClient: SavedObjectsClientContract, - packagePolicies: string[] | PackagePolicy[] + packagePolicies: string[] | PackagePolicy[], + options?: { useBundledPackages?: boolean } ): Promise { if (packagePolicies.length === 0) { return; @@ -38,6 +39,7 @@ export async function storedPackagePoliciesToAgentPermissions( savedObjectsClient: soClient, pkgName: packagePolicy.package.name, pkgVersion: packagePolicy.package.version, + useBundledPackages: options?.useBundledPackages ?? false, }); if (!pkg.data_streams || pkg.data_streams.length === 0) { diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index 69d7a81a24efd..5e61b9fbe3e4f 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -43,6 +43,7 @@ import type { ListResult, UpgradePackagePolicyDryRunResponseItem, RegistryDataStream, + InstallablePackage, } from '../../common'; import { PACKAGE_POLICY_SAVED_OBJECT_TYPE } from '../constants'; import { @@ -50,6 +51,7 @@ import { ingestErrorToResponseOptions, PackagePolicyIneligibleForUpgradeError, PackagePolicyValidationError, + PackageCacheError, } from '../errors'; import { NewPackagePolicySchema, UpdatePackagePolicySchema } from '../types'; import type { @@ -57,7 +59,6 @@ import type { UpdatePackagePolicy, PackagePolicy, PackagePolicySOAttributes, - RegistryPackage, DryRunPackagePolicy, } from '../types'; import type { ExternalCallback } from '..'; @@ -73,6 +74,7 @@ import { appContextService } from '.'; import { removeOldAssets } from './epm/packages/cleanup'; import type { PackageUpdateEvent, UpdateEventType } from './upgrade_sender'; import { sendTelemetryEvents } from './upgrade_sender'; +import { getArchivePackage } from './epm/archive'; export type InputsOverride = Partial & { vars?: Array; @@ -103,6 +105,7 @@ class PackagePolicyService { skipEnsureInstalled?: boolean; skipUniqueNameVerification?: boolean; overwrite?: boolean; + useBundledPackages?: boolean; } ): Promise { if (!options?.skipUniqueNameVerification) { @@ -132,9 +135,11 @@ class PackagePolicyService { savedObjectsClient: soClient, pkgName: packagePolicy.package.name, pkgVersion: packagePolicy.package.version, + useBundledPackages: options?.useBundledPackages, }); - let pkgInfo; + let pkgInfo: PackageInfo; + if (options?.skipEnsureInstalled) pkgInfo = await pkgInfoPromise; else { const [, packageInfo] = await Promise.all([ @@ -162,16 +167,35 @@ class PackagePolicyService { } validatePackagePolicyOrThrow(packagePolicy, pkgInfo); - const registryPkgInfo = await Registry.fetchInfo(pkgInfo.name, pkgInfo.version); + const installablePackage: InstallablePackage = await Registry.fetchInfo( + pkgInfo.name, + pkgInfo.version + ).catch(async (error) => { + if (!options?.useBundledPackages) { + throw error; + } + + // If we're using bundled packages, try to pull the package info from ES instead + const archivePackage = await getArchivePackage({ + name: pkgInfo.name, + version: pkgInfo.version, + }); + + if (!archivePackage) { + throw new PackageCacheError('Unable to find bundled package in cache'); + } + + return archivePackage?.packageInfo; + }); inputs = await this._compilePackagePolicyInputs( - registryPkgInfo, + installablePackage, pkgInfo, packagePolicy.vars || {}, inputs ); - elasticsearch = registryPkgInfo.elasticsearch; + elasticsearch = installablePackage.elasticsearch; } const isoDate = new Date().toISOString(); @@ -798,14 +822,24 @@ class PackagePolicyService { } public async _compilePackagePolicyInputs( - registryPkgInfo: RegistryPackage, + installablePackage: InstallablePackage, pkgInfo: PackageInfo, vars: PackagePolicy['vars'], inputs: PackagePolicyInput[] ): Promise { const inputsPromises = inputs.map(async (input) => { - const compiledInput = await _compilePackagePolicyInput(registryPkgInfo, pkgInfo, vars, input); - const compiledStreams = await _compilePackageStreams(registryPkgInfo, pkgInfo, vars, input); + const compiledInput = await _compilePackagePolicyInput( + installablePackage, + pkgInfo, + vars, + input + ); + const compiledStreams = await _compilePackageStreams( + installablePackage, + pkgInfo, + vars, + input + ); return { ...input, compiled_input: compiledInput, @@ -916,7 +950,7 @@ function assignStreamIdToInput(packagePolicyId: string, input: NewPackagePolicyI } async function _compilePackagePolicyInput( - registryPkgInfo: RegistryPackage, + installablePackage: InstallablePackage, pkgInfo: PackageInfo, vars: PackagePolicy['vars'], input: PackagePolicyInput @@ -941,7 +975,7 @@ async function _compilePackagePolicyInput( return undefined; } - const [pkgInputTemplate] = await getAssetsData(registryPkgInfo, (path: string) => + const [pkgInputTemplate] = await getAssetsData(installablePackage, (path: string) => path.endsWith(`/agent/input/${packageInput.template_path!}`) ); @@ -957,13 +991,13 @@ async function _compilePackagePolicyInput( } async function _compilePackageStreams( - registryPkgInfo: RegistryPackage, + installablePackage: InstallablePackage, pkgInfo: PackageInfo, vars: PackagePolicy['vars'], input: PackagePolicyInput ) { const streamsPromises = input.streams.map((stream) => - _compilePackageStream(registryPkgInfo, pkgInfo, vars, input, stream) + _compilePackageStream(installablePackage, pkgInfo, vars, input, stream) ); return await Promise.all(streamsPromises); @@ -1006,7 +1040,7 @@ export function _applyIndexPrivileges( } async function _compilePackageStream( - registryPkgInfo: RegistryPackage, + installablePackage: InstallablePackage, pkgInfo: PackageInfo, vars: PackagePolicy['vars'], input: PackagePolicyInput, @@ -1049,7 +1083,7 @@ async function _compilePackageStream( const datasetPath = packageDataStream.path; const [pkgStreamTemplate] = await getAssetsData( - registryPkgInfo, + installablePackage, (path: string) => path.endsWith(streamFromPkg.template_path), datasetPath ); diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.ts b/x-pack/plugins/fleet/server/services/preconfiguration.ts index e9d97856a926f..c6da3e91a94a5 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.ts @@ -369,7 +369,8 @@ export async function ensurePreconfiguredPackagesAndPolicies( const packagePolicyUpgradeResults = await upgradeManagedPackagePolicies( soClient, esClient, - allPackagePolicyIds.items + allPackagePolicyIds.items, + true ); return { @@ -424,10 +425,14 @@ async function addPreconfiguredPolicyPackages( ) { // Add packages synchronously to avoid overwriting for (const { installedPackage, id, name, description, inputs } of installedPackagePolicies) { + // Allow requests to the registry to fall back to bundled packages if no registry is reachable in the calls below + const useBundledPackages = true; + const packageInfo = await getPackageInfo({ savedObjectsClient: soClient, pkgName: installedPackage.name, pkgVersion: installedPackage.version, + useBundledPackages, }); await addPackageToAgentPolicy( @@ -440,7 +445,8 @@ async function addPreconfiguredPolicyPackages( id, description, (policy) => preconfigurePackageInputs(policy, packageInfo, inputs), - bumpAgentPolicyRevison + bumpAgentPolicyRevison, + useBundledPackages ); } } From 928b7ef7ced6e6c372e54f8e5eaccf7e91222bdb Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Thu, 10 Feb 2022 09:36:42 -0500 Subject: [PATCH 07/32] Always fall back to bundled packages if available --- .../agent_policies/full_agent_policy.ts | 9 +++------ .../fleet/server/services/agent_policy.ts | 17 ++++------------- .../fleet/server/services/epm/packages/get.ts | 14 +++----------- .../server/services/managed_package_policies.ts | 4 +--- .../package_policies_to_agent_permissions.ts | 4 +--- .../fleet/server/services/package_policy.ts | 13 ++++--------- .../fleet/server/services/preconfiguration.ts | 10 ++-------- 7 files changed, 18 insertions(+), 53 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts index edd29fd5d1af0..f0ee7a532c093 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts @@ -31,11 +31,10 @@ import { getMonitoringPermissions } from './monitoring_permissions'; export async function getFullAgentPolicy( soClient: SavedObjectsClientContract, id: string, - options: Partial<{ standalone: boolean; useBundledPackages: boolean }> = {} + options?: { standalone: boolean } ): Promise { let agentPolicy; - const standalone = options.standalone ?? false; - const useBundledPackages = options.useBundledPackages ?? false; + const standalone = options?.standalone ?? false; try { agentPolicy = await agentPolicyService.get(soClient, id); @@ -112,9 +111,7 @@ export async function getFullAgentPolicy( }; const dataPermissions = - (await storedPackagePoliciesToAgentPermissions(soClient, agentPolicy.package_policies, { - useBundledPackages, - })) || {}; + (await storedPackagePoliciesToAgentPermissions(soClient, agentPolicy.package_policies)) || {}; dataPermissions._elastic_agent_checks = { cluster: DEFAULT_CLUSTER_PERMISSIONS, diff --git a/x-pack/plugins/fleet/server/services/agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policy.ts index 4f37b9672bb0a..d473504622bad 100644 --- a/x-pack/plugins/fleet/server/services/agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policy.ts @@ -602,11 +602,7 @@ class AgentPolicyService { }; } - public async deployPolicy( - soClient: SavedObjectsClientContract, - agentPolicyId: string, - useBundledPackages: boolean = false - ) { + public async deployPolicy(soClient: SavedObjectsClientContract, agentPolicyId: string) { // Use internal ES client so we have permissions to write to .fleet* indices const esClient = appContextService.getInternalUserESClient(); const defaultOutputId = await outputService.getDefaultDataOutputId(soClient); @@ -616,9 +612,7 @@ class AgentPolicyService { } const policy = await agentPolicyService.get(soClient, agentPolicyId); - const fullPolicy = await agentPolicyService.getFullAgentPolicy(soClient, agentPolicyId, { - useBundledPackages, - }); + const fullPolicy = await agentPolicyService.getFullAgentPolicy(soClient, agentPolicyId); if (!policy || !fullPolicy || !fullPolicy.revision) { return; } @@ -721,7 +715,7 @@ class AgentPolicyService { public async getFullAgentPolicy( soClient: SavedObjectsClientContract, id: string, - options: Partial<{ standalone: boolean; useBundledPackages: boolean }> = {} + options?: { standalone: boolean } ): Promise { return getFullAgentPolicy(soClient, id, options); } @@ -739,14 +733,12 @@ export async function addPackageToAgentPolicy( packagePolicyId?: string | number, packagePolicyDescription?: string, transformPackagePolicy?: (p: NewPackagePolicy) => NewPackagePolicy, - bumpAgentPolicyRevison = false, - useBundledPackages = false + bumpAgentPolicyRevison = false ) { const packageInfo = await getPackageInfo({ savedObjectsClient: soClient, pkgName: packageToInstall.name, pkgVersion: packageToInstall.version, - useBundledPackages, }); const basePackagePolicy = packageToPackagePolicy( @@ -775,6 +767,5 @@ export async function addPackageToAgentPolicy( skipUniqueNameVerification: true, overwrite: true, force: true, // To add package to managed policy we need the force flag - useBundledPackages, }); } diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get.ts b/x-pack/plugins/fleet/server/services/epm/packages/get.ts index bab6ac249230e..991a3d6f061b6 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get.ts @@ -103,28 +103,20 @@ export async function getPackageInfo(options: { savedObjectsClient: SavedObjectsClientContract; pkgName: string; pkgVersion: string; - useBundledPackages?: boolean; }): Promise { - const { savedObjectsClient, pkgName, pkgVersion, useBundledPackages = false } = options; + const { savedObjectsClient, pkgName, pkgVersion } = options; const [savedObject, latestPackage] = await Promise.all([ getInstallationObject({ savedObjectsClient, pkgName }), new Promise<{ name: string; version: string }>((resolve, reject) => { - if (!useBundledPackages) { - resolve(Registry.fetchFindLatestPackage(pkgName)); - } - return Registry.fetchFindLatestPackage(pkgName).catch(async (error) => { const bundledPackages = await getBundledPackages(); const bundledPackage = bundledPackages.find( (b) => b.name === pkgName && b.version === pkgVersion ); + // If we don't find a bundled package, reject with the original error if (!bundledPackage) { - return reject( - new BundledPackageNotFoundError( - `No bundled package found with key ${pkgName}-${pkgVersion}` - ) - ); + return reject(error); } resolve({ diff --git a/x-pack/plugins/fleet/server/services/managed_package_policies.ts b/x-pack/plugins/fleet/server/services/managed_package_policies.ts index e2b3fb10a4b7c..c702cfe96d986 100644 --- a/x-pack/plugins/fleet/server/services/managed_package_policies.ts +++ b/x-pack/plugins/fleet/server/services/managed_package_policies.ts @@ -31,8 +31,7 @@ export interface UpgradeManagedPackagePoliciesResult { export const upgradeManagedPackagePolicies = async ( soClient: SavedObjectsClientContract, esClient: ElasticsearchClient, - packagePolicyIds: string[], - useBundledPackages = false + packagePolicyIds: string[] ): Promise => { const results: UpgradeManagedPackagePoliciesResult[] = []; @@ -47,7 +46,6 @@ export const upgradeManagedPackagePolicies = async ( savedObjectsClient: soClient, pkgName: packagePolicy.package.name, pkgVersion: packagePolicy.package.version, - useBundledPackages, }); const installedPackage = await getInstallation({ diff --git a/x-pack/plugins/fleet/server/services/package_policies_to_agent_permissions.ts b/x-pack/plugins/fleet/server/services/package_policies_to_agent_permissions.ts index 70bc87724336e..d361c583c8f6c 100644 --- a/x-pack/plugins/fleet/server/services/package_policies_to_agent_permissions.ts +++ b/x-pack/plugins/fleet/server/services/package_policies_to_agent_permissions.ts @@ -15,8 +15,7 @@ export const DEFAULT_CLUSTER_PERMISSIONS = ['monitor']; export async function storedPackagePoliciesToAgentPermissions( soClient: SavedObjectsClientContract, - packagePolicies: string[] | PackagePolicy[], - options?: { useBundledPackages?: boolean } + packagePolicies: string[] | PackagePolicy[] ): Promise { if (packagePolicies.length === 0) { return; @@ -39,7 +38,6 @@ export async function storedPackagePoliciesToAgentPermissions( savedObjectsClient: soClient, pkgName: packagePolicy.package.name, pkgVersion: packagePolicy.package.version, - useBundledPackages: options?.useBundledPackages ?? false, }); if (!pkg.data_streams || pkg.data_streams.length === 0) { diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index 5e61b9fbe3e4f..8a59d2a70743d 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -51,7 +51,6 @@ import { ingestErrorToResponseOptions, PackagePolicyIneligibleForUpgradeError, PackagePolicyValidationError, - PackageCacheError, } from '../errors'; import { NewPackagePolicySchema, UpdatePackagePolicySchema } from '../types'; import type { @@ -105,7 +104,6 @@ class PackagePolicyService { skipEnsureInstalled?: boolean; skipUniqueNameVerification?: boolean; overwrite?: boolean; - useBundledPackages?: boolean; } ): Promise { if (!options?.skipUniqueNameVerification) { @@ -135,7 +133,6 @@ class PackagePolicyService { savedObjectsClient: soClient, pkgName: packagePolicy.package.name, pkgVersion: packagePolicy.package.version, - useBundledPackages: options?.useBundledPackages, }); let pkgInfo: PackageInfo; @@ -171,18 +168,16 @@ class PackagePolicyService { pkgInfo.name, pkgInfo.version ).catch(async (error) => { - if (!options?.useBundledPackages) { - throw error; - } - - // If we're using bundled packages, try to pull the package info from ES instead + // Attempt to get an ArchivePackage object from the package info cache if the registry + // is not reachable. This will pull a bundled package if it was previously installed. const archivePackage = await getArchivePackage({ name: pkgInfo.name, version: pkgInfo.version, }); + // If we don't find a package in the cache, throw the original error if (!archivePackage) { - throw new PackageCacheError('Unable to find bundled package in cache'); + throw error; } return archivePackage?.packageInfo; diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.ts b/x-pack/plugins/fleet/server/services/preconfiguration.ts index c6da3e91a94a5..e9d97856a926f 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.ts @@ -369,8 +369,7 @@ export async function ensurePreconfiguredPackagesAndPolicies( const packagePolicyUpgradeResults = await upgradeManagedPackagePolicies( soClient, esClient, - allPackagePolicyIds.items, - true + allPackagePolicyIds.items ); return { @@ -425,14 +424,10 @@ async function addPreconfiguredPolicyPackages( ) { // Add packages synchronously to avoid overwriting for (const { installedPackage, id, name, description, inputs } of installedPackagePolicies) { - // Allow requests to the registry to fall back to bundled packages if no registry is reachable in the calls below - const useBundledPackages = true; - const packageInfo = await getPackageInfo({ savedObjectsClient: soClient, pkgName: installedPackage.name, pkgVersion: installedPackage.version, - useBundledPackages, }); await addPackageToAgentPolicy( @@ -445,8 +440,7 @@ async function addPreconfiguredPolicyPackages( id, description, (policy) => preconfigurePackageInputs(policy, packageInfo, inputs), - bumpAgentPolicyRevison, - useBundledPackages + bumpAgentPolicyRevison ); } } From 13e218d3290e4445a75d58b281eef8d9b0827dac Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Thu, 10 Feb 2022 10:04:36 -0500 Subject: [PATCH 08/32] Remove unused import --- x-pack/plugins/fleet/server/services/epm/packages/get.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get.ts b/x-pack/plugins/fleet/server/services/epm/packages/get.ts index 991a3d6f061b6..c3f8a22deff20 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get.ts @@ -21,7 +21,7 @@ import type { GetCategoriesRequest, } from '../../../../common/types'; import type { Installation, PackageInfo } from '../../../types'; -import { BundledPackageNotFoundError, IngestManagerError } from '../../../errors'; +import { IngestManagerError } from '../../../errors'; import { appContextService } from '../../'; import * as Registry from '../registry'; import { getEsPackage } from '../archive/storage'; From e2f2fead581f754c5e083e950951ce464e69bd06 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Thu, 10 Feb 2022 10:43:16 -0500 Subject: [PATCH 09/32] Use packageInfo object instead of RegistryPackage where possible --- .../epm/elasticsearch/template/install.ts | 13 +++++++++++-- .../fleet/server/services/epm/fields/field.ts | 4 ++-- .../fleet/server/services/epm/packages/assets.ts | 6 +++--- .../fleet/server/services/package_policy.ts | 15 ++++----------- 4 files changed, 20 insertions(+), 18 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index 4224ff6b01a19..58a6ace9257ce 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -16,6 +16,7 @@ import type { RegistryElasticsearch, InstallablePackage, IndexTemplate, + PackageInfo, } from '../../../../types'; import { loadFieldsFromYaml, processFields } from '../../fields/field'; import type { Field } from '../../fields/field'; @@ -31,6 +32,8 @@ import type { ESAssetMetadata } from '../meta'; import { getESAssetMetadata } from '../meta'; import { retryTransientEsErrors } from '../retry'; +import { getPackageInfo } from '../../packages'; + import { generateMappings, generateTemplateName, @@ -62,10 +65,16 @@ export const installTemplates = async ( const dataStreams = installablePackage.data_streams; if (!dataStreams) return []; + const packageInfo = await getPackageInfo({ + savedObjectsClient, + pkgName: installablePackage.name, + pkgVersion: installablePackage.version, + }); + const installedTemplatesNested = await Promise.all( dataStreams.map((dataStream) => installTemplateForDataStream({ - pkg: installablePackage, + pkg: packageInfo, esClient, logger, dataStream, @@ -177,7 +186,7 @@ export async function installTemplateForDataStream({ logger, dataStream, }: { - pkg: InstallablePackage; + pkg: PackageInfo; esClient: ElasticsearchClient; logger: Logger; dataStream: RegistryDataStream; diff --git a/x-pack/plugins/fleet/server/services/epm/fields/field.ts b/x-pack/plugins/fleet/server/services/epm/fields/field.ts index d854a0fe8e74d..06ff858df6786 100644 --- a/x-pack/plugins/fleet/server/services/epm/fields/field.ts +++ b/x-pack/plugins/fleet/server/services/epm/fields/field.ts @@ -7,7 +7,7 @@ import { safeLoad } from 'js-yaml'; -import type { InstallablePackage } from '../../../types'; +import type { PackageInfo } from '../../../types'; import { getAssetsData } from '../packages/assets'; // This should become a copy of https://github.com/elastic/beats/blob/d9a4c9c240a9820fab15002592e5bb6db318543b/libbeat/mapping/field.go#L39 @@ -261,7 +261,7 @@ const isFields = (path: string) => { */ export const loadFieldsFromYaml = async ( - pkg: InstallablePackage, + pkg: PackageInfo, datasetName?: string ): Promise => { // Fetch all field definition files diff --git a/x-pack/plugins/fleet/server/services/epm/packages/assets.ts b/x-pack/plugins/fleet/server/services/epm/packages/assets.ts index c28c982f4ea4c..c939ce093a65c 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/assets.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/assets.ts @@ -5,7 +5,7 @@ * 2.0. */ -import type { InstallablePackage } from '../../../types'; +import type { PackageInfo } from '../../../types'; import { getArchiveFilelist, getAsset } from '../archive'; import type { ArchiveEntry } from '../archive'; @@ -17,7 +17,7 @@ import type { ArchiveEntry } from '../archive'; // and different package and version structure export function getAssets( - packageInfo: InstallablePackage, + packageInfo: PackageInfo, filter = (path: string): boolean => true, datasetName?: string ): string[] { @@ -52,7 +52,7 @@ export function getAssets( // ASK: Does getAssetsData need an installSource now? // if so, should it be an Installation vs InstallablePackage or add another argument? export async function getAssetsData( - packageInfo: InstallablePackage, + packageInfo: PackageInfo, filter = (path: string): boolean => true, datasetName?: string ): Promise { diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index 8a59d2a70743d..f978ee5fe0e2f 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -823,12 +823,7 @@ class PackagePolicyService { inputs: PackagePolicyInput[] ): Promise { const inputsPromises = inputs.map(async (input) => { - const compiledInput = await _compilePackagePolicyInput( - installablePackage, - pkgInfo, - vars, - input - ); + const compiledInput = await _compilePackagePolicyInput(pkgInfo, vars, input); const compiledStreams = await _compilePackageStreams( installablePackage, pkgInfo, @@ -945,7 +940,6 @@ function assignStreamIdToInput(packagePolicyId: string, input: NewPackagePolicyI } async function _compilePackagePolicyInput( - installablePackage: InstallablePackage, pkgInfo: PackageInfo, vars: PackagePolicy['vars'], input: PackagePolicyInput @@ -970,7 +964,7 @@ async function _compilePackagePolicyInput( return undefined; } - const [pkgInputTemplate] = await getAssetsData(installablePackage, (path: string) => + const [pkgInputTemplate] = await getAssetsData(pkgInfo, (path: string) => path.endsWith(`/agent/input/${packageInput.template_path!}`) ); @@ -992,7 +986,7 @@ async function _compilePackageStreams( input: PackagePolicyInput ) { const streamsPromises = input.streams.map((stream) => - _compilePackageStream(installablePackage, pkgInfo, vars, input, stream) + _compilePackageStream(pkgInfo, vars, input, stream) ); return await Promise.all(streamsPromises); @@ -1035,7 +1029,6 @@ export function _applyIndexPrivileges( } async function _compilePackageStream( - installablePackage: InstallablePackage, pkgInfo: PackageInfo, vars: PackagePolicy['vars'], input: PackagePolicyInput, @@ -1078,7 +1071,7 @@ async function _compilePackageStream( const datasetPath = packageDataStream.path; const [pkgStreamTemplate] = await getAssetsData( - installablePackage, + pkgInfo, (path: string) => path.endsWith(streamFromPkg.template_path), datasetPath ); From bac2d46e25b0e55c8084a2d09a05bce2772fc34f Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Thu, 10 Feb 2022 11:11:15 -0500 Subject: [PATCH 10/32] Fix type error in assets test --- .../plugins/fleet/server/services/epm/packages/assets.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/assets.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/assets.test.ts index c5b104696aaf4..b019729b65eb1 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/assets.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/assets.test.ts @@ -5,7 +5,7 @@ * 2.0. */ -import type { InstallablePackage } from '../../../types'; +import type { PackageInfo } from '../../../types'; import { getArchiveFilelist } from '../archive/cache'; @@ -66,7 +66,7 @@ const tests = [ test('testGetAssets', () => { for (const value of tests) { // as needed to pretend it is an InstallablePackage - const assets = getAssets(value.package as InstallablePackage, value.filter, value.dataset); + const assets = getAssets(value.package as PackageInfo, value.filter, value.dataset); expect(assets).toStrictEqual(value.expected); } }); From f8331668dda9e747cb59de4e56e3a8305e181b96 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Thu, 10 Feb 2022 14:34:33 -0500 Subject: [PATCH 11/32] Fix test timeouts --- .../fleet/server/services/epm/packages/get.ts | 44 +++++++++++-------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get.ts b/x-pack/plugins/fleet/server/services/epm/packages/get.ts index c3f8a22deff20..b99294644c226 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get.ts @@ -99,32 +99,40 @@ export async function getPackageSavedObjects( export const getInstallations = getPackageSavedObjects; +async function fetchRegistryPackageWithFallbackToBundledPackages( + pkgName: string, + pkgVersion: string +) { + try { + return Registry.fetchFindLatestPackage(pkgName); + } catch (error) { + const bundledPackages = await getBundledPackages(); + + const bundledPackage = bundledPackages.find( + (b) => b.name === pkgName && b.version === pkgVersion + ); + + // If we don't find a bundled package, reject with the original error + if (!bundledPackage) { + throw error; + } + return { + name: bundledPackage.name, + version: bundledPackage.version, + }; + } +} + export async function getPackageInfo(options: { savedObjectsClient: SavedObjectsClientContract; pkgName: string; pkgVersion: string; }): Promise { const { savedObjectsClient, pkgName, pkgVersion } = options; + const [savedObject, latestPackage] = await Promise.all([ getInstallationObject({ savedObjectsClient, pkgName }), - new Promise<{ name: string; version: string }>((resolve, reject) => { - return Registry.fetchFindLatestPackage(pkgName).catch(async (error) => { - const bundledPackages = await getBundledPackages(); - const bundledPackage = bundledPackages.find( - (b) => b.name === pkgName && b.version === pkgVersion - ); - - // If we don't find a bundled package, reject with the original error - if (!bundledPackage) { - return reject(error); - } - - resolve({ - name: bundledPackage.name, - version: bundledPackage.version, - }); - }); - }), + fetchRegistryPackageWithFallbackToBundledPackages(pkgName, pkgVersion), ]); // If no package version is provided, use the installed version in the response From bdcc878c69fb449a132ae4b67b945c94b47ad6e9 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Thu, 10 Feb 2022 14:46:24 -0500 Subject: [PATCH 12/32] Fix promise logic for registry fetch fallback --- x-pack/plugins/fleet/server/services/epm/packages/get.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get.ts b/x-pack/plugins/fleet/server/services/epm/packages/get.ts index b99294644c226..07cc9909cc38f 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get.ts @@ -103,16 +103,14 @@ async function fetchRegistryPackageWithFallbackToBundledPackages( pkgName: string, pkgVersion: string ) { - try { - return Registry.fetchFindLatestPackage(pkgName); - } catch (error) { + return Registry.fetchFindLatestPackage(pkgName).catch(async (error) => { const bundledPackages = await getBundledPackages(); const bundledPackage = bundledPackages.find( (b) => b.name === pkgName && b.version === pkgVersion ); - // If we don't find a bundled package, reject with the original error + // If we don't find a bundled package, re-throw the original error if (!bundledPackage) { throw error; } @@ -120,7 +118,7 @@ async function fetchRegistryPackageWithFallbackToBundledPackages( name: bundledPackage.name, version: bundledPackage.version, }; - } + }); } export async function getPackageInfo(options: { From 49be525f83f0ca918e30cfdce49458be96db0f8c Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Fri, 11 Feb 2022 11:04:04 -0500 Subject: [PATCH 13/32] Use archive package as default in create package policy --- .../fleet/server/services/package_policy.ts | 22 +++++-------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index f978ee5fe0e2f..782f4b8e56f82 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -164,24 +164,12 @@ class PackagePolicyService { } validatePackagePolicyOrThrow(packagePolicy, pkgInfo); - const installablePackage: InstallablePackage = await Registry.fetchInfo( - pkgInfo.name, - pkgInfo.version - ).catch(async (error) => { - // Attempt to get an ArchivePackage object from the package info cache if the registry - // is not reachable. This will pull a bundled package if it was previously installed. - const archivePackage = await getArchivePackage({ - name: pkgInfo.name, - version: pkgInfo.version, - }); - - // If we don't find a package in the cache, throw the original error - if (!archivePackage) { - throw error; - } + let installablePackage: InstallablePackage | undefined = + getArchivePackage(pkgInfo)?.packageInfo; - return archivePackage?.packageInfo; - }); + if (!installablePackage) { + installablePackage = await Registry.fetchInfo(pkgInfo.name, pkgInfo.version); + } inputs = await this._compilePackagePolicyInputs( installablePackage, From f165e5ddcdf77c71b4aa578ffa7f2fbabc59e31e Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Fri, 11 Feb 2022 12:25:22 -0500 Subject: [PATCH 14/32] Always install from bundled package if it exists - regardless of installation context --- .../epm/packages/bulk_install_packages.ts | 93 ++++--------------- .../server/services/epm/packages/install.ts | 24 +++++ .../fleet/server/services/preconfiguration.ts | 3 - 3 files changed, 42 insertions(+), 78 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts b/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts index cdebccb054c3b..c6e793ca243a4 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts @@ -12,8 +12,6 @@ import * as Registry from '../registry'; import type { InstallResult } from '../../../types'; -import { BundledPackageNotFoundError } from '../../../errors'; - import { installPackage, isPackageVersionOrLaterInstalled } from './install'; import type { BulkInstallResponse, IBulkInstallPackageError } from './install'; import { getBundledPackages } from './get_bundled_packages'; @@ -33,10 +31,8 @@ export async function bulkInstallPackages({ esClient, spaceId, force, - preferredSource = 'registry', }: BulkInstallPackagesParams): Promise { const logger = appContextService.getLogger(); - const bundledPackages = await getBundledPackages(); const packagesResults = await Promise.allSettled( @@ -45,35 +41,27 @@ export async function bulkInstallPackages({ return Promise.resolve(pkg); } - return Registry.fetchFindLatestPackage(pkg).catch((error) => { - // For registry preferred bulk installs, consider an inability to reach the configured - // registry a fotal error - if (preferredSource === 'registry') { - return Promise.reject(error); - } - - // For bundled preferred bulk installs, we want to allow the installation to continue, even - // if the registry is not reachable + try { + return Registry.fetchFindLatestPackage(pkg); + } catch (error) { const bundledPackage = bundledPackages.find((b) => b.name === pkg); if (!bundledPackage) { - return Promise.reject( - new BundledPackageNotFoundError(`No bundled package found with name ${pkg}`) - ); + throw error; } - return Promise.resolve({ + return { name: bundledPackage.name, version: bundledPackage.version, - }); - }); + }; + } }) ); logger.debug( - `kicking off bulk install of ${packagesToInstall.join( - ', ' - )} with preferred source of "${preferredSource}"` + `kicking off bulk install of ${packagesToInstall + .map((pkg) => (typeof pkg === 'string' ? pkg : pkg.name)) + .join(', ')}` ); const bulkInstallResults = await Promise.allSettled( @@ -109,61 +97,16 @@ export async function bulkInstallPackages({ }; } - let installResult: InstallResult; const pkgkey = Registry.pkgToPkgKey(pkgKeyProps); - const bundledPackage = bundledPackages.find((pkg) => `${pkg.name}-${pkg.version}` === pkgkey); - - // If preferred source is bundled packages on disk, attempt to install from disk first, then fall back to registry - if (preferredSource === 'bundled') { - if (bundledPackage) { - logger.debug( - `kicking off install of ${pkgKeyProps.name}-${pkgKeyProps.version} from bundled package on disk` - ); - installResult = await installPackage({ - savedObjectsClient, - esClient, - installSource: 'upload', - archiveBuffer: bundledPackage.buffer, - contentType: 'application/zip', - spaceId, - }); - } else { - installResult = await installPackage({ - savedObjectsClient, - esClient, - pkgkey, - installSource: 'registry', - spaceId, - force, - }); - } - } else { - // If preferred source is registry, attempt to install from registry first, then fall back to bundled packages on disk - installResult = await installPackage({ - savedObjectsClient, - esClient, - pkgkey, - installSource: 'registry', - spaceId, - force, - }); - - // If we initially errored, try to install from bundled package on disk - if (installResult.error && bundledPackage) { - logger.debug( - `kicking off install of ${pkgKeyProps.name}-${pkgKeyProps.version} from bundled package on disk` - ); - installResult = await installPackage({ - savedObjectsClient, - esClient, - installSource: 'upload', - archiveBuffer: bundledPackage.buffer, - contentType: 'application/zip', - spaceId, - }); - } - } + const installResult = await installPackage({ + savedObjectsClient, + esClient, + pkgkey, + installSource: 'registry', + spaceId, + force, + }); if (installResult.error) { return { diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.ts index 21f0ae25d6faf..435b19419261a 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.ts @@ -44,6 +44,7 @@ import { removeInstallation } from './remove'; import { getPackageSavedObjects } from './get'; import { _installPackage } from './_install_package'; import { removeOldAssets } from './cleanup'; +import { getBundledPackages } from './get_bundled_packages'; export async function isPackageInstalled(options: { savedObjectsClient: SavedObjectsClientContract; @@ -468,8 +469,31 @@ export async function installPackage(args: InstallPackageParams) { const logger = appContextService.getLogger(); const { savedObjectsClient, esClient } = args; + const bundledPackages = await getBundledPackages(); + if (args.installSource === 'registry') { const { pkgkey, force, spaceId } = args; + + const matchingBundledPackage = bundledPackages.find( + (pkg) => Registry.pkgToPkgKey(pkg) === pkgkey + ); + + if (matchingBundledPackage) { + logger.debug( + `found bundled package for requested install of ${pkgkey} - installing from bundled package archive` + ); + + const response = installPackageByUpload({ + savedObjectsClient, + esClient, + archiveBuffer: matchingBundledPackage.buffer, + contentType: 'application/zip', + spaceId, + }); + + return response; + } + logger.debug(`kicking off install of ${pkgkey} from registry`); const response = installPackageFromRegistry({ savedObjectsClient, diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.ts b/x-pack/plugins/fleet/server/services/preconfiguration.ts index e9d97856a926f..e9c079d435e7e 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.ts @@ -181,9 +181,6 @@ export async function ensurePreconfiguredPackagesAndPolicies( packagesToInstall, force: true, // Always force outdated packages to be installed if a later version isn't installed spaceId, - // During setup, we'll try to install preconfigured packages from the versions bundled with Kibana - // whenever possible - preferredSource: 'bundled', }); const fulfilledPackages = []; From 132b81dc2d8a085b41307cd2b5df86b909b33ab9 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Fri, 11 Feb 2022 15:13:19 -0500 Subject: [PATCH 15/32] Clean up + refactor a bit --- .../epm/packages/bulk_install_packages.ts | 19 ++------------ .../fleet/server/services/epm/packages/get.ts | 25 +------------------ .../server/services/epm/registry/index.ts | 19 ++++++++++++++ 3 files changed, 22 insertions(+), 41 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts b/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts index c6e793ca243a4..518971a51e0ef 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts @@ -14,7 +14,6 @@ import type { InstallResult } from '../../../types'; import { installPackage, isPackageVersionOrLaterInstalled } from './install'; import type { BulkInstallResponse, IBulkInstallPackageError } from './install'; -import { getBundledPackages } from './get_bundled_packages'; interface BulkInstallPackagesParams { savedObjectsClient: SavedObjectsClientContract; @@ -33,28 +32,14 @@ export async function bulkInstallPackages({ force, }: BulkInstallPackagesParams): Promise { const logger = appContextService.getLogger(); - const bundledPackages = await getBundledPackages(); const packagesResults = await Promise.allSettled( - packagesToInstall.map((pkg) => { + packagesToInstall.map(async (pkg) => { if (typeof pkg !== 'string') { return Promise.resolve(pkg); } - try { - return Registry.fetchFindLatestPackage(pkg); - } catch (error) { - const bundledPackage = bundledPackages.find((b) => b.name === pkg); - - if (!bundledPackage) { - throw error; - } - - return { - name: bundledPackage.name, - version: bundledPackage.version, - }; - } + return Registry.fetchFindLatestPackageWithFallbackToBundled(pkg); }) ); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get.ts b/x-pack/plugins/fleet/server/services/epm/packages/get.ts index c8a8664e518dd..364102c24f4c1 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get.ts @@ -29,7 +29,6 @@ import { getArchivePackage } from '../archive'; import { normalizeKuery } from '../../saved_object'; import { createInstallableFrom } from './index'; -import { getBundledPackages } from './get_bundled_packages'; export type { SearchParams } from '../registry'; export { getFile } from '../registry'; @@ -138,28 +137,6 @@ export async function getPackageInfoFromRegistry(options: { return createInstallableFrom(updated, savedObject); } -async function fetchRegistryPackageWithFallbackToBundledPackages( - pkgName: string, - pkgVersion: string -) { - return Registry.fetchFindLatestPackage(pkgName).catch(async (error) => { - const bundledPackages = await getBundledPackages(); - - const bundledPackage = bundledPackages.find( - (b) => b.name === pkgName && b.version === pkgVersion - ); - - // If we don't find a bundled package, re-throw the original error - if (!bundledPackage) { - throw error; - } - return { - name: bundledPackage.name, - version: bundledPackage.version, - }; - }); -} - export async function getPackageInfo(options: { savedObjectsClient: SavedObjectsClientContract; pkgName: string; @@ -169,7 +146,7 @@ export async function getPackageInfo(options: { const [savedObject, latestPackage] = await Promise.all([ getInstallationObject({ savedObjectsClient, pkgName }), - fetchRegistryPackageWithFallbackToBundledPackages(pkgName, pkgVersion), + Registry.fetchFindLatestPackageWithFallbackToBundled(pkgName), ]); // If no package version is provided, use the installed version in the response diff --git a/x-pack/plugins/fleet/server/services/epm/registry/index.ts b/x-pack/plugins/fleet/server/services/epm/registry/index.ts index 6564114f9cd3f..49ec6192aa4d3 100644 --- a/x-pack/plugins/fleet/server/services/epm/registry/index.ts +++ b/x-pack/plugins/fleet/server/services/epm/registry/index.ts @@ -35,6 +35,8 @@ import { streamToBuffer } from '../streams'; import { appContextService } from '../..'; import { PackageNotFoundError, PackageCacheError, RegistryResponseError } from '../../../errors'; +import { getBundledPackages } from '../packages/get_bundled_packages'; + import { fetchUrl, getResponse, getResponseStream } from './requests'; import { getRegistryUrl } from './registry_url'; @@ -80,6 +82,23 @@ export async function fetchFindLatestPackage(packageName: string): Promise { + const bundledPackages = await getBundledPackages(); + const bundledPackage = bundledPackages.find((b) => b.name === pkgName); + + // If we don't find a bundled package, re-throw the original error + if (!bundledPackage) { + throw error; + } + + return { + name: bundledPackage.name, + version: bundledPackage.version, + }; + }); +} + export async function fetchInfo(pkgName: string, pkgVersion: string): Promise { const registryUrl = getRegistryUrl(); try { From ecfa6fa8e2fa9037ee3544994b73e8a81ac71e92 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Fri, 11 Feb 2022 15:34:31 -0500 Subject: [PATCH 16/32] Default to cached package archive for policy updates --- .../plugins/fleet/server/services/package_policy.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index 782f4b8e56f82..e062f2ea999e8 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -406,14 +406,20 @@ class PackagePolicyService { validatePackagePolicyOrThrow(packagePolicy, pkgInfo); - const registryPkgInfo = await Registry.fetchInfo(pkgInfo.name, pkgInfo.version); + let installablePackage: InstallablePackage | undefined = + getArchivePackage(pkgInfo)?.packageInfo; + + if (!installablePackage) { + installablePackage = await Registry.fetchInfo(pkgInfo.name, pkgInfo.version); + } + inputs = await this._compilePackagePolicyInputs( - registryPkgInfo, + installablePackage, pkgInfo, packagePolicy.vars || {}, inputs ); - elasticsearch = registryPkgInfo.elasticsearch; + elasticsearch = installablePackage.elasticsearch; } await soClient.update( From 86348944f8e7b90e05bcf932e5e069a0f6b5d509 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Mon, 14 Feb 2022 08:46:28 -0500 Subject: [PATCH 17/32] Update mock in get.test.ts --- x-pack/plugins/fleet/server/services/epm/packages/get.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/get.test.ts index 76e01ed8b2f27..c59aa609748df 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get.test.ts @@ -184,7 +184,7 @@ describe('When using EPM `get` services', () => { beforeEach(() => { const mockContract = createAppContextStartContractMock(); appContextService.start(mockContract); - MockRegistry.fetchFindLatestPackage.mockResolvedValue({ + MockRegistry.fetchFindLatestPackageWithFallbackToBundled.mockResolvedValue({ name: 'my-package', version: '1.0.0', } as RegistryPackage); From 80f3e652f3e95b8dee717bc705a14084e4c58579 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Mon, 14 Feb 2022 09:29:01 -0500 Subject: [PATCH 18/32] Add test for install from bundled package logic --- .../services/epm/packages/install.test.ts | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts index b74466bc6271a..aa16663227f36 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts @@ -20,6 +20,7 @@ import { licenseService } from '../../license'; import { installPackage } from './install'; import * as install from './_install_package'; import * as obj from './index'; +import { getBundledPackages } from './get_bundled_packages'; jest.mock('../../app_context', () => { return { @@ -40,6 +41,7 @@ jest.mock('../../upgrade_sender'); jest.mock('../../license'); jest.mock('../../upgrade_sender'); jest.mock('./cleanup'); +jest.mock('./get_bundled_packages'); jest.mock('./_install_package', () => { return { _installPackage: jest.fn(() => Promise.resolve()), @@ -60,21 +62,34 @@ jest.mock('../archive', () => { }; }); +const mockGetBundledPackages = getBundledPackages as jest.MockedFunction; + describe('install', () => { beforeEach(() => { jest.spyOn(Registry, 'splitPkgKey').mockImplementation((pkgKey: string) => { const [pkgName, pkgVersion] = pkgKey.split('-'); return { pkgName, pkgVersion }; }); + jest + .spyOn(Registry, 'pkgToPkgKey') + .mockImplementation((pkg: { name: string; version: string }) => { + return `${pkg.name}-${pkg.version}`; + }); jest .spyOn(Registry, 'fetchFindLatestPackage') .mockImplementation(() => Promise.resolve({ version: '1.3.0' } as any)); jest .spyOn(Registry, 'getRegistryPackage') .mockImplementation(() => Promise.resolve({ packageInfo: { license: 'basic' } } as any)); + + mockGetBundledPackages.mockReset(); }); describe('registry', () => { + beforeEach(() => { + mockGetBundledPackages.mockResolvedValue([]); + }); + it('should send telemetry on install failure, out of date', async () => { await installPackage({ spaceId: DEFAULT_SPACE_ID, @@ -187,6 +202,28 @@ describe('install', () => { status: 'failure', }); }); + + it('should install from bundled package if one exists', async () => { + mockGetBundledPackages.mockResolvedValue([ + { + name: 'test_package', + version: '1.0.0', + buffer: Buffer.from('test_package'), + }, + ]); + + await installPackage({ + spaceId: DEFAULT_SPACE_ID, + installSource: 'registry', + pkgkey: 'test_package-1.0.0', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); + + expect(install._installPackage).toHaveBeenCalledWith( + expect.objectContaining({ installSource: 'upload' }) + ); + }); }); describe('upload', () => { From 8f0ad338abeb20df40d2febaaa01335d7ae4b49c Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Mon, 14 Feb 2022 17:37:49 -0500 Subject: [PATCH 19/32] Delete timeout call in security solution tests --- .../fleet/server/services/epm/registry/index.ts | 12 ++++++++---- .../apps/endpoint/endpoint_list.ts | 1 - 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/registry/index.ts b/x-pack/plugins/fleet/server/services/epm/registry/index.ts index 22256224976ad..e14220b1fc5e2 100644 --- a/x-pack/plugins/fleet/server/services/epm/registry/index.ts +++ b/x-pack/plugins/fleet/server/services/epm/registry/index.ts @@ -82,10 +82,14 @@ export async function fetchFindLatestPackage(packageName: string): Promise { +export async function fetchFindLatestPackageWithFallbackToBundled(packageName: string) { + try { + const latestPackage = await fetchFindLatestPackage(packageName); + + return latestPackage; + } catch (error) { const bundledPackages = await getBundledPackages(); - const bundledPackage = bundledPackages.find((b) => b.name === pkgName); + const bundledPackage = bundledPackages.find((b) => b.name === packageName); // If we don't find a bundled package, re-throw the original error if (!bundledPackage) { @@ -96,7 +100,7 @@ export async function fetchFindLatestPackageWithFallbackToBundled(pkgName: strin name: bundledPackage.name, version: bundledPackage.version, }; - }); + } } export async function fetchInfo(pkgName: string, pkgVersion: string): Promise { diff --git a/x-pack/test/security_solution_endpoint/apps/endpoint/endpoint_list.ts b/x-pack/test/security_solution_endpoint/apps/endpoint/endpoint_list.ts index 3a49278bd21a8..5cb8160eb38bf 100644 --- a/x-pack/test/security_solution_endpoint/apps/endpoint/endpoint_list.ts +++ b/x-pack/test/security_solution_endpoint/apps/endpoint/endpoint_list.ts @@ -91,7 +91,6 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { describe('when there is data,', () => { before(async () => { const endpointPackage = await policyTestResources.getEndpointPackage(); - await endpointTestResources.setMetadataTransformFrequency('1s', endpointPackage.version); indexedData = await endpointTestResources.loadEndpointData({ numHosts: 3 }); await pageObjects.endpoint.navigateToEndpointList(); await pageObjects.endpoint.waitForTableToHaveNumberOfEntries('endpointListTable', 3, 90000); From d192922c0007f40249a0f535458ecbe791fb218d Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Mon, 14 Feb 2022 22:34:48 -0500 Subject: [PATCH 20/32] Fix unused var in endpoint test --- .../security_solution_endpoint/apps/endpoint/endpoint_list.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/test/security_solution_endpoint/apps/endpoint/endpoint_list.ts b/x-pack/test/security_solution_endpoint/apps/endpoint/endpoint_list.ts index 5cb8160eb38bf..a6b7afcd46339 100644 --- a/x-pack/test/security_solution_endpoint/apps/endpoint/endpoint_list.ts +++ b/x-pack/test/security_solution_endpoint/apps/endpoint/endpoint_list.ts @@ -90,7 +90,6 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { describe('when there is data,', () => { before(async () => { - const endpointPackage = await policyTestResources.getEndpointPackage(); indexedData = await endpointTestResources.loadEndpointData({ numHosts: 3 }); await pageObjects.endpoint.navigateToEndpointList(); await pageObjects.endpoint.waitForTableToHaveNumberOfEntries('endpointListTable', 3, 90000); From 840c4ecbae3b76702cee52646ac1739ad4c2ef2f Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Mon, 14 Feb 2022 23:26:02 -0500 Subject: [PATCH 21/32] Fix another unused var in endpoint test --- .../security_solution_endpoint/apps/endpoint/endpoint_list.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/test/security_solution_endpoint/apps/endpoint/endpoint_list.ts b/x-pack/test/security_solution_endpoint/apps/endpoint/endpoint_list.ts index a6b7afcd46339..022bd437b989b 100644 --- a/x-pack/test/security_solution_endpoint/apps/endpoint/endpoint_list.ts +++ b/x-pack/test/security_solution_endpoint/apps/endpoint/endpoint_list.ts @@ -20,7 +20,6 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { const testSubjects = getService('testSubjects'); const browser = getService('browser'); const endpointTestResources = getService('endpointTestResources'); - const policyTestResources = getService('policyTestResources'); const expectedData = [ [ From 513dcc121e4ab52e2d482ebb54a45af06c6f6b6c Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Tue, 15 Feb 2022 08:43:31 -0500 Subject: [PATCH 22/32] [Debug] Add some logging to test installation times in CI --- .../server/services/epm/packages/_install_package.ts | 5 +++++ .../server/services/epm/packages/bulk_install_packages.ts | 6 ++++++ x-pack/plugins/fleet/server/services/epm/packages/get.ts | 8 ++++++++ x-pack/test/security_solution_endpoint/config.ts | 3 +++ 4 files changed, 22 insertions(+) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index 05ed45a0e0c7a..e12de62564037 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -69,6 +69,7 @@ export async function _installPackage({ installSource: InstallSource; spaceId: string; }): Promise { + const start = Date.now(); const { name: pkgName, version: pkgVersion } = packageInfo; try { @@ -236,6 +237,10 @@ export async function _installPackage({ await packagePolicyService.upgrade(savedObjectsClient, esClient, policyIdsToUpgrade.items); } + const end = Date.now(); + + logger.debug(`Installation of ${pkgName}-${pkgVersion} took ${end - start}ms`); + return [ ...installedKibanaAssetsRefs, ...installedPipelines, diff --git a/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts b/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts index 518971a51e0ef..2efce474c33a7 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts @@ -65,6 +65,7 @@ export async function bulkInstallPackages({ }); if (installedPackageResult) { + logger.debug(`Retrieved installed package for ${pkgKeyProps.name}-${pkgKeyProps.version}`); const { name, version, @@ -84,6 +85,8 @@ export async function bulkInstallPackages({ const pkgkey = Registry.pkgToPkgKey(pkgKeyProps); + const start = Date.now(); + logger.debug(`Installing ${pkgkey} as part of bulk install`); const installResult = await installPackage({ savedObjectsClient, esClient, @@ -100,6 +103,9 @@ export async function bulkInstallPackages({ installType: installResult.installType, }; } + + const end = Date.now(); + logger.debug(`Installed ${pkgkey} in ${end - start}ms`); return { name: packageName, version: pkgKeyProps.version, diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get.ts b/x-pack/plugins/fleet/server/services/epm/packages/get.ts index 364102c24f4c1..b83dc98f05471 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get.ts @@ -144,6 +144,10 @@ export async function getPackageInfo(options: { }): Promise { const { savedObjectsClient, pkgName, pkgVersion } = options; + const logger = appContextService.getLogger(); + const start = Date.now(); + logger.debug(`Getting package info for ${pkgName}-${pkgVersion}`); + const [savedObject, latestPackage] = await Promise.all([ getInstallationObject({ savedObjectsClient, pkgName }), Registry.fetchFindLatestPackageWithFallbackToBundled(pkgName), @@ -176,6 +180,10 @@ export async function getPackageInfo(options: { }; const updated = { ...packageInfo, ...additions }; + const end = Date.now(); + + logger.debug(`Retrieved package info in ${end - start}ms`); + return createInstallableFrom(updated, savedObject); } diff --git a/x-pack/test/security_solution_endpoint/config.ts b/x-pack/test/security_solution_endpoint/config.ts index b00df7732ea4f..10ed6f2c2044f 100644 --- a/x-pack/test/security_solution_endpoint/config.ts +++ b/x-pack/test/security_solution_endpoint/config.ts @@ -44,6 +44,9 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { // always install Endpoint package by default when Fleet sets up `--xpack.fleet.packages.0.name=endpoint`, `--xpack.fleet.packages.0.version=latest`, + '--logging.loggers[0].name=plugins.fleet', + '--logging.loggers[0].level=debug', + '--logging.loggers[0].appenders[0]=console', ], }, layout: { From 21baf092b09b6916b6574250a33c5adbd0d062eb Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Tue, 15 Feb 2022 10:43:56 -0500 Subject: [PATCH 23/32] Revert "[Debug] Add some logging to test installation times in CI" This reverts commit 513dcc121e4ab52e2d482ebb54a45af06c6f6b6c. --- .../server/services/epm/packages/_install_package.ts | 5 ----- .../server/services/epm/packages/bulk_install_packages.ts | 6 ------ x-pack/plugins/fleet/server/services/epm/packages/get.ts | 8 -------- x-pack/test/security_solution_endpoint/config.ts | 3 --- 4 files changed, 22 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index e12de62564037..05ed45a0e0c7a 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -69,7 +69,6 @@ export async function _installPackage({ installSource: InstallSource; spaceId: string; }): Promise { - const start = Date.now(); const { name: pkgName, version: pkgVersion } = packageInfo; try { @@ -237,10 +236,6 @@ export async function _installPackage({ await packagePolicyService.upgrade(savedObjectsClient, esClient, policyIdsToUpgrade.items); } - const end = Date.now(); - - logger.debug(`Installation of ${pkgName}-${pkgVersion} took ${end - start}ms`); - return [ ...installedKibanaAssetsRefs, ...installedPipelines, diff --git a/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts b/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts index 2efce474c33a7..518971a51e0ef 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts @@ -65,7 +65,6 @@ export async function bulkInstallPackages({ }); if (installedPackageResult) { - logger.debug(`Retrieved installed package for ${pkgKeyProps.name}-${pkgKeyProps.version}`); const { name, version, @@ -85,8 +84,6 @@ export async function bulkInstallPackages({ const pkgkey = Registry.pkgToPkgKey(pkgKeyProps); - const start = Date.now(); - logger.debug(`Installing ${pkgkey} as part of bulk install`); const installResult = await installPackage({ savedObjectsClient, esClient, @@ -103,9 +100,6 @@ export async function bulkInstallPackages({ installType: installResult.installType, }; } - - const end = Date.now(); - logger.debug(`Installed ${pkgkey} in ${end - start}ms`); return { name: packageName, version: pkgKeyProps.version, diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get.ts b/x-pack/plugins/fleet/server/services/epm/packages/get.ts index b83dc98f05471..364102c24f4c1 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get.ts @@ -144,10 +144,6 @@ export async function getPackageInfo(options: { }): Promise { const { savedObjectsClient, pkgName, pkgVersion } = options; - const logger = appContextService.getLogger(); - const start = Date.now(); - logger.debug(`Getting package info for ${pkgName}-${pkgVersion}`); - const [savedObject, latestPackage] = await Promise.all([ getInstallationObject({ savedObjectsClient, pkgName }), Registry.fetchFindLatestPackageWithFallbackToBundled(pkgName), @@ -180,10 +176,6 @@ export async function getPackageInfo(options: { }; const updated = { ...packageInfo, ...additions }; - const end = Date.now(); - - logger.debug(`Retrieved package info in ${end - start}ms`); - return createInstallableFrom(updated, savedObject); } diff --git a/x-pack/test/security_solution_endpoint/config.ts b/x-pack/test/security_solution_endpoint/config.ts index 10ed6f2c2044f..b00df7732ea4f 100644 --- a/x-pack/test/security_solution_endpoint/config.ts +++ b/x-pack/test/security_solution_endpoint/config.ts @@ -44,9 +44,6 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { // always install Endpoint package by default when Fleet sets up `--xpack.fleet.packages.0.name=endpoint`, `--xpack.fleet.packages.0.version=latest`, - '--logging.loggers[0].name=plugins.fleet', - '--logging.loggers[0].level=debug', - '--logging.loggers[0].appenders[0]=console', ], }, layout: { From d79e9676ecb2a4c6a040652be930c76fc70b9254 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Tue, 15 Feb 2022 10:52:20 -0500 Subject: [PATCH 24/32] Update docker images for registry --- x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts | 2 +- x-pack/test/fleet_api_integration/apis/package_policy/create.ts | 2 +- x-pack/test/fleet_api_integration/config.ts | 2 +- x-pack/test/functional/config.js | 2 +- x-pack/test/functional_synthetics/config.js | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts b/x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts index 8fc87b49a6607..2895fb4bdd4fc 100644 --- a/x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts +++ b/x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts @@ -13,7 +13,7 @@ import { packageRegistryPort } from './ftr_config'; import { FtrProviderContext } from './ftr_provider_context'; export const dockerImage = - 'docker.elastic.co/package-registry/distribution@sha256:de952debe048d903fc73e8a4472bb48bb95028d440cba852f21b863d47020c61'; + 'docker.elastic.co/package-registry/distribution@sha256:fc696e0f358f9720612dbd3716fc6dc48590e58bc9ba9866fff3903894dde324'; async function ftrConfigRun({ readConfigFile }: FtrConfigProviderContext) { const kibanaConfig = await readConfigFile(require.resolve('./ftr_config.ts')); diff --git a/x-pack/test/fleet_api_integration/apis/package_policy/create.ts b/x-pack/test/fleet_api_integration/apis/package_policy/create.ts index a803b7224d0b4..0eb5785198a1b 100644 --- a/x-pack/test/fleet_api_integration/apis/package_policy/create.ts +++ b/x-pack/test/fleet_api_integration/apis/package_policy/create.ts @@ -218,7 +218,7 @@ export default function (providerContext: FtrProviderContext) { package: { name: 'endpoint', title: 'Endpoint', - version: '1.3.0-dev.0', + version: '1.4.1', }, }) .expect(200); diff --git a/x-pack/test/fleet_api_integration/config.ts b/x-pack/test/fleet_api_integration/config.ts index fb9dc7b6b4ce6..1aff78020aa52 100644 --- a/x-pack/test/fleet_api_integration/config.ts +++ b/x-pack/test/fleet_api_integration/config.ts @@ -15,7 +15,7 @@ import { defineDockerServersConfig } from '@kbn/test'; // example: https://beats-ci.elastic.co/blue/organizations/jenkins/Ingest-manager%2Fpackage-storage/detail/snapshot/74/pipeline/257#step-302-log-1. // It should be updated any time there is a new Docker image published for the Snapshot Distribution of the Package Registry. export const dockerImage = - 'docker.elastic.co/package-registry/distribution@sha256:de952debe048d903fc73e8a4472bb48bb95028d440cba852f21b863d47020c61'; + 'docker.elastic.co/package-registry/distribution@sha256:fc696e0f358f9720612dbd3716fc6dc48590e58bc9ba9866fff3903894dde324'; export default async function ({ readConfigFile }: FtrConfigProviderContext) { const xPackAPITestsConfig = await readConfigFile(require.resolve('../api_integration/config.ts')); diff --git a/x-pack/test/functional/config.js b/x-pack/test/functional/config.js index 67c2f9b386425..ac0e6bef0817b 100644 --- a/x-pack/test/functional/config.js +++ b/x-pack/test/functional/config.js @@ -15,7 +15,7 @@ import { pageObjects } from './page_objects'; // example: https://beats-ci.elastic.co/blue/organizations/jenkins/Ingest-manager%2Fpackage-storage/detail/snapshot/74/pipeline/257#step-302-log-1. // It should be updated any time there is a new Docker image published for the Snapshot Distribution of the Package Registry. export const dockerImage = - 'docker.elastic.co/package-registry/distribution:ffcbe0ba25b9bae09a671249cbb1b25af0aa1994'; + 'docker.elastic.co/package-registry/distribution:fc696e0f358f9720612dbd3716fc6dc48590e58bc9ba9866fff3903894dde324'; // the default export of config files must be a config provider // that returns an object with the projects config values diff --git a/x-pack/test/functional_synthetics/config.js b/x-pack/test/functional_synthetics/config.js index 28cd7e3b099dc..f75d2b0635fe7 100644 --- a/x-pack/test/functional_synthetics/config.js +++ b/x-pack/test/functional_synthetics/config.js @@ -17,7 +17,7 @@ import { pageObjects } from './page_objects'; // example: https://beats-ci.elastic.co/blue/organizations/jenkins/Ingest-manager%2Fpackage-storage/detail/snapshot/74/pipeline/257#step-302-log-1. // It should be updated any time there is a new Docker image published for the Snapshot Distribution of the Package Registry that updates Synthetics. export const dockerImage = - 'docker.elastic.co/package-registry/distribution:48202133e7506873aff3cc7c3b1d284158727779'; + 'docker.elastic.co/package-registry/distribution:ffcbe0ba25b9bae09a671249cbb1b25af0aa1994'; // the default export of config files must be a config provider // that returns an object with the projects config values From 2310ec8377d00732c492446f96fa83ead7df174e Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Tue, 15 Feb 2022 11:40:02 -0500 Subject: [PATCH 25/32] Update docker image digest again --- x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts | 2 +- .../fleet/server/integration_tests/docker_registry_helper.ts | 2 +- x-pack/test/fleet_api_integration/config.ts | 2 +- x-pack/test/functional/config.js | 2 +- x-pack/test/functional_synthetics/config.js | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts b/x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts index 2895fb4bdd4fc..3f8a7758527f7 100644 --- a/x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts +++ b/x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts @@ -13,7 +13,7 @@ import { packageRegistryPort } from './ftr_config'; import { FtrProviderContext } from './ftr_provider_context'; export const dockerImage = - 'docker.elastic.co/package-registry/distribution@sha256:fc696e0f358f9720612dbd3716fc6dc48590e58bc9ba9866fff3903894dde324'; + 'docker.elastic.co/package-registry/distribution@sha256:a30d5b428bc3b22e87a4ae3c4d06c00bbd757ea41ef9a2ccf01c670f2f111577'; async function ftrConfigRun({ readConfigFile }: FtrConfigProviderContext) { const kibanaConfig = await readConfigFile(require.resolve('./ftr_config.ts')); diff --git a/x-pack/plugins/fleet/server/integration_tests/docker_registry_helper.ts b/x-pack/plugins/fleet/server/integration_tests/docker_registry_helper.ts index 31b0831d7f3e5..ef3621558033b 100644 --- a/x-pack/plugins/fleet/server/integration_tests/docker_registry_helper.ts +++ b/x-pack/plugins/fleet/server/integration_tests/docker_registry_helper.ts @@ -24,7 +24,7 @@ export function useDockerRegistry() { let dockerProcess: ChildProcess | undefined; async function startDockerRegistryServer() { - const dockerImage = `docker.elastic.co/package-registry/distribution@sha256:de952debe048d903fc73e8a4472bb48bb95028d440cba852f21b863d47020c61`; + const dockerImage = `docker.elastic.co/package-registry/distribution@sha256:a30d5b428bc3b22e87a4ae3c4d06c00bbd757ea41ef9a2ccf01c670f2f111577`; const args = ['run', '--rm', '-p', `${packageRegistryPort}:8080`, dockerImage]; diff --git a/x-pack/test/fleet_api_integration/config.ts b/x-pack/test/fleet_api_integration/config.ts index 1aff78020aa52..4b244cdc8c842 100644 --- a/x-pack/test/fleet_api_integration/config.ts +++ b/x-pack/test/fleet_api_integration/config.ts @@ -15,7 +15,7 @@ import { defineDockerServersConfig } from '@kbn/test'; // example: https://beats-ci.elastic.co/blue/organizations/jenkins/Ingest-manager%2Fpackage-storage/detail/snapshot/74/pipeline/257#step-302-log-1. // It should be updated any time there is a new Docker image published for the Snapshot Distribution of the Package Registry. export const dockerImage = - 'docker.elastic.co/package-registry/distribution@sha256:fc696e0f358f9720612dbd3716fc6dc48590e58bc9ba9866fff3903894dde324'; + 'docker.elastic.co/package-registry/distribution@sha256:a30d5b428bc3b22e87a4ae3c4d06c00bbd757ea41ef9a2ccf01c670f2f111577'; export default async function ({ readConfigFile }: FtrConfigProviderContext) { const xPackAPITestsConfig = await readConfigFile(require.resolve('../api_integration/config.ts')); diff --git a/x-pack/test/functional/config.js b/x-pack/test/functional/config.js index ac0e6bef0817b..d67d95ac11507 100644 --- a/x-pack/test/functional/config.js +++ b/x-pack/test/functional/config.js @@ -15,7 +15,7 @@ import { pageObjects } from './page_objects'; // example: https://beats-ci.elastic.co/blue/organizations/jenkins/Ingest-manager%2Fpackage-storage/detail/snapshot/74/pipeline/257#step-302-log-1. // It should be updated any time there is a new Docker image published for the Snapshot Distribution of the Package Registry. export const dockerImage = - 'docker.elastic.co/package-registry/distribution:fc696e0f358f9720612dbd3716fc6dc48590e58bc9ba9866fff3903894dde324'; + 'docker.elastic.co/package-registry/distribution@sha256:a30d5b428bc3b22e87a4ae3c4d06c00bbd757ea41ef9a2ccf01c670f2f111577'; // the default export of config files must be a config provider // that returns an object with the projects config values diff --git a/x-pack/test/functional_synthetics/config.js b/x-pack/test/functional_synthetics/config.js index f75d2b0635fe7..1716243d07eba 100644 --- a/x-pack/test/functional_synthetics/config.js +++ b/x-pack/test/functional_synthetics/config.js @@ -17,7 +17,7 @@ import { pageObjects } from './page_objects'; // example: https://beats-ci.elastic.co/blue/organizations/jenkins/Ingest-manager%2Fpackage-storage/detail/snapshot/74/pipeline/257#step-302-log-1. // It should be updated any time there is a new Docker image published for the Snapshot Distribution of the Package Registry that updates Synthetics. export const dockerImage = - 'docker.elastic.co/package-registry/distribution:ffcbe0ba25b9bae09a671249cbb1b25af0aa1994'; + 'docker.elastic.co/package-registry/distribution@sha256:a30d5b428bc3b22e87a4ae3c4d06c00bbd757ea41ef9a2ccf01c670f2f111577'; // the default export of config files must be a config provider // that returns an object with the projects config values From 53f3b60b81080c186de2061b8ea6ed87ad5ad713 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Tue, 15 Feb 2022 14:49:16 -0500 Subject: [PATCH 26/32] Refactor latest package fetching to fix broken logic/tests --- .../server/services/epm/package_service.ts | 4 +- .../server/services/epm/packages/get.test.ts | 6 +- .../fleet/server/services/epm/packages/get.ts | 4 +- .../server/services/epm/packages/install.ts | 6 +- .../server/services/epm/registry/index.ts | 72 ++++++++++++------- 5 files changed, 56 insertions(+), 36 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/package_service.ts b/x-pack/plugins/fleet/server/services/epm/package_service.ts index 0d9b8cb74b503..68327932afdbf 100644 --- a/x-pack/plugins/fleet/server/services/epm/package_service.ts +++ b/x-pack/plugins/fleet/server/services/epm/package_service.ts @@ -25,7 +25,7 @@ import { checkSuperuser } from '../../routes/security'; import { FleetUnauthorizedError } from '../../errors'; import { installTransform, isTransform } from './elasticsearch/transform/install'; -import { fetchFindLatestPackage, getRegistryPackage } from './registry'; +import { fetchFindLatestPackageOrThrow, getRegistryPackage } from './registry'; import { ensureInstalledPackage, getInstallation } from './packages'; export type InstalledAssetType = EsAssetReference; @@ -117,7 +117,7 @@ class PackageClientImpl implements PackageClient { public async fetchFindLatestPackage(packageName: string) { await this.#runPreflight(); - return fetchFindLatestPackage(packageName); + return fetchFindLatestPackageOrThrow(packageName); } public async getRegistryPackage(packageName: string, packageVersion: string) { diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/get.test.ts index 8bed40a9a2522..5c4c9bb2a907b 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get.test.ts @@ -283,8 +283,8 @@ describe('When using EPM `get` services', () => { }); describe('registry fetch errors', () => { - it('throws when a package that is not installed is not available in the registry', async () => { - MockRegistry.fetchFindLatestPackage.mockResolvedValue(undefined); + it('throws when a package that is not installed is not available in the registry and not bundled', async () => { + MockRegistry.fetchFindLatestPackageWithFallbackToBundled.mockResolvedValue(undefined); const soClient = savedObjectsClientMock.create(); soClient.get.mockRejectedValue(SavedObjectsErrorHelpers.createGenericNotFoundError()); @@ -298,7 +298,7 @@ describe('When using EPM `get` services', () => { }); it('sets the latestVersion to installed version when an installed package is not available in the registry', async () => { - MockRegistry.fetchFindLatestPackage.mockResolvedValue(undefined); + MockRegistry.fetchFindLatestPackageWithFallbackToBundled.mockResolvedValue(undefined); const soClient = savedObjectsClientMock.create(); soClient.get.mockResolvedValue({ id: 'my-package', diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get.ts b/x-pack/plugins/fleet/server/services/epm/packages/get.ts index cf739c837da5a..c0f45083617aa 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get.ts @@ -106,7 +106,7 @@ export async function getPackageInfoFromRegistry(options: { const { savedObjectsClient, pkgName, pkgVersion } = options; const [savedObject, latestPackage] = await Promise.all([ getInstallationObject({ savedObjectsClient, pkgName }), - Registry.fetchFindLatestPackage(pkgName), + Registry.fetchFindLatestPackageOrThrow(pkgName), ]); // If no package version is provided, use the installed version in the response @@ -146,7 +146,7 @@ export async function getPackageInfo(options: { const [savedObject, latestPackage] = await Promise.all([ getInstallationObject({ savedObjectsClient, pkgName }), - Registry.fetchFindLatestPackageWithFallbackToBundled(pkgName), + Registry.fetchFindLatestPackageWithFallbackToBundled(pkgName, { throwIfNotFound: false }), ]); if (!savedObject && !latestPackage) { diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.ts index b441bd4f432a6..fd3664d2c9ea3 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.ts @@ -89,7 +89,7 @@ export async function ensureInstalledPackage(options: { // If pkgVersion isn't specified, find the latest package version const pkgKeyProps = pkgVersion ? { name: pkgName, version: pkgVersion } - : await Registry.fetchFindLatestPackage(pkgName); + : await Registry.fetchFindLatestPackageOrThrow(pkgName); const installedPackageResult = await isPackageVersionOrLaterInstalled({ savedObjectsClient, @@ -252,7 +252,9 @@ async function installPackageFromRegistry({ installType = getInstallType({ pkgVersion, installedPkg }); // get latest package version - const latestPackage = await Registry.fetchFindLatestPackage(pkgName, { ignoreConstraints }); + const latestPackage = await Registry.fetchFindLatestPackageOrThrow(pkgName, { + ignoreConstraints, + }); // let the user install if using the force flag or needing to reinstall or install a previous version due to failed update const installOutOfDateVersionOk = diff --git a/x-pack/plugins/fleet/server/services/epm/registry/index.ts b/x-pack/plugins/fleet/server/services/epm/registry/index.ts index 05843db24e115..e2c9ce6d0b2b5 100644 --- a/x-pack/plugins/fleet/server/services/epm/registry/index.ts +++ b/x-pack/plugins/fleet/server/services/epm/registry/index.ts @@ -21,7 +21,6 @@ import type { InstallSource, RegistryPackage, RegistrySearchResults, - RegistrySearchResult, GetCategoriesRequest, } from '../../../types'; import { @@ -67,20 +66,16 @@ export async function fetchList(params?: SearchParams): Promise; -export async function fetchFindLatestPackage( - packageName: string, - options: { ignoreConstraints?: boolean; throwIfNotFound: false } -): Promise; -export async function fetchFindLatestPackage( +interface FetchFindLatestPackageOptions { + ignoreConstraints?: boolean; +} + +async function _fetchFindLatestPackage( packageName: string, - options?: { ignoreConstraints?: boolean; throwIfNotFound?: boolean } -): Promise { - const { ignoreConstraints = false, throwIfNotFound = true } = options ?? {}; + options?: FetchFindLatestPackageOptions +) { + const { ignoreConstraints = false } = options ?? {}; + const registryUrl = getRegistryUrl(); const url = new URL(`${registryUrl}/search?package=${packageName}&experimental=true`); @@ -89,34 +84,57 @@ export async function fetchFindLatestPackage( } const res = await fetchUrl(url.toString(), 1); - const searchResults = JSON.parse(res); - if (searchResults.length) { - return searchResults[0]; - } else if (throwIfNotFound) { + const searchResults: RegistryPackage[] = JSON.parse(res); + + return searchResults; +} + +export async function fetchFindLatestPackageOrThrow( + packageName: string, + options?: FetchFindLatestPackageOptions +) { + const searchResults = await _fetchFindLatestPackage(packageName, options); + + if (!searchResults.length) { throw new PackageNotFoundError(`[${packageName}] package not found in registry`); } + + return searchResults[0]; } -export async function fetchFindLatestPackageWithFallbackToBundled( +export async function fetchFindLatestPackageOrUndefined( packageName: string, - options?: { ignoreConstraints?: boolean } + options?: FetchFindLatestPackageOptions ) { - const { ignoreConstraints = false } = options ?? {}; + const searchResults = await _fetchFindLatestPackage(packageName, options); + + if (!searchResults.length) { + return undefined; + } + return searchResults[0]; +} + +export async function fetchFindLatestPackageWithFallbackToBundled( + packageName: string, + options?: FetchFindLatestPackageOptions & { throwIfNotFound?: boolean } +) { try { - const latestPackage = await fetchFindLatestPackage(packageName, { - ignoreConstraints, - throwIfNotFound: true, - }); + const latestPackage = await fetchFindLatestPackageOrThrow(packageName, options); return latestPackage; } catch (error) { const bundledPackages = await getBundledPackages(); const bundledPackage = bundledPackages.find((b) => b.name === packageName); - // If we don't find a bundled package, re-throw the original error if (!bundledPackage) { - throw error; + // Callers must opt into re-throwing the original error from the registry request + if (options?.throwIfNotFound) { + throw error; + } else { + // By default, we'll just return undefined + return undefined; + } } return { From 883e216749d6e77c7146a34d7b0cb434c46d5115 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Tue, 15 Feb 2022 15:41:01 -0500 Subject: [PATCH 27/32] Fix a bunch of type errors around renamed fetch latest package version methods --- .../plugins/fleet/common/types/models/epm.ts | 6 ++ .../services/epm/package_service.test.ts | 2 +- .../server/services/epm/package_service.ts | 4 +- .../epm/packages/bulk_install_packages.ts | 2 +- ...undled_packages.ts => bundled_packages.ts} | 16 +++--- .../server/services/epm/packages/get.test.ts | 6 +- .../fleet/server/services/epm/packages/get.ts | 2 +- .../services/epm/packages/install.test.ts | 6 +- .../server/services/epm/packages/install.ts | 2 +- .../server/services/epm/registry/index.ts | 57 ++++++++----------- .../server/services/preconfiguration.test.ts | 4 +- x-pack/plugins/fleet/server/types/index.tsx | 1 + 12 files changed, 54 insertions(+), 54 deletions(-) rename x-pack/plugins/fleet/server/services/epm/packages/{get_bundled_packages.ts => bundled_packages.ts} (80%) diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index 5ea4f86f39a43..64ea5665241e1 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -109,6 +109,12 @@ export type ArchivePackage = PackageSpecManifest & // should an uploaded package be able to specify `internal`? Pick; +export interface BundledPackage { + name: string; + version: string; + buffer: Buffer; +} + export type RegistryPackage = PackageSpecManifest & Partial & RegistryAdditionalProperties & diff --git a/x-pack/plugins/fleet/server/services/epm/package_service.test.ts b/x-pack/plugins/fleet/server/services/epm/package_service.test.ts index 97ee5acc71023..5c48ddb050ff2 100644 --- a/x-pack/plugins/fleet/server/services/epm/package_service.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/package_service.test.ts @@ -91,7 +91,7 @@ function getTest( test = { method: mocks.packageClient.fetchFindLatestPackage.bind(mocks.packageClient), args: ['package name'], - spy: jest.spyOn(epmRegistry, 'fetchFindLatestPackage'), + spy: jest.spyOn(epmRegistry, 'fetchFindLatestPackageOrThrow'), spyArgs: ['package name'], spyResponse: { name: 'fetchFindLatestPackage test' }, }; diff --git a/x-pack/plugins/fleet/server/services/epm/package_service.ts b/x-pack/plugins/fleet/server/services/epm/package_service.ts index 68327932afdbf..cac69fe4bd3b8 100644 --- a/x-pack/plugins/fleet/server/services/epm/package_service.ts +++ b/x-pack/plugins/fleet/server/services/epm/package_service.ts @@ -19,7 +19,7 @@ import type { InstallablePackage, Installation, RegistryPackage, - RegistrySearchResult, + BundledPackage, } from '../../types'; import { checkSuperuser } from '../../routes/security'; import { FleetUnauthorizedError } from '../../errors'; @@ -44,7 +44,7 @@ export interface PackageClient { spaceId?: string; }): Promise; - fetchFindLatestPackage(packageName: string): Promise; + fetchFindLatestPackage(packageName: string): Promise; getRegistryPackage( packageName: string, diff --git a/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts b/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts index 518971a51e0ef..d68b2f67e3295 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts @@ -39,7 +39,7 @@ export async function bulkInstallPackages({ return Promise.resolve(pkg); } - return Registry.fetchFindLatestPackageWithFallbackToBundled(pkg); + return Registry.fetchFindLatestPackageOrThrow(pkg); }) ); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get_bundled_packages.ts b/x-pack/plugins/fleet/server/services/epm/packages/bundled_packages.ts similarity index 80% rename from x-pack/plugins/fleet/server/services/epm/packages/get_bundled_packages.ts rename to x-pack/plugins/fleet/server/services/epm/packages/bundled_packages.ts index a7bdda0d68c7e..8ccd2006ad846 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get_bundled_packages.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/bundled_packages.ts @@ -5,20 +5,15 @@ * 2.0. */ -import path from 'path'; import fs from 'fs/promises'; +import path from 'path'; +import type { BundledPackage } from '../../../types'; import { appContextService } from '../../app_context'; import { splitPkgKey } from '../registry'; const BUNDLED_PACKAGE_DIRECTORY = path.join(__dirname, '../../../bundled_packages'); -interface BundledPackage { - name: string; - version: string; - buffer: Buffer; -} - export async function getBundledPackages(): Promise { try { const dirContents = await fs.readdir(BUNDLED_PACKAGE_DIRECTORY); @@ -46,3 +41,10 @@ export async function getBundledPackages(): Promise { return []; } } + +export async function getBundledPackageByName(name: string): Promise { + const bundledPackages = await getBundledPackages(); + const bundledPackage = bundledPackages.find((pkg) => pkg.name === name); + + return bundledPackage; +} diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/get.test.ts index 5c4c9bb2a907b..b15c61cebd778 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get.test.ts @@ -186,7 +186,7 @@ describe('When using EPM `get` services', () => { beforeEach(() => { const mockContract = createAppContextStartContractMock(); appContextService.start(mockContract); - MockRegistry.fetchFindLatestPackageWithFallbackToBundled.mockResolvedValue({ + MockRegistry.fetchFindLatestPackageOrUndefined.mockResolvedValue({ name: 'my-package', version: '1.0.0', } as RegistryPackage); @@ -284,7 +284,7 @@ describe('When using EPM `get` services', () => { describe('registry fetch errors', () => { it('throws when a package that is not installed is not available in the registry and not bundled', async () => { - MockRegistry.fetchFindLatestPackageWithFallbackToBundled.mockResolvedValue(undefined); + MockRegistry.fetchFindLatestPackageOrUndefined.mockResolvedValue(undefined); const soClient = savedObjectsClientMock.create(); soClient.get.mockRejectedValue(SavedObjectsErrorHelpers.createGenericNotFoundError()); @@ -298,7 +298,7 @@ describe('When using EPM `get` services', () => { }); it('sets the latestVersion to installed version when an installed package is not available in the registry', async () => { - MockRegistry.fetchFindLatestPackageWithFallbackToBundled.mockResolvedValue(undefined); + MockRegistry.fetchFindLatestPackageOrUndefined.mockResolvedValue(undefined); const soClient = savedObjectsClientMock.create(); soClient.get.mockResolvedValue({ id: 'my-package', diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get.ts b/x-pack/plugins/fleet/server/services/epm/packages/get.ts index c0f45083617aa..fd24b3f438319 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get.ts @@ -146,7 +146,7 @@ export async function getPackageInfo(options: { const [savedObject, latestPackage] = await Promise.all([ getInstallationObject({ savedObjectsClient, pkgName }), - Registry.fetchFindLatestPackageWithFallbackToBundled(pkgName, { throwIfNotFound: false }), + Registry.fetchFindLatestPackageOrUndefined(pkgName), ]); if (!savedObject && !latestPackage) { diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts index aa16663227f36..1a1f1aa617f54 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts @@ -20,7 +20,7 @@ import { licenseService } from '../../license'; import { installPackage } from './install'; import * as install from './_install_package'; import * as obj from './index'; -import { getBundledPackages } from './get_bundled_packages'; +import { getBundledPackages } from './bundled_packages'; jest.mock('../../app_context', () => { return { @@ -41,7 +41,7 @@ jest.mock('../../upgrade_sender'); jest.mock('../../license'); jest.mock('../../upgrade_sender'); jest.mock('./cleanup'); -jest.mock('./get_bundled_packages'); +jest.mock('./bundled_packages'); jest.mock('./_install_package', () => { return { _installPackage: jest.fn(() => Promise.resolve()), @@ -76,7 +76,7 @@ describe('install', () => { return `${pkg.name}-${pkg.version}`; }); jest - .spyOn(Registry, 'fetchFindLatestPackage') + .spyOn(Registry, 'fetchFindLatestPackageOrThrow') .mockImplementation(() => Promise.resolve({ version: '1.3.0' } as any)); jest .spyOn(Registry, 'getRegistryPackage') diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.ts index fd3664d2c9ea3..107b906a969c8 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.ts @@ -44,7 +44,7 @@ import { removeInstallation } from './remove'; import { getPackageSavedObjects } from './get'; import { _installPackage } from './_install_package'; import { removeOldAssets } from './cleanup'; -import { getBundledPackages } from './get_bundled_packages'; +import { getBundledPackages } from './bundled_packages'; export async function isPackageInstalled(options: { savedObjectsClient: SavedObjectsClientContract; diff --git a/x-pack/plugins/fleet/server/services/epm/registry/index.ts b/x-pack/plugins/fleet/server/services/epm/registry/index.ts index e2c9ce6d0b2b5..8d716f5c7eac9 100644 --- a/x-pack/plugins/fleet/server/services/epm/registry/index.ts +++ b/x-pack/plugins/fleet/server/services/epm/registry/index.ts @@ -34,7 +34,7 @@ import { streamToBuffer } from '../streams'; import { appContextService } from '../..'; import { PackageNotFoundError, PackageCacheError, RegistryResponseError } from '../../../errors'; -import { getBundledPackages } from '../packages/get_bundled_packages'; +import { getBundledPackages, getBundledPackageByName } from '../packages/bundled_packages'; import { fetchUrl, getResponse, getResponseStream } from './requests'; import { getRegistryUrl } from './registry_url'; @@ -93,54 +93,45 @@ export async function fetchFindLatestPackageOrThrow( packageName: string, options?: FetchFindLatestPackageOptions ) { - const searchResults = await _fetchFindLatestPackage(packageName, options); + try { + const searchResults = await _fetchFindLatestPackage(packageName, options); - if (!searchResults.length) { - throw new PackageNotFoundError(`[${packageName}] package not found in registry`); - } + if (!searchResults.length) { + throw new PackageNotFoundError(`[${packageName}] package not found in registry`); + } - return searchResults[0]; -} + return searchResults[0]; + } catch (error) { + const bundledPackage = await getBundledPackageByName(packageName); -export async function fetchFindLatestPackageOrUndefined( - packageName: string, - options?: FetchFindLatestPackageOptions -) { - const searchResults = await _fetchFindLatestPackage(packageName, options); + if (!bundledPackage) { + throw error; + } - if (!searchResults.length) { - return undefined; + return bundledPackage; } - - return searchResults[0]; } -export async function fetchFindLatestPackageWithFallbackToBundled( +export async function fetchFindLatestPackageOrUndefined( packageName: string, - options?: FetchFindLatestPackageOptions & { throwIfNotFound?: boolean } + options?: FetchFindLatestPackageOptions ) { try { - const latestPackage = await fetchFindLatestPackageOrThrow(packageName, options); + const searchResults = await _fetchFindLatestPackage(packageName, options); + + if (!searchResults.length) { + return undefined; + } - return latestPackage; + return searchResults[0]; } catch (error) { - const bundledPackages = await getBundledPackages(); - const bundledPackage = bundledPackages.find((b) => b.name === packageName); + const bundledPackage = await getBundledPackageByName(packageName); if (!bundledPackage) { - // Callers must opt into re-throwing the original error from the registry request - if (options?.throwIfNotFound) { - throw error; - } else { - // By default, we'll just return undefined - return undefined; - } + return undefined; } - return { - name: bundledPackage.name, - version: bundledPackage.version, - }; + return bundledPackage; } } diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts index e0ebec5fc8e82..a4b192d38cd6d 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts @@ -33,12 +33,12 @@ import { } from './preconfiguration'; import { outputService } from './output'; import { packagePolicyService } from './package_policy'; -import { getBundledPackages } from './epm/packages/get_bundled_packages'; +import { getBundledPackages } from './epm/packages/bundled_packages'; import type { InstallPackageParams } from './epm/packages/install'; jest.mock('./agent_policy_update'); jest.mock('./output'); -jest.mock('./epm/packages/get_bundled_packages'); +jest.mock('./epm/packages/bundled_packages'); jest.mock('./epm/archive'); const mockedOutputService = outputService as jest.Mocked; diff --git a/x-pack/plugins/fleet/server/types/index.tsx b/x-pack/plugins/fleet/server/types/index.tsx index 9d3e912864785..91303046485d9 100644 --- a/x-pack/plugins/fleet/server/types/index.tsx +++ b/x-pack/plugins/fleet/server/types/index.tsx @@ -50,6 +50,7 @@ export type { EsAssetReference, KibanaAssetReference, RegistryPackage, + BundledPackage, InstallablePackage, AssetType, Installable, From 3541e4f55db582379217c942fbedad7721ea6b54 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Tue, 15 Feb 2022 16:09:01 -0500 Subject: [PATCH 28/32] Remove unused import --- x-pack/plugins/fleet/server/services/epm/registry/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/fleet/server/services/epm/registry/index.ts b/x-pack/plugins/fleet/server/services/epm/registry/index.ts index 8d716f5c7eac9..c70b064684a96 100644 --- a/x-pack/plugins/fleet/server/services/epm/registry/index.ts +++ b/x-pack/plugins/fleet/server/services/epm/registry/index.ts @@ -34,7 +34,7 @@ import { streamToBuffer } from '../streams'; import { appContextService } from '../..'; import { PackageNotFoundError, PackageCacheError, RegistryResponseError } from '../../../errors'; -import { getBundledPackages, getBundledPackageByName } from '../packages/bundled_packages'; +import { getBundledPackageByName } from '../packages/bundled_packages'; import { fetchUrl, getResponse, getResponseStream } from './requests'; import { getRegistryUrl } from './registry_url'; From 3a828d578a0975f36617021f8b5f1b50ec4b173b Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Tue, 15 Feb 2022 20:52:34 -0500 Subject: [PATCH 29/32] Bump docker version to latest snapshot (again) --- x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts | 2 +- .../fleet/server/integration_tests/docker_registry_helper.ts | 2 +- x-pack/plugins/fleet/server/services/preconfiguration.test.ts | 2 +- x-pack/test/fleet_api_integration/config.ts | 2 +- x-pack/test/functional/config.js | 2 +- x-pack/test/functional_synthetics/config.js | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts b/x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts index 3f8a7758527f7..6a4c2e03fac08 100644 --- a/x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts +++ b/x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts @@ -13,7 +13,7 @@ import { packageRegistryPort } from './ftr_config'; import { FtrProviderContext } from './ftr_provider_context'; export const dockerImage = - 'docker.elastic.co/package-registry/distribution@sha256:a30d5b428bc3b22e87a4ae3c4d06c00bbd757ea41ef9a2ccf01c670f2f111577'; + 'docker.elastic.co/package-registry/distribution@sha256:c5bf8e058727de72e561b228f4b254a14a6f880e582190d01bd5ff74318e1d0b'; async function ftrConfigRun({ readConfigFile }: FtrConfigProviderContext) { const kibanaConfig = await readConfigFile(require.resolve('./ftr_config.ts')); diff --git a/x-pack/plugins/fleet/server/integration_tests/docker_registry_helper.ts b/x-pack/plugins/fleet/server/integration_tests/docker_registry_helper.ts index ef3621558033b..ccea6d4bd919b 100644 --- a/x-pack/plugins/fleet/server/integration_tests/docker_registry_helper.ts +++ b/x-pack/plugins/fleet/server/integration_tests/docker_registry_helper.ts @@ -24,7 +24,7 @@ export function useDockerRegistry() { let dockerProcess: ChildProcess | undefined; async function startDockerRegistryServer() { - const dockerImage = `docker.elastic.co/package-registry/distribution@sha256:a30d5b428bc3b22e87a4ae3c4d06c00bbd757ea41ef9a2ccf01c670f2f111577`; + const dockerImage = `docker.elastic.co/package-registry/distribution@sha256:c5bf8e058727de72e561b228f4b254a14a6f880e582190d01bd5ff74318e1d0b`; const args = ['run', '--rm', '-p', `${packageRegistryPort}:8080`, dockerImage]; diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts index a4b192d38cd6d..518b79b9e8547 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts @@ -121,7 +121,7 @@ function getPutPreconfiguredPackagesMock() { jest.mock('./epm/registry', () => ({ ...jest.requireActual('./epm/registry'), - async fetchFindLatestPackage(packageName: string): Promise { + async fetchFindLatestPackageOrThrow(packageName: string): Promise { return { name: packageName, version: '1.0.0', diff --git a/x-pack/test/fleet_api_integration/config.ts b/x-pack/test/fleet_api_integration/config.ts index 4b244cdc8c842..28af25c20181a 100644 --- a/x-pack/test/fleet_api_integration/config.ts +++ b/x-pack/test/fleet_api_integration/config.ts @@ -15,7 +15,7 @@ import { defineDockerServersConfig } from '@kbn/test'; // example: https://beats-ci.elastic.co/blue/organizations/jenkins/Ingest-manager%2Fpackage-storage/detail/snapshot/74/pipeline/257#step-302-log-1. // It should be updated any time there is a new Docker image published for the Snapshot Distribution of the Package Registry. export const dockerImage = - 'docker.elastic.co/package-registry/distribution@sha256:a30d5b428bc3b22e87a4ae3c4d06c00bbd757ea41ef9a2ccf01c670f2f111577'; + 'docker.elastic.co/package-registry/distribution@sha256:c5bf8e058727de72e561b228f4b254a14a6f880e582190d01bd5ff74318e1d0b'; export default async function ({ readConfigFile }: FtrConfigProviderContext) { const xPackAPITestsConfig = await readConfigFile(require.resolve('../api_integration/config.ts')); diff --git a/x-pack/test/functional/config.js b/x-pack/test/functional/config.js index d67d95ac11507..7e1ac8e5481bc 100644 --- a/x-pack/test/functional/config.js +++ b/x-pack/test/functional/config.js @@ -15,7 +15,7 @@ import { pageObjects } from './page_objects'; // example: https://beats-ci.elastic.co/blue/organizations/jenkins/Ingest-manager%2Fpackage-storage/detail/snapshot/74/pipeline/257#step-302-log-1. // It should be updated any time there is a new Docker image published for the Snapshot Distribution of the Package Registry. export const dockerImage = - 'docker.elastic.co/package-registry/distribution@sha256:a30d5b428bc3b22e87a4ae3c4d06c00bbd757ea41ef9a2ccf01c670f2f111577'; + 'docker.elastic.co/package-registry/distribution@sha256:c5bf8e058727de72e561b228f4b254a14a6f880e582190d01bd5ff74318e1d0b'; // the default export of config files must be a config provider // that returns an object with the projects config values diff --git a/x-pack/test/functional_synthetics/config.js b/x-pack/test/functional_synthetics/config.js index 1716243d07eba..f9074812a7e22 100644 --- a/x-pack/test/functional_synthetics/config.js +++ b/x-pack/test/functional_synthetics/config.js @@ -17,7 +17,7 @@ import { pageObjects } from './page_objects'; // example: https://beats-ci.elastic.co/blue/organizations/jenkins/Ingest-manager%2Fpackage-storage/detail/snapshot/74/pipeline/257#step-302-log-1. // It should be updated any time there is a new Docker image published for the Snapshot Distribution of the Package Registry that updates Synthetics. export const dockerImage = - 'docker.elastic.co/package-registry/distribution@sha256:a30d5b428bc3b22e87a4ae3c4d06c00bbd757ea41ef9a2ccf01c670f2f111577'; + 'docker.elastic.co/package-registry/distribution@sha256:c5bf8e058727de72e561b228f4b254a14a6f880e582190d01bd5ff74318e1d0b'; // the default export of config files must be a config provider // that returns an object with the projects config values From 9e7b122e000e905837b8a2b151afc39b4f84ba5c Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Wed, 16 Feb 2022 09:26:25 -0500 Subject: [PATCH 30/32] Revert changes to endpoint tests --- .../security_solution_endpoint/apps/endpoint/endpoint_list.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/test/security_solution_endpoint/apps/endpoint/endpoint_list.ts b/x-pack/test/security_solution_endpoint/apps/endpoint/endpoint_list.ts index 022bd437b989b..3a49278bd21a8 100644 --- a/x-pack/test/security_solution_endpoint/apps/endpoint/endpoint_list.ts +++ b/x-pack/test/security_solution_endpoint/apps/endpoint/endpoint_list.ts @@ -20,6 +20,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { const testSubjects = getService('testSubjects'); const browser = getService('browser'); const endpointTestResources = getService('endpointTestResources'); + const policyTestResources = getService('policyTestResources'); const expectedData = [ [ @@ -89,6 +90,8 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { describe('when there is data,', () => { before(async () => { + const endpointPackage = await policyTestResources.getEndpointPackage(); + await endpointTestResources.setMetadataTransformFrequency('1s', endpointPackage.version); indexedData = await endpointTestResources.loadEndpointData({ numHosts: 3 }); await pageObjects.endpoint.navigateToEndpointList(); await pageObjects.endpoint.waitForTableToHaveNumberOfEntries('endpointListTable', 3, 90000); From 351ce75b5034474f7560d0030a8f593d9314e796 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Wed, 16 Feb 2022 09:56:21 -0500 Subject: [PATCH 31/32] Pass experimental flag in synthetics tests --- .../functional_synthetics/services/uptime/synthetics_package.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/test/functional_synthetics/services/uptime/synthetics_package.ts b/x-pack/test/functional_synthetics/services/uptime/synthetics_package.ts index b0d935c408e4d..898d527245b16 100644 --- a/x-pack/test/functional_synthetics/services/uptime/synthetics_package.ts +++ b/x-pack/test/functional_synthetics/services/uptime/synthetics_package.ts @@ -50,6 +50,7 @@ export function SyntheticsPackageProvider({ getService }: FtrProviderContext) { apiRequest = retry.try(() => { return supertest .get(INGEST_API_EPM_PACKAGES) + .query({ experimental: true }) .set('kbn-xsrf', 'xxx') .expect(200) .catch((error) => { From 7938cd7031f2a2c95ca801a0b7488fe4f50afe03 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Wed, 16 Feb 2022 12:55:27 -0500 Subject: [PATCH 32/32] Fix endpoint version in fleet api integration test --- x-pack/test/fleet_api_integration/apis/package_policy/create.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/fleet_api_integration/apis/package_policy/create.ts b/x-pack/test/fleet_api_integration/apis/package_policy/create.ts index 0eb5785198a1b..da8efafe8b637 100644 --- a/x-pack/test/fleet_api_integration/apis/package_policy/create.ts +++ b/x-pack/test/fleet_api_integration/apis/package_policy/create.ts @@ -236,7 +236,7 @@ export default function (providerContext: FtrProviderContext) { package: { name: 'endpoint', title: 'Endpoint', - version: '1.3.0-dev.0', + version: '1.3.0', }, }) .expect(400);