From ba8c339d84c1d26441f049650296ab4072147128 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Wed, 16 Feb 2022 14:39:15 -0500 Subject: [PATCH] [Fleet] Allow bundled installs to occur even if EPR is unreachable (#125127) * Allow bundled installs to occur even if EPR is unreachable * Fix type errors in test * Fix failing test * fixup! Fix failing test * Remove unused object in mock * Make creation of preconfigured agent policy functional * Always fall back to bundled packages if available * Remove unused import * Use packageInfo object instead of RegistryPackage where possible * Fix type error in assets test * Fix test timeouts * Fix promise logic for registry fetch fallback * Use archive package as default in create package policy * Always install from bundled package if it exists - regardless of installation context * Clean up + refactor a bit * Default to cached package archive for policy updates * Update mock in get.test.ts * Add test for install from bundled package logic * Delete timeout call in security solution tests * Fix unused var in endpoint test * Fix another unused var in endpoint test * [Debug] Add some logging to test installation times in CI * Revert "[Debug] Add some logging to test installation times in CI" This reverts commit 513dcc121e4ab52e2d482ebb54a45af06c6f6b6c. * Update docker images for registry * Update docker image digest again * Refactor latest package fetching to fix broken logic/tests * Fix a bunch of type errors around renamed fetch latest package version methods * Remove unused import * Bump docker version to latest snapshot (again) * Revert changes to endpoint tests * Pass experimental flag in synthetics tests * Fix endpoint version in fleet api integration test Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts | 2 +- .../plugins/fleet/common/types/models/epm.ts | 8 +- x-pack/plugins/fleet/server/errors/index.ts | 1 + .../docker_registry_helper.ts | 2 +- .../agent_policies/full_agent_policy.ts | 2 +- .../epm/elasticsearch/template/install.ts | 13 ++- .../fleet/server/services/epm/fields/field.ts | 4 +- .../services/epm/package_service.test.ts | 2 +- .../server/services/epm/package_service.ts | 8 +- .../services/epm/packages/assets.test.ts | 4 +- .../server/services/epm/packages/assets.ts | 6 +- .../epm/packages/bulk_install_packages.ts | 80 ++++--------------- ...undled_packages.ts => bundled_packages.ts} | 21 +++-- .../server/services/epm/packages/get.test.ts | 8 +- .../fleet/server/services/epm/packages/get.ts | 5 +- .../services/epm/packages/install.test.ts | 39 ++++++++- .../server/services/epm/packages/install.ts | 30 ++++++- .../server/services/epm/registry/index.ts | 78 +++++++++++++----- .../server/services/epm/registry/requests.ts | 15 ++-- .../fleet/server/services/package_policy.ts | 50 ++++++++---- .../server/services/preconfiguration.test.ts | 16 ++-- .../fleet/server/services/preconfiguration.ts | 3 - x-pack/plugins/fleet/server/types/index.tsx | 1 + .../apis/package_policy/create.ts | 4 +- x-pack/test/fleet_api_integration/config.ts | 2 +- x-pack/test/functional/config.js | 2 +- x-pack/test/functional_synthetics/config.js | 2 +- .../services/uptime/synthetics_package.ts | 1 + 28 files changed, 253 insertions(+), 156 deletions(-) rename x-pack/plugins/fleet/server/services/epm/packages/{get_bundled_packages.ts => bundled_packages.ts} (69%) diff --git a/x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts b/x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts index 8fc87b49a660..6a4c2e03fac0 100644 --- a/x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts +++ b/x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts @@ -13,7 +13,7 @@ import { packageRegistryPort } from './ftr_config'; import { FtrProviderContext } from './ftr_provider_context'; export const dockerImage = - 'docker.elastic.co/package-registry/distribution@sha256:de952debe048d903fc73e8a4472bb48bb95028d440cba852f21b863d47020c61'; + 'docker.elastic.co/package-registry/distribution@sha256:c5bf8e058727de72e561b228f4b254a14a6f880e582190d01bd5ff74318e1d0b'; async function ftrConfigRun({ readConfigFile }: FtrConfigProviderContext) { const kibanaConfig = await readConfigFile(require.resolve('./ftr_config.ts')); diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index 983ee7fff3db..64ea5665241e 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -107,7 +107,13 @@ export type InstallablePackage = RegistryPackage | ArchivePackage; export type ArchivePackage = PackageSpecManifest & // should an uploaded package be able to specify `internal`? - Pick; + Pick; + +export interface BundledPackage { + name: string; + version: string; + buffer: Buffer; +} export type RegistryPackage = PackageSpecManifest & Partial & diff --git a/x-pack/plugins/fleet/server/errors/index.ts b/x-pack/plugins/fleet/server/errors/index.ts index 380862d36fe0..0d8627c13b3d 100644 --- a/x-pack/plugins/fleet/server/errors/index.ts +++ b/x-pack/plugins/fleet/server/errors/index.ts @@ -43,6 +43,7 @@ export class ConcurrentInstallOperationError extends IngestManagerError {} export class AgentReassignmentError extends IngestManagerError {} export class PackagePolicyIneligibleForUpgradeError extends IngestManagerError {} export class PackagePolicyValidationError extends IngestManagerError {} +export class BundledPackageNotFoundError extends IngestManagerError {} export class HostedAgentPolicyRestrictionRelatedError extends IngestManagerError { constructor(message = 'Cannot perform that action') { super( diff --git a/x-pack/plugins/fleet/server/integration_tests/docker_registry_helper.ts b/x-pack/plugins/fleet/server/integration_tests/docker_registry_helper.ts index 31b0831d7f3e..ccea6d4bd919 100644 --- a/x-pack/plugins/fleet/server/integration_tests/docker_registry_helper.ts +++ b/x-pack/plugins/fleet/server/integration_tests/docker_registry_helper.ts @@ -24,7 +24,7 @@ export function useDockerRegistry() { let dockerProcess: ChildProcess | undefined; async function startDockerRegistryServer() { - const dockerImage = `docker.elastic.co/package-registry/distribution@sha256:de952debe048d903fc73e8a4472bb48bb95028d440cba852f21b863d47020c61`; + const dockerImage = `docker.elastic.co/package-registry/distribution@sha256:c5bf8e058727de72e561b228f4b254a14a6f880e582190d01bd5ff74318e1d0b`; const args = ['run', '--rm', '-p', `${packageRegistryPort}:8080`, dockerImage]; diff --git a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts index 17762ad5ada8..9f522875544e 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts @@ -34,7 +34,7 @@ export async function getFullAgentPolicy( options?: { standalone: boolean } ): Promise { let agentPolicy; - const standalone = options?.standalone; + const standalone = options?.standalone ?? false; try { agentPolicy = await agentPolicyService.get(soClient, id); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index 6bd346f3aff8..1303db1a36c0 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -16,6 +16,7 @@ import type { RegistryElasticsearch, InstallablePackage, IndexTemplate, + PackageInfo, } from '../../../../types'; import { loadFieldsFromYaml, processFields } from '../../fields/field'; import type { Field } from '../../fields/field'; @@ -31,6 +32,8 @@ import type { ESAssetMetadata } from '../meta'; import { getESAssetMetadata } from '../meta'; import { retryTransientEsErrors } from '../retry'; +import { getPackageInfo } from '../../packages'; + import { generateMappings, generateTemplateName, @@ -62,10 +65,16 @@ export const installTemplates = async ( const dataStreams = installablePackage.data_streams; if (!dataStreams) return []; + const packageInfo = await getPackageInfo({ + savedObjectsClient, + pkgName: installablePackage.name, + pkgVersion: installablePackage.version, + }); + const installedTemplatesNested = await Promise.all( dataStreams.map((dataStream) => installTemplateForDataStream({ - pkg: installablePackage, + pkg: packageInfo, esClient, logger, dataStream, @@ -177,7 +186,7 @@ export async function installTemplateForDataStream({ logger, dataStream, }: { - pkg: InstallablePackage; + pkg: PackageInfo; esClient: ElasticsearchClient; logger: Logger; dataStream: RegistryDataStream; diff --git a/x-pack/plugins/fleet/server/services/epm/fields/field.ts b/x-pack/plugins/fleet/server/services/epm/fields/field.ts index d854a0fe8e74..06ff858df678 100644 --- a/x-pack/plugins/fleet/server/services/epm/fields/field.ts +++ b/x-pack/plugins/fleet/server/services/epm/fields/field.ts @@ -7,7 +7,7 @@ import { safeLoad } from 'js-yaml'; -import type { InstallablePackage } from '../../../types'; +import type { PackageInfo } from '../../../types'; import { getAssetsData } from '../packages/assets'; // This should become a copy of https://github.com/elastic/beats/blob/d9a4c9c240a9820fab15002592e5bb6db318543b/libbeat/mapping/field.go#L39 @@ -261,7 +261,7 @@ const isFields = (path: string) => { */ export const loadFieldsFromYaml = async ( - pkg: InstallablePackage, + pkg: PackageInfo, datasetName?: string ): Promise => { // Fetch all field definition files diff --git a/x-pack/plugins/fleet/server/services/epm/package_service.test.ts b/x-pack/plugins/fleet/server/services/epm/package_service.test.ts index 97ee5acc7102..5c48ddb050ff 100644 --- a/x-pack/plugins/fleet/server/services/epm/package_service.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/package_service.test.ts @@ -91,7 +91,7 @@ function getTest( test = { method: mocks.packageClient.fetchFindLatestPackage.bind(mocks.packageClient), args: ['package name'], - spy: jest.spyOn(epmRegistry, 'fetchFindLatestPackage'), + spy: jest.spyOn(epmRegistry, 'fetchFindLatestPackageOrThrow'), spyArgs: ['package name'], spyResponse: { name: 'fetchFindLatestPackage test' }, }; diff --git a/x-pack/plugins/fleet/server/services/epm/package_service.ts b/x-pack/plugins/fleet/server/services/epm/package_service.ts index 0d9b8cb74b50..cac69fe4bd3b 100644 --- a/x-pack/plugins/fleet/server/services/epm/package_service.ts +++ b/x-pack/plugins/fleet/server/services/epm/package_service.ts @@ -19,13 +19,13 @@ import type { InstallablePackage, Installation, RegistryPackage, - RegistrySearchResult, + BundledPackage, } from '../../types'; import { checkSuperuser } from '../../routes/security'; import { FleetUnauthorizedError } from '../../errors'; import { installTransform, isTransform } from './elasticsearch/transform/install'; -import { fetchFindLatestPackage, getRegistryPackage } from './registry'; +import { fetchFindLatestPackageOrThrow, getRegistryPackage } from './registry'; import { ensureInstalledPackage, getInstallation } from './packages'; export type InstalledAssetType = EsAssetReference; @@ -44,7 +44,7 @@ export interface PackageClient { spaceId?: string; }): Promise; - fetchFindLatestPackage(packageName: string): Promise; + fetchFindLatestPackage(packageName: string): Promise; getRegistryPackage( packageName: string, @@ -117,7 +117,7 @@ class PackageClientImpl implements PackageClient { public async fetchFindLatestPackage(packageName: string) { await this.#runPreflight(); - return fetchFindLatestPackage(packageName); + return fetchFindLatestPackageOrThrow(packageName); } public async getRegistryPackage(packageName: string, packageVersion: string) { diff --git a/x-pack/plugins/fleet/server/services/epm/packages/assets.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/assets.test.ts index c5b104696aaf..b019729b65eb 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/assets.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/assets.test.ts @@ -5,7 +5,7 @@ * 2.0. */ -import type { InstallablePackage } from '../../../types'; +import type { PackageInfo } from '../../../types'; import { getArchiveFilelist } from '../archive/cache'; @@ -66,7 +66,7 @@ const tests = [ test('testGetAssets', () => { for (const value of tests) { // as needed to pretend it is an InstallablePackage - const assets = getAssets(value.package as InstallablePackage, value.filter, value.dataset); + const assets = getAssets(value.package as PackageInfo, value.filter, value.dataset); expect(assets).toStrictEqual(value.expected); } }); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/assets.ts b/x-pack/plugins/fleet/server/services/epm/packages/assets.ts index c28c982f4ea4..c939ce093a65 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/assets.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/assets.ts @@ -5,7 +5,7 @@ * 2.0. */ -import type { InstallablePackage } from '../../../types'; +import type { PackageInfo } from '../../../types'; import { getArchiveFilelist, getAsset } from '../archive'; import type { ArchiveEntry } from '../archive'; @@ -17,7 +17,7 @@ import type { ArchiveEntry } from '../archive'; // and different package and version structure export function getAssets( - packageInfo: InstallablePackage, + packageInfo: PackageInfo, filter = (path: string): boolean => true, datasetName?: string ): string[] { @@ -52,7 +52,7 @@ export function getAssets( // ASK: Does getAssetsData need an installSource now? // if so, should it be an Installation vs InstallablePackage or add another argument? export async function getAssetsData( - packageInfo: InstallablePackage, + packageInfo: PackageInfo, filter = (path: string): boolean => true, datasetName?: string ): Promise { diff --git a/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts b/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts index a32809672e1b..d68b2f67e329 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts @@ -14,7 +14,6 @@ import type { InstallResult } from '../../../types'; import { installPackage, isPackageVersionOrLaterInstalled } from './install'; import type { BulkInstallResponse, IBulkInstallPackageError } from './install'; -import { getBundledPackages } from './get_bundled_packages'; interface BulkInstallPackagesParams { savedObjectsClient: SavedObjectsClientContract; @@ -31,23 +30,23 @@ export async function bulkInstallPackages({ esClient, spaceId, force, - preferredSource = 'registry', }: BulkInstallPackagesParams): Promise { const logger = appContextService.getLogger(); - const bundledPackages = await getBundledPackages(); - const packagesResults = await Promise.allSettled( - packagesToInstall.map((pkg) => { - if (typeof pkg === 'string') return Registry.fetchFindLatestPackage(pkg); - return Promise.resolve(pkg); + packagesToInstall.map(async (pkg) => { + if (typeof pkg !== 'string') { + return Promise.resolve(pkg); + } + + return Registry.fetchFindLatestPackageOrThrow(pkg); }) ); logger.debug( - `kicking off bulk install of ${packagesToInstall.join( - ', ' - )} with preferred source of "${preferredSource}"` + `kicking off bulk install of ${packagesToInstall + .map((pkg) => (typeof pkg === 'string' ? pkg : pkg.name)) + .join(', ')}` ); const bulkInstallResults = await Promise.allSettled( @@ -83,61 +82,16 @@ export async function bulkInstallPackages({ }; } - let installResult: InstallResult; const pkgkey = Registry.pkgToPkgKey(pkgKeyProps); - const bundledPackage = bundledPackages.find((pkg) => pkg.name === pkgkey); - - // If preferred source is bundled packages on disk, attempt to install from disk first, then fall back to registry - if (preferredSource === 'bundled') { - if (bundledPackage) { - logger.debug( - `kicking off install of ${pkgKeyProps.name}-${pkgKeyProps.version} from bundled package on disk` - ); - installResult = await installPackage({ - savedObjectsClient, - esClient, - installSource: 'upload', - archiveBuffer: bundledPackage.buffer, - contentType: 'application/zip', - spaceId, - }); - } else { - installResult = await installPackage({ - savedObjectsClient, - esClient, - pkgkey, - installSource: 'registry', - spaceId, - force, - }); - } - } else { - // If preferred source is registry, attempt to install from registry first, then fall back to bundled packages on disk - installResult = await installPackage({ - savedObjectsClient, - esClient, - pkgkey, - installSource: 'registry', - spaceId, - force, - }); - - // If we initially errored, try to install from bundled package on disk - if (installResult.error && bundledPackage) { - logger.debug( - `kicking off install of ${pkgKeyProps.name}-${pkgKeyProps.version} from bundled package on disk` - ); - installResult = await installPackage({ - savedObjectsClient, - esClient, - installSource: 'upload', - archiveBuffer: bundledPackage.buffer, - contentType: 'application/zip', - spaceId, - }); - } - } + const installResult = await installPackage({ + savedObjectsClient, + esClient, + pkgkey, + installSource: 'registry', + spaceId, + force, + }); if (installResult.error) { return { diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get_bundled_packages.ts b/x-pack/plugins/fleet/server/services/epm/packages/bundled_packages.ts similarity index 69% rename from x-pack/plugins/fleet/server/services/epm/packages/get_bundled_packages.ts rename to x-pack/plugins/fleet/server/services/epm/packages/bundled_packages.ts index a9f9b754640c..8ccd2006ad84 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get_bundled_packages.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/bundled_packages.ts @@ -5,18 +5,15 @@ * 2.0. */ -import path from 'path'; import fs from 'fs/promises'; +import path from 'path'; +import type { BundledPackage } from '../../../types'; import { appContextService } from '../../app_context'; +import { splitPkgKey } from '../registry'; const BUNDLED_PACKAGE_DIRECTORY = path.join(__dirname, '../../../bundled_packages'); -interface BundledPackage { - name: string; - buffer: Buffer; -} - export async function getBundledPackages(): Promise { try { const dirContents = await fs.readdir(BUNDLED_PACKAGE_DIRECTORY); @@ -26,8 +23,11 @@ export async function getBundledPackages(): Promise { zipFiles.map(async (zipFile) => { const file = await fs.readFile(path.join(BUNDLED_PACKAGE_DIRECTORY, zipFile)); + const { pkgName, pkgVersion } = splitPkgKey(zipFile.replace(/\.zip$/, '')); + return { - name: zipFile.replace(/\.zip$/, ''), + name: pkgName, + version: pkgVersion, buffer: file, }; }) @@ -41,3 +41,10 @@ export async function getBundledPackages(): Promise { return []; } } + +export async function getBundledPackageByName(name: string): Promise { + const bundledPackages = await getBundledPackages(); + const bundledPackage = bundledPackages.find((pkg) => pkg.name === name); + + return bundledPackage; +} diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/get.test.ts index 53b4d341beec..b15c61cebd77 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get.test.ts @@ -186,7 +186,7 @@ describe('When using EPM `get` services', () => { beforeEach(() => { const mockContract = createAppContextStartContractMock(); appContextService.start(mockContract); - MockRegistry.fetchFindLatestPackage.mockResolvedValue({ + MockRegistry.fetchFindLatestPackageOrUndefined.mockResolvedValue({ name: 'my-package', version: '1.0.0', } as RegistryPackage); @@ -283,8 +283,8 @@ describe('When using EPM `get` services', () => { }); describe('registry fetch errors', () => { - it('throws when a package that is not installed is not available in the registry', async () => { - MockRegistry.fetchFindLatestPackage.mockResolvedValue(undefined); + it('throws when a package that is not installed is not available in the registry and not bundled', async () => { + MockRegistry.fetchFindLatestPackageOrUndefined.mockResolvedValue(undefined); const soClient = savedObjectsClientMock.create(); soClient.get.mockRejectedValue(SavedObjectsErrorHelpers.createGenericNotFoundError()); @@ -298,7 +298,7 @@ describe('When using EPM `get` services', () => { }); it('sets the latestVersion to installed version when an installed package is not available in the registry', async () => { - MockRegistry.fetchFindLatestPackage.mockResolvedValue(undefined); + MockRegistry.fetchFindLatestPackageOrUndefined.mockResolvedValue(undefined); const soClient = savedObjectsClientMock.create(); soClient.get.mockResolvedValue({ id: 'my-package', diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get.ts b/x-pack/plugins/fleet/server/services/epm/packages/get.ts index c78f107cce71..fd24b3f43831 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get.ts @@ -106,7 +106,7 @@ export async function getPackageInfoFromRegistry(options: { const { savedObjectsClient, pkgName, pkgVersion } = options; const [savedObject, latestPackage] = await Promise.all([ getInstallationObject({ savedObjectsClient, pkgName }), - Registry.fetchFindLatestPackage(pkgName), + Registry.fetchFindLatestPackageOrThrow(pkgName), ]); // If no package version is provided, use the installed version in the response @@ -143,9 +143,10 @@ export async function getPackageInfo(options: { pkgVersion: string; }): Promise { const { savedObjectsClient, pkgName, pkgVersion } = options; + const [savedObject, latestPackage] = await Promise.all([ getInstallationObject({ savedObjectsClient, pkgName }), - Registry.fetchFindLatestPackage(pkgName, { throwIfNotFound: false }), + Registry.fetchFindLatestPackageOrUndefined(pkgName), ]); if (!savedObject && !latestPackage) { diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts index b74466bc6271..1a1f1aa617f5 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts @@ -20,6 +20,7 @@ import { licenseService } from '../../license'; import { installPackage } from './install'; import * as install from './_install_package'; import * as obj from './index'; +import { getBundledPackages } from './bundled_packages'; jest.mock('../../app_context', () => { return { @@ -40,6 +41,7 @@ jest.mock('../../upgrade_sender'); jest.mock('../../license'); jest.mock('../../upgrade_sender'); jest.mock('./cleanup'); +jest.mock('./bundled_packages'); jest.mock('./_install_package', () => { return { _installPackage: jest.fn(() => Promise.resolve()), @@ -60,6 +62,8 @@ jest.mock('../archive', () => { }; }); +const mockGetBundledPackages = getBundledPackages as jest.MockedFunction; + describe('install', () => { beforeEach(() => { jest.spyOn(Registry, 'splitPkgKey').mockImplementation((pkgKey: string) => { @@ -67,14 +71,25 @@ describe('install', () => { return { pkgName, pkgVersion }; }); jest - .spyOn(Registry, 'fetchFindLatestPackage') + .spyOn(Registry, 'pkgToPkgKey') + .mockImplementation((pkg: { name: string; version: string }) => { + return `${pkg.name}-${pkg.version}`; + }); + jest + .spyOn(Registry, 'fetchFindLatestPackageOrThrow') .mockImplementation(() => Promise.resolve({ version: '1.3.0' } as any)); jest .spyOn(Registry, 'getRegistryPackage') .mockImplementation(() => Promise.resolve({ packageInfo: { license: 'basic' } } as any)); + + mockGetBundledPackages.mockReset(); }); describe('registry', () => { + beforeEach(() => { + mockGetBundledPackages.mockResolvedValue([]); + }); + it('should send telemetry on install failure, out of date', async () => { await installPackage({ spaceId: DEFAULT_SPACE_ID, @@ -187,6 +202,28 @@ describe('install', () => { status: 'failure', }); }); + + it('should install from bundled package if one exists', async () => { + mockGetBundledPackages.mockResolvedValue([ + { + name: 'test_package', + version: '1.0.0', + buffer: Buffer.from('test_package'), + }, + ]); + + await installPackage({ + spaceId: DEFAULT_SPACE_ID, + installSource: 'registry', + pkgkey: 'test_package-1.0.0', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); + + expect(install._installPackage).toHaveBeenCalledWith( + expect.objectContaining({ installSource: 'upload' }) + ); + }); }); describe('upload', () => { diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.ts index 9ffae48cb02d..107b906a969c 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.ts @@ -44,6 +44,7 @@ import { removeInstallation } from './remove'; import { getPackageSavedObjects } from './get'; import { _installPackage } from './_install_package'; import { removeOldAssets } from './cleanup'; +import { getBundledPackages } from './bundled_packages'; export async function isPackageInstalled(options: { savedObjectsClient: SavedObjectsClientContract; @@ -88,7 +89,7 @@ export async function ensureInstalledPackage(options: { // If pkgVersion isn't specified, find the latest package version const pkgKeyProps = pkgVersion ? { name: pkgName, version: pkgVersion } - : await Registry.fetchFindLatestPackage(pkgName); + : await Registry.fetchFindLatestPackageOrThrow(pkgName); const installedPackageResult = await isPackageVersionOrLaterInstalled({ savedObjectsClient, @@ -251,7 +252,9 @@ async function installPackageFromRegistry({ installType = getInstallType({ pkgVersion, installedPkg }); // get latest package version - const latestPackage = await Registry.fetchFindLatestPackage(pkgName, { ignoreConstraints }); + const latestPackage = await Registry.fetchFindLatestPackageOrThrow(pkgName, { + ignoreConstraints, + }); // let the user install if using the force flag or needing to reinstall or install a previous version due to failed update const installOutOfDateVersionOk = @@ -470,8 +473,31 @@ export async function installPackage(args: InstallPackageParams) { const logger = appContextService.getLogger(); const { savedObjectsClient, esClient } = args; + const bundledPackages = await getBundledPackages(); + if (args.installSource === 'registry') { const { pkgkey, force, ignoreConstraints, spaceId } = args; + + const matchingBundledPackage = bundledPackages.find( + (pkg) => Registry.pkgToPkgKey(pkg) === pkgkey + ); + + if (matchingBundledPackage) { + logger.debug( + `found bundled package for requested install of ${pkgkey} - installing from bundled package archive` + ); + + const response = installPackageByUpload({ + savedObjectsClient, + esClient, + archiveBuffer: matchingBundledPackage.buffer, + contentType: 'application/zip', + spaceId, + }); + + return response; + } + logger.debug(`kicking off install of ${pkgkey} from registry`); const response = installPackageFromRegistry({ savedObjectsClient, diff --git a/x-pack/plugins/fleet/server/services/epm/registry/index.ts b/x-pack/plugins/fleet/server/services/epm/registry/index.ts index 12712905b1d3..c70b064684a9 100644 --- a/x-pack/plugins/fleet/server/services/epm/registry/index.ts +++ b/x-pack/plugins/fleet/server/services/epm/registry/index.ts @@ -21,7 +21,6 @@ import type { InstallSource, RegistryPackage, RegistrySearchResults, - RegistrySearchResult, GetCategoriesRequest, } from '../../../types'; import { @@ -35,6 +34,8 @@ import { streamToBuffer } from '../streams'; import { appContextService } from '../..'; import { PackageNotFoundError, PackageCacheError, RegistryResponseError } from '../../../errors'; +import { getBundledPackageByName } from '../packages/bundled_packages'; + import { fetchUrl, getResponse, getResponseStream } from './requests'; import { getRegistryUrl } from './registry_url'; @@ -65,20 +66,16 @@ export async function fetchList(params?: SearchParams): Promise; -export async function fetchFindLatestPackage( - packageName: string, - options: { ignoreConstraints?: boolean; throwIfNotFound: false } -): Promise; -export async function fetchFindLatestPackage( +interface FetchFindLatestPackageOptions { + ignoreConstraints?: boolean; +} + +async function _fetchFindLatestPackage( packageName: string, - options?: { ignoreConstraints?: boolean; throwIfNotFound?: boolean } -): Promise { - const { ignoreConstraints = false, throwIfNotFound = true } = options ?? {}; + options?: FetchFindLatestPackageOptions +) { + const { ignoreConstraints = false } = options ?? {}; + const registryUrl = getRegistryUrl(); const url = new URL(`${registryUrl}/search?package=${packageName}&experimental=true`); @@ -86,12 +83,55 @@ export async function fetchFindLatestPackage( setKibanaVersion(url); } - const res = await fetchUrl(url.toString()); - const searchResults = JSON.parse(res); - if (searchResults.length) { + const res = await fetchUrl(url.toString(), 1); + const searchResults: RegistryPackage[] = JSON.parse(res); + + return searchResults; +} + +export async function fetchFindLatestPackageOrThrow( + packageName: string, + options?: FetchFindLatestPackageOptions +) { + try { + const searchResults = await _fetchFindLatestPackage(packageName, options); + + if (!searchResults.length) { + throw new PackageNotFoundError(`[${packageName}] package not found in registry`); + } + + return searchResults[0]; + } catch (error) { + const bundledPackage = await getBundledPackageByName(packageName); + + if (!bundledPackage) { + throw error; + } + + return bundledPackage; + } +} + +export async function fetchFindLatestPackageOrUndefined( + packageName: string, + options?: FetchFindLatestPackageOptions +) { + try { + const searchResults = await _fetchFindLatestPackage(packageName, options); + + if (!searchResults.length) { + return undefined; + } + return searchResults[0]; - } else if (throwIfNotFound) { - throw new PackageNotFoundError(`[${packageName}] package not found in registry`); + } catch (error) { + const bundledPackage = await getBundledPackageByName(packageName); + + if (!bundledPackage) { + return undefined; + } + + return bundledPackage; } } diff --git a/x-pack/plugins/fleet/server/services/epm/registry/requests.ts b/x-pack/plugins/fleet/server/services/epm/registry/requests.ts index f5cabadc5c60..47084b601a27 100644 --- a/x-pack/plugins/fleet/server/services/epm/registry/requests.ts +++ b/x-pack/plugins/fleet/server/services/epm/registry/requests.ts @@ -34,13 +34,13 @@ async function registryFetch(url: string) { } } -export async function getResponse(url: string): Promise { +export async function getResponse(url: string, retries: number = 5): Promise { try { // we only want to retry certain failures like network issues // the rest should only try the one time then fail as they do now const response = await pRetry(() => registryFetch(url), { factor: 2, - retries: 5, + retries, onFailedAttempt: (error) => { // we only want to retry certain types of errors, like `ECONNREFUSED` and other operational errors // and let the others through without retrying @@ -67,13 +67,16 @@ export async function getResponse(url: string): Promise { } } -export async function getResponseStream(url: string): Promise { - const res = await getResponse(url); +export async function getResponseStream( + url: string, + retries?: number +): Promise { + const res = await getResponse(url, retries); return res.body; } -export async function fetchUrl(url: string): Promise { - return getResponseStream(url).then(streamToString); +export async function fetchUrl(url: string, retries?: number): Promise { + return getResponseStream(url, retries).then(streamToString); } // node-fetch throws a FetchError for those types of errors and diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index 641136b89fb3..13a0f452fe9f 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -41,6 +41,7 @@ import type { ListResult, UpgradePackagePolicyDryRunResponseItem, RegistryDataStream, + InstallablePackage, } from '../../common'; import { PACKAGE_POLICY_SAVED_OBJECT_TYPE } from '../constants'; import { @@ -55,7 +56,6 @@ import type { UpdatePackagePolicy, PackagePolicy, PackagePolicySOAttributes, - RegistryPackage, DryRunPackagePolicy, } from '../types'; import type { ExternalCallback } from '..'; @@ -71,6 +71,7 @@ import { appContextService } from '.'; import { removeOldAssets } from './epm/packages/cleanup'; import type { PackageUpdateEvent, UpdateEventType } from './upgrade_sender'; import { sendTelemetryEvents } from './upgrade_sender'; +import { getArchivePackage } from './epm/archive'; export type InputsOverride = Partial & { vars?: Array; @@ -134,7 +135,8 @@ class PackagePolicyService { pkgVersion: packagePolicy.package.version, }); - let pkgInfo; + let pkgInfo: PackageInfo; + if (options?.skipEnsureInstalled) pkgInfo = await pkgInfoPromise; else { const [, packageInfo] = await Promise.all([ @@ -162,16 +164,21 @@ class PackagePolicyService { } validatePackagePolicyOrThrow(packagePolicy, pkgInfo); - const registryPkgInfo = await Registry.fetchInfo(pkgInfo.name, pkgInfo.version); + let installablePackage: InstallablePackage | undefined = + getArchivePackage(pkgInfo)?.packageInfo; + + if (!installablePackage) { + installablePackage = await Registry.fetchInfo(pkgInfo.name, pkgInfo.version); + } inputs = await this._compilePackagePolicyInputs( - registryPkgInfo, + installablePackage, pkgInfo, packagePolicy.vars || {}, inputs ); - elasticsearch = registryPkgInfo.elasticsearch; + elasticsearch = installablePackage.elasticsearch; } const isoDate = new Date().toISOString(); @@ -400,14 +407,20 @@ class PackagePolicyService { validatePackagePolicyOrThrow(packagePolicy, pkgInfo); - const registryPkgInfo = await Registry.fetchInfo(pkgInfo.name, pkgInfo.version); + let installablePackage: InstallablePackage | undefined = + getArchivePackage(pkgInfo)?.packageInfo; + + if (!installablePackage) { + installablePackage = await Registry.fetchInfo(pkgInfo.name, pkgInfo.version); + } + inputs = await this._compilePackagePolicyInputs( - registryPkgInfo, + installablePackage, pkgInfo, packagePolicy.vars || {}, inputs ); - elasticsearch = registryPkgInfo.elasticsearch; + elasticsearch = installablePackage.elasticsearch; } await soClient.update( @@ -799,14 +812,19 @@ class PackagePolicyService { } public async _compilePackagePolicyInputs( - registryPkgInfo: RegistryPackage, + installablePackage: InstallablePackage, pkgInfo: PackageInfo, vars: PackagePolicy['vars'], inputs: PackagePolicyInput[] ): Promise { const inputsPromises = inputs.map(async (input) => { - const compiledInput = await _compilePackagePolicyInput(registryPkgInfo, pkgInfo, vars, input); - const compiledStreams = await _compilePackageStreams(registryPkgInfo, pkgInfo, vars, input); + const compiledInput = await _compilePackagePolicyInput(pkgInfo, vars, input); + const compiledStreams = await _compilePackageStreams( + installablePackage, + pkgInfo, + vars, + input + ); return { ...input, compiled_input: compiledInput, @@ -917,7 +935,6 @@ function assignStreamIdToInput(packagePolicyId: string, input: NewPackagePolicyI } async function _compilePackagePolicyInput( - registryPkgInfo: RegistryPackage, pkgInfo: PackageInfo, vars: PackagePolicy['vars'], input: PackagePolicyInput @@ -942,7 +959,7 @@ async function _compilePackagePolicyInput( return undefined; } - const [pkgInputTemplate] = await getAssetsData(registryPkgInfo, (path: string) => + const [pkgInputTemplate] = await getAssetsData(pkgInfo, (path: string) => path.endsWith(`/agent/input/${packageInput.template_path!}`) ); @@ -958,13 +975,13 @@ async function _compilePackagePolicyInput( } async function _compilePackageStreams( - registryPkgInfo: RegistryPackage, + installablePackage: InstallablePackage, pkgInfo: PackageInfo, vars: PackagePolicy['vars'], input: PackagePolicyInput ) { const streamsPromises = input.streams.map((stream) => - _compilePackageStream(registryPkgInfo, pkgInfo, vars, input, stream) + _compilePackageStream(pkgInfo, vars, input, stream) ); return await Promise.all(streamsPromises); @@ -1007,7 +1024,6 @@ export function _applyIndexPrivileges( } async function _compilePackageStream( - registryPkgInfo: RegistryPackage, pkgInfo: PackageInfo, vars: PackagePolicy['vars'], input: PackagePolicyInput, @@ -1050,7 +1066,7 @@ async function _compilePackageStream( const datasetPath = packageDataStream.path; const [pkgStreamTemplate] = await getAssetsData( - registryPkgInfo, + pkgInfo, (path: string) => path.endsWith(streamFromPkg.template_path), datasetPath ); diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts index 6d6d641381da..518b79b9e854 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts @@ -33,12 +33,12 @@ import { } from './preconfiguration'; import { outputService } from './output'; import { packagePolicyService } from './package_policy'; -import { getBundledPackages } from './epm/packages/get_bundled_packages'; +import { getBundledPackages } from './epm/packages/bundled_packages'; import type { InstallPackageParams } from './epm/packages/install'; jest.mock('./agent_policy_update'); jest.mock('./output'); -jest.mock('./epm/packages/get_bundled_packages'); +jest.mock('./epm/packages/bundled_packages'); jest.mock('./epm/archive'); const mockedOutputService = outputService as jest.Mocked; @@ -121,7 +121,7 @@ function getPutPreconfiguredPackagesMock() { jest.mock('./epm/registry', () => ({ ...jest.requireActual('./epm/registry'), - async fetchFindLatestPackage(packageName: string): Promise { + async fetchFindLatestPackageOrThrow(packageName: string): Promise { return { name: packageName, version: '1.0.0', @@ -164,12 +164,6 @@ jest.mock('./epm/packages/install', () => ({ // Treat the buffer value passed in tests as the package's name for simplicity const pkgName = archiveBuffer.toString('utf8'); - const installedPackage = mockInstalledPackages.get(pkgName); - - if (installedPackage) { - return installedPackage; - } - // Just install every bundled package at version '1.0.0' const packageInstallation = { name: pkgName, version: '1.0.0', title: pkgName }; mockInstalledPackages.set(pkgName, packageInstallation); @@ -743,11 +737,13 @@ describe('policy preconfiguration', () => { mockedGetBundledPackages.mockResolvedValue([ { name: 'test_package', + version: '1.0.0', buffer: Buffer.from('test_package'), }, { name: 'test_package_2', + version: '1.0.0', buffer: Buffer.from('test_package_2'), }, ]); @@ -784,6 +780,7 @@ describe('policy preconfiguration', () => { mockedGetBundledPackages.mockResolvedValue([ { name: 'test_package', + version: '1.0.0', buffer: Buffer.from('test_package'), }, ]); @@ -823,6 +820,7 @@ describe('policy preconfiguration', () => { mockedGetBundledPackages.mockResolvedValue([ { name: 'test_package', + version: '1.0.0', buffer: Buffer.from('test_package'), }, ]); diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.ts b/x-pack/plugins/fleet/server/services/preconfiguration.ts index e9d97856a926..e9c079d435e7 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.ts @@ -181,9 +181,6 @@ export async function ensurePreconfiguredPackagesAndPolicies( packagesToInstall, force: true, // Always force outdated packages to be installed if a later version isn't installed spaceId, - // During setup, we'll try to install preconfigured packages from the versions bundled with Kibana - // whenever possible - preferredSource: 'bundled', }); const fulfilledPackages = []; diff --git a/x-pack/plugins/fleet/server/types/index.tsx b/x-pack/plugins/fleet/server/types/index.tsx index 9d3e91286478..91303046485d 100644 --- a/x-pack/plugins/fleet/server/types/index.tsx +++ b/x-pack/plugins/fleet/server/types/index.tsx @@ -50,6 +50,7 @@ export type { EsAssetReference, KibanaAssetReference, RegistryPackage, + BundledPackage, InstallablePackage, AssetType, Installable, diff --git a/x-pack/test/fleet_api_integration/apis/package_policy/create.ts b/x-pack/test/fleet_api_integration/apis/package_policy/create.ts index a803b7224d0b..da8efafe8b63 100644 --- a/x-pack/test/fleet_api_integration/apis/package_policy/create.ts +++ b/x-pack/test/fleet_api_integration/apis/package_policy/create.ts @@ -218,7 +218,7 @@ export default function (providerContext: FtrProviderContext) { package: { name: 'endpoint', title: 'Endpoint', - version: '1.3.0-dev.0', + version: '1.4.1', }, }) .expect(200); @@ -236,7 +236,7 @@ export default function (providerContext: FtrProviderContext) { package: { name: 'endpoint', title: 'Endpoint', - version: '1.3.0-dev.0', + version: '1.3.0', }, }) .expect(400); diff --git a/x-pack/test/fleet_api_integration/config.ts b/x-pack/test/fleet_api_integration/config.ts index fb9dc7b6b4ce..28af25c20181 100644 --- a/x-pack/test/fleet_api_integration/config.ts +++ b/x-pack/test/fleet_api_integration/config.ts @@ -15,7 +15,7 @@ import { defineDockerServersConfig } from '@kbn/test'; // example: https://beats-ci.elastic.co/blue/organizations/jenkins/Ingest-manager%2Fpackage-storage/detail/snapshot/74/pipeline/257#step-302-log-1. // It should be updated any time there is a new Docker image published for the Snapshot Distribution of the Package Registry. export const dockerImage = - 'docker.elastic.co/package-registry/distribution@sha256:de952debe048d903fc73e8a4472bb48bb95028d440cba852f21b863d47020c61'; + 'docker.elastic.co/package-registry/distribution@sha256:c5bf8e058727de72e561b228f4b254a14a6f880e582190d01bd5ff74318e1d0b'; export default async function ({ readConfigFile }: FtrConfigProviderContext) { const xPackAPITestsConfig = await readConfigFile(require.resolve('../api_integration/config.ts')); diff --git a/x-pack/test/functional/config.js b/x-pack/test/functional/config.js index 67c2f9b38642..7e1ac8e5481b 100644 --- a/x-pack/test/functional/config.js +++ b/x-pack/test/functional/config.js @@ -15,7 +15,7 @@ import { pageObjects } from './page_objects'; // example: https://beats-ci.elastic.co/blue/organizations/jenkins/Ingest-manager%2Fpackage-storage/detail/snapshot/74/pipeline/257#step-302-log-1. // It should be updated any time there is a new Docker image published for the Snapshot Distribution of the Package Registry. export const dockerImage = - 'docker.elastic.co/package-registry/distribution:ffcbe0ba25b9bae09a671249cbb1b25af0aa1994'; + 'docker.elastic.co/package-registry/distribution@sha256:c5bf8e058727de72e561b228f4b254a14a6f880e582190d01bd5ff74318e1d0b'; // the default export of config files must be a config provider // that returns an object with the projects config values diff --git a/x-pack/test/functional_synthetics/config.js b/x-pack/test/functional_synthetics/config.js index 28cd7e3b099d..f9074812a7e2 100644 --- a/x-pack/test/functional_synthetics/config.js +++ b/x-pack/test/functional_synthetics/config.js @@ -17,7 +17,7 @@ import { pageObjects } from './page_objects'; // example: https://beats-ci.elastic.co/blue/organizations/jenkins/Ingest-manager%2Fpackage-storage/detail/snapshot/74/pipeline/257#step-302-log-1. // It should be updated any time there is a new Docker image published for the Snapshot Distribution of the Package Registry that updates Synthetics. export const dockerImage = - 'docker.elastic.co/package-registry/distribution:48202133e7506873aff3cc7c3b1d284158727779'; + 'docker.elastic.co/package-registry/distribution@sha256:c5bf8e058727de72e561b228f4b254a14a6f880e582190d01bd5ff74318e1d0b'; // the default export of config files must be a config provider // that returns an object with the projects config values diff --git a/x-pack/test/functional_synthetics/services/uptime/synthetics_package.ts b/x-pack/test/functional_synthetics/services/uptime/synthetics_package.ts index b0d935c408e4..898d527245b1 100644 --- a/x-pack/test/functional_synthetics/services/uptime/synthetics_package.ts +++ b/x-pack/test/functional_synthetics/services/uptime/synthetics_package.ts @@ -50,6 +50,7 @@ export function SyntheticsPackageProvider({ getService }: FtrProviderContext) { apiRequest = retry.try(() => { return supertest .get(INGEST_API_EPM_PACKAGES) + .query({ experimental: true }) .set('kbn-xsrf', 'xxx') .expect(200) .catch((error) => {