Skip to content

Commit

Permalink
[Fleet] Allow bundled installs to occur even if EPR is unreachable (#…
Browse files Browse the repository at this point in the history
…125127)

* Allow bundled installs to occur even if EPR is unreachable

* Fix type errors in test

* Fix failing test

* fixup! Fix failing test

* Remove unused object in mock

* Make creation of preconfigured agent policy functional

* Always fall back to bundled packages if available

* Remove unused import

* Use packageInfo object instead of RegistryPackage where possible

* Fix type error in assets test

* Fix test timeouts

* Fix promise logic for registry fetch fallback

* Use archive package as default in create package policy

* Always install from bundled package if it exists - regardless of installation context

* Clean up + refactor a bit

* Default to cached package archive for policy updates

* Update mock in get.test.ts

* Add test for install from bundled package logic

* Delete timeout call in security solution tests

* Fix unused var in endpoint test

* Fix another unused var in endpoint test

* [Debug] Add some logging to test installation times in CI

* Revert "[Debug] Add some logging to test installation times in CI"

This reverts commit 513dcc1.

* Update docker images for registry

* Update docker image digest again

* Refactor latest package fetching to fix broken logic/tests

* Fix a bunch of type errors around renamed fetch latest package version methods

* Remove unused import

* Bump docker version to latest snapshot (again)

* Revert changes to endpoint tests

* Pass experimental flag in synthetics tests

* Fix endpoint version in fleet api integration test

Co-authored-by: Kibana Machine <[email protected]>
  • Loading branch information
kpollich and kibanamachine authored Feb 16, 2022
1 parent 1486741 commit ba8c339
Show file tree
Hide file tree
Showing 28 changed files with 253 additions and 156 deletions.
2 changes: 1 addition & 1 deletion x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import { packageRegistryPort } from './ftr_config';
import { FtrProviderContext } from './ftr_provider_context';

export const dockerImage =
'docker.elastic.co/package-registry/distribution@sha256:de952debe048d903fc73e8a4472bb48bb95028d440cba852f21b863d47020c61';
'docker.elastic.co/package-registry/distribution@sha256:c5bf8e058727de72e561b228f4b254a14a6f880e582190d01bd5ff74318e1d0b';

async function ftrConfigRun({ readConfigFile }: FtrConfigProviderContext) {
const kibanaConfig = await readConfigFile(require.resolve('./ftr_config.ts'));
Expand Down
8 changes: 7 additions & 1 deletion x-pack/plugins/fleet/common/types/models/epm.ts
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,13 @@ export type InstallablePackage = RegistryPackage | ArchivePackage;

export type ArchivePackage = PackageSpecManifest &
// should an uploaded package be able to specify `internal`?
Pick<RegistryPackage, 'readme' | 'assets' | 'data_streams' | 'internal'>;
Pick<RegistryPackage, 'readme' | 'assets' | 'data_streams' | 'internal' | 'elasticsearch'>;

export interface BundledPackage {
name: string;
version: string;
buffer: Buffer;
}

export type RegistryPackage = PackageSpecManifest &
Partial<RegistryOverridesToOptional> &
Expand Down
1 change: 1 addition & 0 deletions x-pack/plugins/fleet/server/errors/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ export class ConcurrentInstallOperationError extends IngestManagerError {}
export class AgentReassignmentError extends IngestManagerError {}
export class PackagePolicyIneligibleForUpgradeError extends IngestManagerError {}
export class PackagePolicyValidationError extends IngestManagerError {}
export class BundledPackageNotFoundError extends IngestManagerError {}
export class HostedAgentPolicyRestrictionRelatedError extends IngestManagerError {
constructor(message = 'Cannot perform that action') {
super(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ export function useDockerRegistry() {

let dockerProcess: ChildProcess | undefined;
async function startDockerRegistryServer() {
const dockerImage = `docker.elastic.co/package-registry/distribution@sha256:de952debe048d903fc73e8a4472bb48bb95028d440cba852f21b863d47020c61`;
const dockerImage = `docker.elastic.co/package-registry/distribution@sha256:c5bf8e058727de72e561b228f4b254a14a6f880e582190d01bd5ff74318e1d0b`;

const args = ['run', '--rm', '-p', `${packageRegistryPort}:8080`, dockerImage];

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ export async function getFullAgentPolicy(
options?: { standalone: boolean }
): Promise<FullAgentPolicy | null> {
let agentPolicy;
const standalone = options?.standalone;
const standalone = options?.standalone ?? false;

try {
agentPolicy = await agentPolicyService.get(soClient, id);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import type {
RegistryElasticsearch,
InstallablePackage,
IndexTemplate,
PackageInfo,
} from '../../../../types';
import { loadFieldsFromYaml, processFields } from '../../fields/field';
import type { Field } from '../../fields/field';
Expand All @@ -31,6 +32,8 @@ import type { ESAssetMetadata } from '../meta';
import { getESAssetMetadata } from '../meta';
import { retryTransientEsErrors } from '../retry';

import { getPackageInfo } from '../../packages';

import {
generateMappings,
generateTemplateName,
Expand Down Expand Up @@ -62,10 +65,16 @@ export const installTemplates = async (
const dataStreams = installablePackage.data_streams;
if (!dataStreams) return [];

const packageInfo = await getPackageInfo({
savedObjectsClient,
pkgName: installablePackage.name,
pkgVersion: installablePackage.version,
});

const installedTemplatesNested = await Promise.all(
dataStreams.map((dataStream) =>
installTemplateForDataStream({
pkg: installablePackage,
pkg: packageInfo,
esClient,
logger,
dataStream,
Expand Down Expand Up @@ -177,7 +186,7 @@ export async function installTemplateForDataStream({
logger,
dataStream,
}: {
pkg: InstallablePackage;
pkg: PackageInfo;
esClient: ElasticsearchClient;
logger: Logger;
dataStream: RegistryDataStream;
Expand Down
4 changes: 2 additions & 2 deletions x-pack/plugins/fleet/server/services/epm/fields/field.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

import { safeLoad } from 'js-yaml';

import type { InstallablePackage } from '../../../types';
import type { PackageInfo } from '../../../types';
import { getAssetsData } from '../packages/assets';

// This should become a copy of https://github.com/elastic/beats/blob/d9a4c9c240a9820fab15002592e5bb6db318543b/libbeat/mapping/field.go#L39
Expand Down Expand Up @@ -261,7 +261,7 @@ const isFields = (path: string) => {
*/

export const loadFieldsFromYaml = async (
pkg: InstallablePackage,
pkg: PackageInfo,
datasetName?: string
): Promise<Field[]> => {
// Fetch all field definition files
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ function getTest(
test = {
method: mocks.packageClient.fetchFindLatestPackage.bind(mocks.packageClient),
args: ['package name'],
spy: jest.spyOn(epmRegistry, 'fetchFindLatestPackage'),
spy: jest.spyOn(epmRegistry, 'fetchFindLatestPackageOrThrow'),
spyArgs: ['package name'],
spyResponse: { name: 'fetchFindLatestPackage test' },
};
Expand Down
8 changes: 4 additions & 4 deletions x-pack/plugins/fleet/server/services/epm/package_service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,13 @@ import type {
InstallablePackage,
Installation,
RegistryPackage,
RegistrySearchResult,
BundledPackage,
} from '../../types';
import { checkSuperuser } from '../../routes/security';
import { FleetUnauthorizedError } from '../../errors';

import { installTransform, isTransform } from './elasticsearch/transform/install';
import { fetchFindLatestPackage, getRegistryPackage } from './registry';
import { fetchFindLatestPackageOrThrow, getRegistryPackage } from './registry';
import { ensureInstalledPackage, getInstallation } from './packages';

export type InstalledAssetType = EsAssetReference;
Expand All @@ -44,7 +44,7 @@ export interface PackageClient {
spaceId?: string;
}): Promise<Installation | undefined>;

fetchFindLatestPackage(packageName: string): Promise<RegistrySearchResult>;
fetchFindLatestPackage(packageName: string): Promise<RegistryPackage | BundledPackage>;

getRegistryPackage(
packageName: string,
Expand Down Expand Up @@ -117,7 +117,7 @@ class PackageClientImpl implements PackageClient {

public async fetchFindLatestPackage(packageName: string) {
await this.#runPreflight();
return fetchFindLatestPackage(packageName);
return fetchFindLatestPackageOrThrow(packageName);
}

public async getRegistryPackage(packageName: string, packageVersion: string) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
* 2.0.
*/

import type { InstallablePackage } from '../../../types';
import type { PackageInfo } from '../../../types';

import { getArchiveFilelist } from '../archive/cache';

Expand Down Expand Up @@ -66,7 +66,7 @@ const tests = [
test('testGetAssets', () => {
for (const value of tests) {
// as needed to pretend it is an InstallablePackage
const assets = getAssets(value.package as InstallablePackage, value.filter, value.dataset);
const assets = getAssets(value.package as PackageInfo, value.filter, value.dataset);
expect(assets).toStrictEqual(value.expected);
}
});
6 changes: 3 additions & 3 deletions x-pack/plugins/fleet/server/services/epm/packages/assets.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
* 2.0.
*/

import type { InstallablePackage } from '../../../types';
import type { PackageInfo } from '../../../types';
import { getArchiveFilelist, getAsset } from '../archive';
import type { ArchiveEntry } from '../archive';

Expand All @@ -17,7 +17,7 @@ import type { ArchiveEntry } from '../archive';
// and different package and version structure

export function getAssets(
packageInfo: InstallablePackage,
packageInfo: PackageInfo,
filter = (path: string): boolean => true,
datasetName?: string
): string[] {
Expand Down Expand Up @@ -52,7 +52,7 @@ export function getAssets(
// ASK: Does getAssetsData need an installSource now?
// if so, should it be an Installation vs InstallablePackage or add another argument?
export async function getAssetsData(
packageInfo: InstallablePackage,
packageInfo: PackageInfo,
filter = (path: string): boolean => true,
datasetName?: string
): Promise<ArchiveEntry[]> {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ import type { InstallResult } from '../../../types';

import { installPackage, isPackageVersionOrLaterInstalled } from './install';
import type { BulkInstallResponse, IBulkInstallPackageError } from './install';
import { getBundledPackages } from './get_bundled_packages';

interface BulkInstallPackagesParams {
savedObjectsClient: SavedObjectsClientContract;
Expand All @@ -31,23 +30,23 @@ export async function bulkInstallPackages({
esClient,
spaceId,
force,
preferredSource = 'registry',
}: BulkInstallPackagesParams): Promise<BulkInstallResponse[]> {
const logger = appContextService.getLogger();

const bundledPackages = await getBundledPackages();

const packagesResults = await Promise.allSettled(
packagesToInstall.map((pkg) => {
if (typeof pkg === 'string') return Registry.fetchFindLatestPackage(pkg);
return Promise.resolve(pkg);
packagesToInstall.map(async (pkg) => {
if (typeof pkg !== 'string') {
return Promise.resolve(pkg);
}

return Registry.fetchFindLatestPackageOrThrow(pkg);
})
);

logger.debug(
`kicking off bulk install of ${packagesToInstall.join(
', '
)} with preferred source of "${preferredSource}"`
`kicking off bulk install of ${packagesToInstall
.map((pkg) => (typeof pkg === 'string' ? pkg : pkg.name))
.join(', ')}`
);

const bulkInstallResults = await Promise.allSettled(
Expand Down Expand Up @@ -83,61 +82,16 @@ export async function bulkInstallPackages({
};
}

let installResult: InstallResult;
const pkgkey = Registry.pkgToPkgKey(pkgKeyProps);

const bundledPackage = bundledPackages.find((pkg) => pkg.name === pkgkey);

// If preferred source is bundled packages on disk, attempt to install from disk first, then fall back to registry
if (preferredSource === 'bundled') {
if (bundledPackage) {
logger.debug(
`kicking off install of ${pkgKeyProps.name}-${pkgKeyProps.version} from bundled package on disk`
);
installResult = await installPackage({
savedObjectsClient,
esClient,
installSource: 'upload',
archiveBuffer: bundledPackage.buffer,
contentType: 'application/zip',
spaceId,
});
} else {
installResult = await installPackage({
savedObjectsClient,
esClient,
pkgkey,
installSource: 'registry',
spaceId,
force,
});
}
} else {
// If preferred source is registry, attempt to install from registry first, then fall back to bundled packages on disk
installResult = await installPackage({
savedObjectsClient,
esClient,
pkgkey,
installSource: 'registry',
spaceId,
force,
});

// If we initially errored, try to install from bundled package on disk
if (installResult.error && bundledPackage) {
logger.debug(
`kicking off install of ${pkgKeyProps.name}-${pkgKeyProps.version} from bundled package on disk`
);
installResult = await installPackage({
savedObjectsClient,
esClient,
installSource: 'upload',
archiveBuffer: bundledPackage.buffer,
contentType: 'application/zip',
spaceId,
});
}
}
const installResult = await installPackage({
savedObjectsClient,
esClient,
pkgkey,
installSource: 'registry',
spaceId,
force,
});

if (installResult.error) {
return {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,18 +5,15 @@
* 2.0.
*/

import path from 'path';
import fs from 'fs/promises';
import path from 'path';

import type { BundledPackage } from '../../../types';
import { appContextService } from '../../app_context';
import { splitPkgKey } from '../registry';

const BUNDLED_PACKAGE_DIRECTORY = path.join(__dirname, '../../../bundled_packages');

interface BundledPackage {
name: string;
buffer: Buffer;
}

export async function getBundledPackages(): Promise<BundledPackage[]> {
try {
const dirContents = await fs.readdir(BUNDLED_PACKAGE_DIRECTORY);
Expand All @@ -26,8 +23,11 @@ export async function getBundledPackages(): Promise<BundledPackage[]> {
zipFiles.map(async (zipFile) => {
const file = await fs.readFile(path.join(BUNDLED_PACKAGE_DIRECTORY, zipFile));

const { pkgName, pkgVersion } = splitPkgKey(zipFile.replace(/\.zip$/, ''));

return {
name: zipFile.replace(/\.zip$/, ''),
name: pkgName,
version: pkgVersion,
buffer: file,
};
})
Expand All @@ -41,3 +41,10 @@ export async function getBundledPackages(): Promise<BundledPackage[]> {
return [];
}
}

export async function getBundledPackageByName(name: string): Promise<BundledPackage | undefined> {
const bundledPackages = await getBundledPackages();
const bundledPackage = bundledPackages.find((pkg) => pkg.name === name);

return bundledPackage;
}
8 changes: 4 additions & 4 deletions x-pack/plugins/fleet/server/services/epm/packages/get.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,7 @@ describe('When using EPM `get` services', () => {
beforeEach(() => {
const mockContract = createAppContextStartContractMock();
appContextService.start(mockContract);
MockRegistry.fetchFindLatestPackage.mockResolvedValue({
MockRegistry.fetchFindLatestPackageOrUndefined.mockResolvedValue({
name: 'my-package',
version: '1.0.0',
} as RegistryPackage);
Expand Down Expand Up @@ -283,8 +283,8 @@ describe('When using EPM `get` services', () => {
});

describe('registry fetch errors', () => {
it('throws when a package that is not installed is not available in the registry', async () => {
MockRegistry.fetchFindLatestPackage.mockResolvedValue(undefined);
it('throws when a package that is not installed is not available in the registry and not bundled', async () => {
MockRegistry.fetchFindLatestPackageOrUndefined.mockResolvedValue(undefined);
const soClient = savedObjectsClientMock.create();
soClient.get.mockRejectedValue(SavedObjectsErrorHelpers.createGenericNotFoundError());

Expand All @@ -298,7 +298,7 @@ describe('When using EPM `get` services', () => {
});

it('sets the latestVersion to installed version when an installed package is not available in the registry', async () => {
MockRegistry.fetchFindLatestPackage.mockResolvedValue(undefined);
MockRegistry.fetchFindLatestPackageOrUndefined.mockResolvedValue(undefined);
const soClient = savedObjectsClientMock.create();
soClient.get.mockResolvedValue({
id: 'my-package',
Expand Down
Loading

0 comments on commit ba8c339

Please sign in to comment.