From 3f213e5bb634adc5688a8ea539578717efe9ef4e Mon Sep 17 00:00:00 2001 From: John Schulz Date: Sat, 12 Dec 2020 06:29:30 -0500 Subject: [PATCH] Use exisiting validation for stored packages --- .../server/services/epm/archive/storage.ts | 99 ++++--------------- .../server/services/epm/archive/validation.ts | 89 ++++++++++------- .../fleet/server/services/epm/packages/get.ts | 12 ++- 3 files changed, 84 insertions(+), 116 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/archive/storage.ts b/x-pack/plugins/fleet/server/services/epm/archive/storage.ts index 974728b1befb7..cfdea90149842 100644 --- a/x-pack/plugins/fleet/server/services/epm/archive/storage.ts +++ b/x-pack/plugins/fleet/server/services/epm/archive/storage.ts @@ -5,26 +5,18 @@ */ import { extname } from 'path'; -import { uniq } from 'lodash'; -import yaml from 'js-yaml'; import { isBinaryFile } from 'isbinaryfile'; import mime from 'mime-types'; import uuidv5 from 'uuid/v5'; -import { - SavedObjectsClientContract, - SavedObjectsBulkCreateObject, - SavedObjectsBulkGetObject, -} from 'src/core/server'; +import { SavedObjectsClientContract, SavedObjectsBulkCreateObject } from 'src/core/server'; import { ASSETS_SAVED_OBJECT_TYPE, InstallablePackage, InstallSource, PackageAssetReference, - RegistryDataStream, } from '../../../../common'; -import { getArchiveEntry } from './index'; -import { parseAndVerifyPolicyTemplates, parseAndVerifyStreams } from './validation'; -import { pkgToPkgKey } from '../registry'; +import { ArchiveEntry, getArchiveEntry, setArchiveEntry } from './index'; +import { preloadManifests, parseAndVerifyArchive } from './validation'; // could be anything, picked this from https://github.com/elastic/elastic-agent-client/issues/17 const MAX_ES_ASSET_BYTES = 4 * 1024 * 1024; @@ -154,76 +146,25 @@ export const getEsPackage = async ( packageAssets: PackageAssetReference[], savedObjectsClient: SavedObjectsClientContract ) => { - const pkgKey = pkgToPkgKey({ name: pkgName, version: pkgVersion }); - const bulkBody: SavedObjectsBulkGetObject[] = packageAssets.map((ref) => ({ - id: ref.id, - type: ref.type, - fields: ['asset_path'], - })); - const soRes = await savedObjectsClient.bulkGet(bulkBody); - const paths = soRes.saved_objects.map((asset) => asset.attributes.asset_path); - - // create the packageInfo - // TODO: this is mostly copied from validtion.ts, but we should save packageInfo somewhere - // so we don't need to do this again as this was already done either in registry or through upload - - const manifestPath = `${pkgName}-${pkgVersion}/manifest.yml`; - const soResManifest = await getAsset({ path: manifestPath, savedObjectsClient }); - if (!soResManifest) throw new Error(`cannot find ${manifestPath}`); - const packageInfo = yaml.load(soResManifest.data_utf8); - - const readmePath = `${pkgName}-${pkgVersion}/docs/README.md`; - const readmeRes = await getAsset({ path: readmePath, savedObjectsClient }); - if (readmeRes) { - packageInfo.readme = `package/${readmePath}`; - } + const bulkRes = await savedObjectsClient.bulkGet(packageAssets); + const entries: ArchiveEntry[] = bulkRes.saved_objects.map((so) => { + const { asset_path: path, data_utf8: utf8, data_base64: base64 } = so.attributes; + const buffer = utf8 ? Buffer.from(utf8, 'utf8') : Buffer.from(base64, 'base64'); + + if (path && buffer) setArchiveEntry(path, buffer); + + return { + path, + buffer, + }; + }); + preloadManifests(entries); + + const paths: string[] = entries.map(({ path }) => path); + const packageInfo = parseAndVerifyArchive(paths); - let dataStreamPaths: string[] = []; - const dataStreams: RegistryDataStream[] = []; - paths - .filter((path) => path.startsWith(`${pkgKey}/data_stream/`)) - .forEach((path) => { - const parts = path.split('/'); - if (parts.length > 2 && parts[2]) dataStreamPaths.push(parts[2]); - }); - - dataStreamPaths = uniq(dataStreamPaths); - - await Promise.all( - dataStreamPaths.map(async (dataStreamPath) => { - const dataStreamManifestPath = `${pkgKey}/data_stream/${dataStreamPath}/manifest.yml`; - const soResDataStreamManifest = await getAsset({ - path: dataStreamManifestPath, - savedObjectsClient, - }); - if (!soResDataStreamManifest) throw new Error(`cannot find ${dataStreamPath}`); - const dataStreamManifest = yaml.load(soResDataStreamManifest.data_utf8); - - const { - title: dataStreamTitle, - release, - ingest_pipeline: ingestPipeline, - type, - dataset, - } = dataStreamManifest; - const streams = parseAndVerifyStreams(dataStreamManifest, dataStreamPath); - - dataStreams.push({ - dataset: dataset || `${pkgName}.${dataStreamPath}`, - title: dataStreamTitle, - release, - package: pkgName, - ingest_pipeline: ingestPipeline || 'default', - path: dataStreamPath, - type, - streams, - }); - }) - ); - packageInfo.policy_templates = parseAndVerifyPolicyTemplates(packageInfo); - packageInfo.data_streams = dataStreams; return { - paths, packageInfo, + paths, }; }; diff --git a/x-pack/plugins/fleet/server/services/epm/archive/validation.ts b/x-pack/plugins/fleet/server/services/epm/archive/validation.ts index 60ba80fb45f9a..4efcb1ca8b330 100644 --- a/x-pack/plugins/fleet/server/services/epm/archive/validation.ts +++ b/x-pack/plugins/fleet/server/services/epm/archive/validation.ts @@ -16,7 +16,7 @@ import { PackageSpecManifest, } from '../../../../common/types'; import { PackageInvalidArchiveError } from '../../../errors'; -import { unpackBufferEntries } from './index'; +import { ArchiveEntry, unpackBufferEntries } from './index'; import { pkgToPkgKey } from '../registry'; const MANIFESTS: Record = {}; @@ -69,19 +69,22 @@ export async function parseAndVerifyArchiveBuffer( contentType: string ): Promise<{ paths: string[]; packageInfo: ArchivePackage }> { const entries = await unpackBufferEntries(archiveBuffer, contentType); - const paths: string[] = []; - entries.forEach(({ path, buffer }) => { - paths.push(path); - if (path.endsWith(MANIFEST_NAME) && buffer) MANIFESTS[path] = buffer; - }); + preloadManifests(entries); + const paths: string[] = entries.map(({ path }) => path); return { packageInfo: parseAndVerifyArchive(paths), paths, }; } -function parseAndVerifyArchive(paths: string[]): ArchivePackage { +export function preloadManifests(entries: ArchiveEntry[]) { + entries.forEach(({ path, buffer }) => { + if (path.endsWith(MANIFEST_NAME) && buffer) MANIFESTS[path] = buffer; + }); +} + +export function parseAndVerifyArchive(paths: string[]): ArchivePackage { // The top-level directory must match pkgName-pkgVersion, and no other top-level files or directories may be present const toplevelDir = paths[0].split('/')[0]; paths.forEach((path) => { @@ -97,29 +100,9 @@ function parseAndVerifyArchive(paths: string[]): ArchivePackage { throw new PackageInvalidArchiveError(`Package must contain a top-level ${MANIFEST_NAME} file.`); } - // ... which must be valid YAML - let manifest: ArchivePackage; - try { - manifest = yaml.load(manifestBuffer.toString()); - } catch (error) { - throw new PackageInvalidArchiveError(`Could not parse top-level package manifest: ${error}.`); - } - - // must have mandatory fields - const reqGiven = pick(manifest, requiredArchivePackageProps); - const requiredKeysMatch = - Object.keys(reqGiven).toString() === requiredArchivePackageProps.toString(); - if (!requiredKeysMatch) { - const list = requiredArchivePackageProps.join(', '); - throw new PackageInvalidArchiveError( - `Invalid top-level package manifest: one or more fields missing of ${list}` - ); - } - - // at least have all required properties - // get optional values and combine into one object for the remaining operations - const optGiven = pick(manifest, optionalArchivePackageProps); - const parsed: ArchivePackage = { ...reqGiven, ...optGiven }; + // load & parse the file + const manifest = loadYamlManifest(manifestBuffer); + const parsed: ArchivePackage = parsePackageManifest(manifest); // Package name and version from the manifest must match those from the toplevel directory const pkgKey = pkgToPkgKey({ name: parsed.name, version: parsed.version }); @@ -129,9 +112,9 @@ function parseAndVerifyArchive(paths: string[]): ArchivePackage { ); } + // Add any properties not present in the manifest parsed.data_streams = parseAndVerifyDataStreams(paths, parsed.name, parsed.version); - parsed.policy_templates = parseAndVerifyPolicyTemplates(manifest); - // add readme if exists + parsed.policy_templates = parseAndVerifyPolicyTemplates(parsed); const readme = parseAndVerifyReadme(paths, parsed.name, parsed.version); if (readme) { parsed.readme = readme; @@ -139,11 +122,48 @@ function parseAndVerifyArchive(paths: string[]): ArchivePackage { return parsed; } + +// at least has all required properties +// include any allowed optional properties +// ignore unknown properties +function parsePackageManifest(manifest: ArchivePackage) { + const reqGiven = manifestHasRequiredFields(manifest); + const optGiven = pick(manifest, optionalArchivePackageProps); + const parsed: ArchivePackage = { ...reqGiven, ...optGiven }; + + return parsed; +} + +function manifestHasRequiredFields(manifest: ArchivePackage) { + const reqGiven = pick(manifest, requiredArchivePackageProps); + const requiredKeysMatch = + Object.keys(reqGiven).toString() === requiredArchivePackageProps.toString(); + if (!requiredKeysMatch) { + const list = requiredArchivePackageProps.join(', '); + throw new PackageInvalidArchiveError( + `Invalid top-level package manifest: one or more fields missing of ${list}` + ); + } + return reqGiven; +} + +// is present & valid YAML +function loadYamlManifest(given: Buffer | string): ArchivePackage { + let manifest; + try { + manifest = yaml.load(given.toString()); + } catch (error) { + throw new PackageInvalidArchiveError(`Could not parse top-level package manifest: ${error}.`); + } + return manifest; +} + function parseAndVerifyReadme(paths: string[], pkgName: string, pkgVersion: string): string | null { const readmeRelPath = `/docs/README.md`; const readmePath = `${pkgName}-${pkgVersion}${readmeRelPath}`; return paths.includes(readmePath) ? `/package/${pkgName}/${pkgVersion}${readmeRelPath}` : null; } + export function parseAndVerifyDataStreams( paths: string[], pkgName: string, @@ -190,9 +210,9 @@ export function parseAndVerifyDataStreams( type, dataset, } = manifest; - if (!(dataStreamTitle && release && type)) { + if (!dataStreamTitle) { throw new PackageInvalidArchiveError( - `Invalid manifest for data stream '${dataStreamPath}': one or more fields missing of 'title', 'release', 'type'` + `Invalid manifest for data stream '${dataStreamPath}': one or more fields missing of 'title'` ); } const streams = parseAndVerifyStreams(manifest, dataStreamPath); @@ -212,6 +232,7 @@ export function parseAndVerifyDataStreams( return dataStreams; } + export function parseAndVerifyStreams(manifest: any, dataStreamPath: string): RegistryStream[] { const streams: RegistryStream[] = []; const manifestStreams = manifest.streams; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get.ts b/x-pack/plugins/fleet/server/services/epm/packages/get.ts index 669492f04a2de..bafecf6902a8d 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get.ts @@ -12,7 +12,7 @@ import { Installation, PackageInfo, KibanaAssetType } from '../../../types'; import * as Registry from '../registry'; import { createInstallableFrom, isRequiredPackage } from './index'; import { getEsPackage } from '../archive/storage'; -import { getArchivePackage } from '../archive'; +import { getArchivePackage, setPackageInfo, setArchiveFilelist } from '../archive'; export { getFile, SearchParams } from '../registry'; @@ -163,10 +163,16 @@ export async function getPackageFromSource(options: { // else package is not installed or installed and missing from cache and storage and installed from registry res = await Registry.getRegistryPackage(pkgName, pkgVersion); } + if (!res) throw new Error(`package info for ${pkgName}-${pkgVersion} does not exist`); + + const { paths, packageInfo } = res; + setArchiveFilelist({ name: pkgName, version: pkgVersion }, paths); + setPackageInfo({ name: pkgName, version: pkgVersion, packageInfo }); + return { - paths: res.paths, - packageInfo: res.packageInfo, + paths, + packageInfo, }; }