Skip to content

Commit

Permalink
Use exisiting validation for stored packages
Browse files Browse the repository at this point in the history
  • Loading branch information
John Schulz committed Dec 12, 2020
1 parent a593dfd commit 3f213e5
Show file tree
Hide file tree
Showing 3 changed files with 84 additions and 116 deletions.
99 changes: 20 additions & 79 deletions x-pack/plugins/fleet/server/services/epm/archive/storage.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,26 +5,18 @@
*/

import { extname } from 'path';
import { uniq } from 'lodash';
import yaml from 'js-yaml';
import { isBinaryFile } from 'isbinaryfile';
import mime from 'mime-types';
import uuidv5 from 'uuid/v5';
import {
SavedObjectsClientContract,
SavedObjectsBulkCreateObject,
SavedObjectsBulkGetObject,
} from 'src/core/server';
import { SavedObjectsClientContract, SavedObjectsBulkCreateObject } from 'src/core/server';
import {
ASSETS_SAVED_OBJECT_TYPE,
InstallablePackage,
InstallSource,
PackageAssetReference,
RegistryDataStream,
} from '../../../../common';
import { getArchiveEntry } from './index';
import { parseAndVerifyPolicyTemplates, parseAndVerifyStreams } from './validation';
import { pkgToPkgKey } from '../registry';
import { ArchiveEntry, getArchiveEntry, setArchiveEntry } from './index';
import { preloadManifests, parseAndVerifyArchive } from './validation';

// could be anything, picked this from https://github.com/elastic/elastic-agent-client/issues/17
const MAX_ES_ASSET_BYTES = 4 * 1024 * 1024;
Expand Down Expand Up @@ -154,76 +146,25 @@ export const getEsPackage = async (
packageAssets: PackageAssetReference[],
savedObjectsClient: SavedObjectsClientContract
) => {
const pkgKey = pkgToPkgKey({ name: pkgName, version: pkgVersion });
const bulkBody: SavedObjectsBulkGetObject[] = packageAssets.map((ref) => ({
id: ref.id,
type: ref.type,
fields: ['asset_path'],
}));
const soRes = await savedObjectsClient.bulkGet<PackageAsset>(bulkBody);
const paths = soRes.saved_objects.map((asset) => asset.attributes.asset_path);

// create the packageInfo
// TODO: this is mostly copied from validtion.ts, but we should save packageInfo somewhere
// so we don't need to do this again as this was already done either in registry or through upload

const manifestPath = `${pkgName}-${pkgVersion}/manifest.yml`;
const soResManifest = await getAsset({ path: manifestPath, savedObjectsClient });
if (!soResManifest) throw new Error(`cannot find ${manifestPath}`);
const packageInfo = yaml.load(soResManifest.data_utf8);

const readmePath = `${pkgName}-${pkgVersion}/docs/README.md`;
const readmeRes = await getAsset({ path: readmePath, savedObjectsClient });
if (readmeRes) {
packageInfo.readme = `package/${readmePath}`;
}
const bulkRes = await savedObjectsClient.bulkGet<PackageAsset>(packageAssets);
const entries: ArchiveEntry[] = bulkRes.saved_objects.map((so) => {
const { asset_path: path, data_utf8: utf8, data_base64: base64 } = so.attributes;
const buffer = utf8 ? Buffer.from(utf8, 'utf8') : Buffer.from(base64, 'base64');

if (path && buffer) setArchiveEntry(path, buffer);

return {
path,
buffer,
};
});
preloadManifests(entries);

const paths: string[] = entries.map(({ path }) => path);
const packageInfo = parseAndVerifyArchive(paths);

let dataStreamPaths: string[] = [];
const dataStreams: RegistryDataStream[] = [];
paths
.filter((path) => path.startsWith(`${pkgKey}/data_stream/`))
.forEach((path) => {
const parts = path.split('/');
if (parts.length > 2 && parts[2]) dataStreamPaths.push(parts[2]);
});

dataStreamPaths = uniq(dataStreamPaths);

await Promise.all(
dataStreamPaths.map(async (dataStreamPath) => {
const dataStreamManifestPath = `${pkgKey}/data_stream/${dataStreamPath}/manifest.yml`;
const soResDataStreamManifest = await getAsset({
path: dataStreamManifestPath,
savedObjectsClient,
});
if (!soResDataStreamManifest) throw new Error(`cannot find ${dataStreamPath}`);
const dataStreamManifest = yaml.load(soResDataStreamManifest.data_utf8);

const {
title: dataStreamTitle,
release,
ingest_pipeline: ingestPipeline,
type,
dataset,
} = dataStreamManifest;
const streams = parseAndVerifyStreams(dataStreamManifest, dataStreamPath);

dataStreams.push({
dataset: dataset || `${pkgName}.${dataStreamPath}`,
title: dataStreamTitle,
release,
package: pkgName,
ingest_pipeline: ingestPipeline || 'default',
path: dataStreamPath,
type,
streams,
});
})
);
packageInfo.policy_templates = parseAndVerifyPolicyTemplates(packageInfo);
packageInfo.data_streams = dataStreams;
return {
paths,
packageInfo,
paths,
};
};
89 changes: 55 additions & 34 deletions x-pack/plugins/fleet/server/services/epm/archive/validation.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import {
PackageSpecManifest,
} from '../../../../common/types';
import { PackageInvalidArchiveError } from '../../../errors';
import { unpackBufferEntries } from './index';
import { ArchiveEntry, unpackBufferEntries } from './index';
import { pkgToPkgKey } from '../registry';

const MANIFESTS: Record<string, Buffer> = {};
Expand Down Expand Up @@ -69,19 +69,22 @@ export async function parseAndVerifyArchiveBuffer(
contentType: string
): Promise<{ paths: string[]; packageInfo: ArchivePackage }> {
const entries = await unpackBufferEntries(archiveBuffer, contentType);
const paths: string[] = [];
entries.forEach(({ path, buffer }) => {
paths.push(path);
if (path.endsWith(MANIFEST_NAME) && buffer) MANIFESTS[path] = buffer;
});
preloadManifests(entries);

const paths: string[] = entries.map(({ path }) => path);
return {
packageInfo: parseAndVerifyArchive(paths),
paths,
};
}

function parseAndVerifyArchive(paths: string[]): ArchivePackage {
export function preloadManifests(entries: ArchiveEntry[]) {
entries.forEach(({ path, buffer }) => {
if (path.endsWith(MANIFEST_NAME) && buffer) MANIFESTS[path] = buffer;
});
}

export function parseAndVerifyArchive(paths: string[]): ArchivePackage {
// The top-level directory must match pkgName-pkgVersion, and no other top-level files or directories may be present
const toplevelDir = paths[0].split('/')[0];
paths.forEach((path) => {
Expand All @@ -97,29 +100,9 @@ function parseAndVerifyArchive(paths: string[]): ArchivePackage {
throw new PackageInvalidArchiveError(`Package must contain a top-level ${MANIFEST_NAME} file.`);
}

// ... which must be valid YAML
let manifest: ArchivePackage;
try {
manifest = yaml.load(manifestBuffer.toString());
} catch (error) {
throw new PackageInvalidArchiveError(`Could not parse top-level package manifest: ${error}.`);
}

// must have mandatory fields
const reqGiven = pick(manifest, requiredArchivePackageProps);
const requiredKeysMatch =
Object.keys(reqGiven).toString() === requiredArchivePackageProps.toString();
if (!requiredKeysMatch) {
const list = requiredArchivePackageProps.join(', ');
throw new PackageInvalidArchiveError(
`Invalid top-level package manifest: one or more fields missing of ${list}`
);
}

// at least have all required properties
// get optional values and combine into one object for the remaining operations
const optGiven = pick(manifest, optionalArchivePackageProps);
const parsed: ArchivePackage = { ...reqGiven, ...optGiven };
// load & parse the file
const manifest = loadYamlManifest(manifestBuffer);
const parsed: ArchivePackage = parsePackageManifest(manifest);

// Package name and version from the manifest must match those from the toplevel directory
const pkgKey = pkgToPkgKey({ name: parsed.name, version: parsed.version });
Expand All @@ -129,21 +112,58 @@ function parseAndVerifyArchive(paths: string[]): ArchivePackage {
);
}

// Add any properties not present in the manifest
parsed.data_streams = parseAndVerifyDataStreams(paths, parsed.name, parsed.version);
parsed.policy_templates = parseAndVerifyPolicyTemplates(manifest);
// add readme if exists
parsed.policy_templates = parseAndVerifyPolicyTemplates(parsed);
const readme = parseAndVerifyReadme(paths, parsed.name, parsed.version);
if (readme) {
parsed.readme = readme;
}

return parsed;
}

// at least has all required properties
// include any allowed optional properties
// ignore unknown properties
function parsePackageManifest(manifest: ArchivePackage) {
const reqGiven = manifestHasRequiredFields(manifest);
const optGiven = pick(manifest, optionalArchivePackageProps);
const parsed: ArchivePackage = { ...reqGiven, ...optGiven };

return parsed;
}

function manifestHasRequiredFields(manifest: ArchivePackage) {
const reqGiven = pick(manifest, requiredArchivePackageProps);
const requiredKeysMatch =
Object.keys(reqGiven).toString() === requiredArchivePackageProps.toString();
if (!requiredKeysMatch) {
const list = requiredArchivePackageProps.join(', ');
throw new PackageInvalidArchiveError(
`Invalid top-level package manifest: one or more fields missing of ${list}`
);
}
return reqGiven;
}

// is present & valid YAML
function loadYamlManifest(given: Buffer | string): ArchivePackage {
let manifest;
try {
manifest = yaml.load(given.toString());
} catch (error) {
throw new PackageInvalidArchiveError(`Could not parse top-level package manifest: ${error}.`);
}
return manifest;
}

function parseAndVerifyReadme(paths: string[], pkgName: string, pkgVersion: string): string | null {
const readmeRelPath = `/docs/README.md`;
const readmePath = `${pkgName}-${pkgVersion}${readmeRelPath}`;
return paths.includes(readmePath) ? `/package/${pkgName}/${pkgVersion}${readmeRelPath}` : null;
}

export function parseAndVerifyDataStreams(
paths: string[],
pkgName: string,
Expand Down Expand Up @@ -190,9 +210,9 @@ export function parseAndVerifyDataStreams(
type,
dataset,
} = manifest;
if (!(dataStreamTitle && release && type)) {
if (!dataStreamTitle) {
throw new PackageInvalidArchiveError(
`Invalid manifest for data stream '${dataStreamPath}': one or more fields missing of 'title', 'release', 'type'`
`Invalid manifest for data stream '${dataStreamPath}': one or more fields missing of 'title'`
);
}
const streams = parseAndVerifyStreams(manifest, dataStreamPath);
Expand All @@ -212,6 +232,7 @@ export function parseAndVerifyDataStreams(

return dataStreams;
}

export function parseAndVerifyStreams(manifest: any, dataStreamPath: string): RegistryStream[] {
const streams: RegistryStream[] = [];
const manifestStreams = manifest.streams;
Expand Down
12 changes: 9 additions & 3 deletions x-pack/plugins/fleet/server/services/epm/packages/get.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import { Installation, PackageInfo, KibanaAssetType } from '../../../types';
import * as Registry from '../registry';
import { createInstallableFrom, isRequiredPackage } from './index';
import { getEsPackage } from '../archive/storage';
import { getArchivePackage } from '../archive';
import { getArchivePackage, setPackageInfo, setArchiveFilelist } from '../archive';

export { getFile, SearchParams } from '../registry';

Expand Down Expand Up @@ -163,10 +163,16 @@ export async function getPackageFromSource(options: {
// else package is not installed or installed and missing from cache and storage and installed from registry
res = await Registry.getRegistryPackage(pkgName, pkgVersion);
}

if (!res) throw new Error(`package info for ${pkgName}-${pkgVersion} does not exist`);

const { paths, packageInfo } = res;
setArchiveFilelist({ name: pkgName, version: pkgVersion }, paths);
setPackageInfo({ name: pkgName, version: pkgVersion, packageInfo });

return {
paths: res.paths,
packageInfo: res.packageInfo,
paths,
packageInfo,
};
}

Expand Down

0 comments on commit 3f213e5

Please sign in to comment.