From a466281c0bd019a1e90b4f0841b9818b19898b96 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Wed, 27 May 2020 23:31:43 -0400 Subject: [PATCH 001/106] stub out task for the exceptions list packager --- x-pack/plugins/security_solution/kibana.json | 1 + .../security_solution/server/plugin.ts | 23 +++++++++++ .../siem/server/lib/exceptions/index.ts | 8 ++++ .../siem/server/lib/exceptions/task.ts | 40 +++++++++++++++++++ .../siem/server/lib/exceptions/task_runner.ts | 35 ++++++++++++++++ 5 files changed, 107 insertions(+) create mode 100644 x-pack/plugins/siem/server/lib/exceptions/index.ts create mode 100644 x-pack/plugins/siem/server/lib/exceptions/task.ts create mode 100644 x-pack/plugins/siem/server/lib/exceptions/task_runner.ts diff --git a/x-pack/plugins/security_solution/kibana.json b/x-pack/plugins/security_solution/kibana.json index 8ce8820a8e57d..f6f2d5171312c 100644 --- a/x-pack/plugins/security_solution/kibana.json +++ b/x-pack/plugins/security_solution/kibana.json @@ -12,6 +12,7 @@ "features", "home", "ingestManager", + "taskManager", "inspector", "licensing", "maps", diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 00f2cba304c88..7242522020bc6 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -24,6 +24,7 @@ import { EncryptedSavedObjectsPluginSetup as EncryptedSavedObjectsSetup } from ' import { SpacesPluginSetup as SpacesSetup } from '../../spaces/server'; import { LicensingPluginSetup } from '../../licensing/server'; import { IngestManagerStartContract } from '../../ingest_manager/server'; +import { TaskManagerSetupContract, TaskManagerStartContract } from '../../task_manager/server'; import { initServer } from './init_server'; import { compose } from './lib/compose/kibana'; import { initRoutes } from './routes'; @@ -32,6 +33,7 @@ import { signalRulesAlertType } from './lib/detection_engine/signals/signal_rule import { rulesNotificationAlertType } from './lib/detection_engine/notifications/rules_notification_alert_type'; import { isNotificationAlertExecutor } from './lib/detection_engine/notifications/types'; import { hasListsFeature, listsEnvFeatureFlagName } from './lib/detection_engine/feature_flags'; +import { ExceptionsPackagerTask, ExceptionsPackagerTaskRunner } from './lib/exceptions'; import { initSavedObjects, savedObjectTypes } from './saved_objects'; import { AppClientFactory } from './client'; import { createConfig$, ConfigType } from './config'; @@ -52,12 +54,14 @@ export interface SetupPlugins { licensing: LicensingPluginSetup; security?: SecuritySetup; spaces?: SpacesSetup; + taskManager: TaskManagerSetupContract; ml?: MlSetup; lists?: ListPluginSetup; } export interface StartPlugins { ingestManager: IngestManagerStartContract; + taskManager: TaskManagerStartContract; } // eslint-disable-next-line @typescript-eslint/no-empty-interface @@ -71,6 +75,8 @@ export class Plugin implements IPlugin { + return { + run: async () => { + await this.run(); + }, + cancel: async () => {}, + }; + }, + }, + }); + } + + private async run() { + this.logger!.debug('HELLO WORLD'); + } +} diff --git a/x-pack/plugins/siem/server/lib/exceptions/task_runner.ts b/x-pack/plugins/siem/server/lib/exceptions/task_runner.ts new file mode 100644 index 0000000000000..d30e581ac97a1 --- /dev/null +++ b/x-pack/plugins/siem/server/lib/exceptions/task_runner.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Logger, LoggerFactory } from '../../../../../../src/core/server'; +import { TaskManagerStartContract } from '../../../../../plugins/task_manager/server'; + +export class ExceptionsPackagerTaskRunner { + private readonly logger?: Logger; + private readonly taskManager?: TaskManagerStartContract; + + constructor(logger: LoggerFactory, taskManager: TaskManagerStartContract) { + this.logger! = logger.get('exceptions_packager_task_runner'); + this.taskManager! = taskManager; + } + + public async schedule() { + try { + await this.taskManager!.ensureScheduled({ + id: 'siem-endpoint:exceptions-packager', + taskType: 'endpoint:exceptions-packager', + scope: ['siem'], + schedule: { + interval: '5s', + }, + state: {}, + params: {}, + }); + } catch (e) { + this.logger!.debug(`Error scheduling task, received ${e.message}`); + } + } +} From ba1f2e7f670bc989602413b900020eae72bf3b46 Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Thu, 28 May 2020 11:03:24 -0400 Subject: [PATCH 002/106] Hits list code and pages --- .../security_solution/server/plugin.ts | 4 +- .../security_solution/server/routes/index.ts | 5 ++ .../exceptions/get_endpoint_exception_list.ts | 83 +++++++++++++++++++ .../get_endpoint_exception_manifest.ts | 59 +++++++++++++ 4 files changed, 149 insertions(+), 2 deletions(-) create mode 100644 x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list.ts create mode 100644 x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_manifest.ts diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 7242522020bc6..f33d37b73d562 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -65,9 +65,9 @@ export interface StartPlugins { } // eslint-disable-next-line @typescript-eslint/no-empty-interface -export interface PluginSetup {} +export interface PluginSetup { } // eslint-disable-next-line @typescript-eslint/no-empty-interface -export interface PluginStart {} +export interface PluginStart { } export class Plugin implements IPlugin { private readonly logger: Logger; diff --git a/x-pack/plugins/security_solution/server/routes/index.ts b/x-pack/plugins/security_solution/server/routes/index.ts index 54d7dcccba815..dbc5b208b4a31 100644 --- a/x-pack/plugins/security_solution/server/routes/index.ts +++ b/x-pack/plugins/security_solution/server/routes/index.ts @@ -28,6 +28,8 @@ import { importRulesRoute } from '../lib/detection_engine/routes/rules/import_ru import { exportRulesRoute } from '../lib/detection_engine/routes/rules/export_rules_route'; import { findRulesStatusesRoute } from '../lib/detection_engine/routes/rules/find_rules_status_route'; import { getPrepackagedRulesStatusRoute } from '../lib/detection_engine/routes/rules/get_prepackaged_rules_status_route'; +import { getEndpointExceptionList } from '../lib/detection_engine/routes/exceptions/get_endpoint_exception_list'; +import { getEndpointExceptionManifest } from '../lib/detection_engine/routes/exceptions/get_endpoint_exception_manifest'; import { importTimelinesRoute } from '../lib/timeline/routes/import_timelines_route'; import { exportTimelinesRoute } from '../lib/timeline/routes/export_timelines_route'; import { createTimelinesRoute } from '../lib/timeline/routes/create_timelines_route'; @@ -65,6 +67,9 @@ export const initRoutes = ( importRulesRoute(router, config, ml); exportRulesRoute(router, config); + getEndpointExceptionList(router); + getEndpointExceptionManifest(router); + importTimelinesRoute(router, config, security); exportTimelinesRoute(router, config); getDraftTimelinesRoute(router, config, security); diff --git a/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list.ts b/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list.ts new file mode 100644 index 0000000000000..e5acb838de25b --- /dev/null +++ b/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list.ts @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IRouter, RequestHandlerContext, APICaller, SavedObjectsClient } from 'kibana/server'; +// eslint-disable-next-line @kbn/eslint/no-restricted-paths +import { findExceptionListItem } from '../../../../../../lists/server/services/exception_lists/find_exception_list_item'; + +const allowlistBaseRoute: string = '/api/endpoint/allowlist'; + +/** + * Registers the exception list route to enable sensors to download a compressed exception list + */ +export function getEndpointExceptionList(router: IRouter) { + router.get( + { + path: `${allowlistBaseRoute}`, + validate: {}, + options: { authRequired: true }, + }, + handleEndpointExceptionDownload + ); +} + +async function getFullEndpointExceptionList(soClient: SavedObjectsClient) { + let exceptions = []; // TODO: type me + let numResponses = 0; + let page = 1; + + do { + console.log(`Fetching page ${page}`); + const response = await findExceptionListItem({ + listId: 'endpoint_list', // TODO + namespaceType: 'single', // ? + savedObjectsClient: soClient, + filter: undefined, + perPage: 100, + page, + sortField: undefined, + sortOrder: undefined, + }); + + if (response?.data !== undefined) { + console.log(`Found ${response.data.length} exceptions`); + numResponses = response.data.length; + exceptions = exceptions.concat(response.data); + page++; + } else { + break; + } + } while (numResponses > 0); + + return exceptions; +} + +/** + * Handles the GET request for downloading the whitelist + */ +async function handleEndpointExceptionDownload(context, req, res) { + try { + // const whitelistHash: string = req.params.hash; + // const bufferHash = createHash('sha256') + // .update(whitelistArtifactCache.toString('utf8'), 'utf8') + // .digest('hex'); + // if (whitelistHash !== bufferHash) { + // return res.badRequest({ + // body: i18n.translate('allowlist.download.fail', { + // defaultMessage: + // 'The requested artifact with hash {whitelistHash} does not match current hash of {bufferHash}', + // values: { whitelistHash, bufferHash }, + // description: 'Allowlist download failure.', + // }), + // }); + // } + const soClient: SavedObjectsClient = context.core.savedObjects.client; + const exceptions = await getFullEndpointExceptionList(soClient); + return res.ok({ body: exceptions }); + } catch (err) { + return res.internalError({ body: err }); + } +} diff --git a/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_manifest.ts b/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_manifest.ts new file mode 100644 index 0000000000000..5ca8d41cc247e --- /dev/null +++ b/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_manifest.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IRouter, RequestHandlerContext, APICaller } from 'kibana/server'; +import { createHash } from 'crypto'; + +const allowlistBaseRoute: string = '/api/endpoint/allowlist'; + +/** + * Registers the exception list route to enable sensors to retrieve a manifest of available lists + */ +export function getEndpointExceptionManifest(router: IRouter) { + router.get( + { + path: '/api/endpoint/manifest', + validate: {}, + options: { authRequired: true }, + }, + handleWhitelistManifest + ); +} + +/** + * Handles the GET request for whitelist manifest + */ +async function handleWhitelistManifest(context, req, res) { + try { + const manifest = await getWhitelistManifest(context); + return res.ok({ body: manifest }); + } catch (err) { + return res.internalError({ body: err }); + } +} + +/** + * Creates the manifest for the whitelist + */ +async function getWhitelistManifest(ctx) { + const hash = createHash('sha256') + .update(whitelistArtifactCache.toString('utf8'), 'utf8') + .digest('hex'); + + const manifest = { + schemaVersion: '1.0.0', + manifestVersion: '1.0.0', + artifacts: { + 'global-whitelist': { + url: `${allowlistBaseRoute}/download/${hash}`, + sha256: hash, + size: whitelistArtifactCache.byteLength, + encoding: 'xz', + }, + }, + }; + return manifest; +} \ No newline at end of file From 9464711271262e02b92a644c524aededdbc0d034 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Thu, 28 May 2020 12:56:08 -0400 Subject: [PATCH 003/106] refactor --- .../security_solution/server/plugin.ts | 23 ++--- .../siem/server/lib/exceptions/index.ts | 1 - .../siem/server/lib/exceptions/task.ts | 92 +++++++++++++------ .../siem/server/lib/exceptions/task_runner.ts | 35 ------- 4 files changed, 71 insertions(+), 80 deletions(-) delete mode 100644 x-pack/plugins/siem/server/lib/exceptions/task_runner.ts diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index f33d37b73d562..4a36dc43ed3e1 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -33,7 +33,7 @@ import { signalRulesAlertType } from './lib/detection_engine/signals/signal_rule import { rulesNotificationAlertType } from './lib/detection_engine/notifications/rules_notification_alert_type'; import { isNotificationAlertExecutor } from './lib/detection_engine/notifications/types'; import { hasListsFeature, listsEnvFeatureFlagName } from './lib/detection_engine/feature_flags'; -import { ExceptionsPackagerTask, ExceptionsPackagerTaskRunner } from './lib/exceptions'; +import { PackagerTask, setupPackagerTask } from './lib/exceptions'; import { initSavedObjects, savedObjectTypes } from './saved_objects'; import { AppClientFactory } from './client'; import { createConfig$, ConfigType } from './config'; @@ -75,8 +75,7 @@ export class Plugin implements IPlugin { - return { - run: async () => { - await this.run(); +import { CoreSetup, Logger } from '../../../../../../src/core/server'; +import { + TaskManagerSetupContract, + TaskManagerStartContract, +} from '../../../../../plugins/task_manager/server'; + +export interface PackagerTask { + getTaskRunner: (taskManager: TaskManagerStartContract) => PackagerTaskRunner; +} + +interface PackagerTaskRunner { + run: () => void; +} + +interface PackagerTaskContext { + core: CoreSetup; + logger: Logger; + taskManager: TaskManagerSetupContract; +} + +export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { + const run = async () => { + context.logger.debug('HELLO WORLD'); + }; + + const getTaskRunner = (taskManager: TaskManagerStartContract) => { + return { + run: async () => { + try { + await taskManager.ensureScheduled({ + id: 'siem:endpoint:exceptions-packager', + taskType: 'siem:endpoint:exceptions-packager', + scope: ['siem'], + schedule: { + interval: '5s', }, - cancel: async () => {}, - }; - }, + state: {}, + params: {}, + }); + } catch (e) { + context.logger.debug(`Error scheduling task, received ${e.message}`); + } + }, + }; + }; + + context.taskManager.registerTaskDefinitions({ + 'siem:endpoint:exceptions-packager': { + title: 'SIEM Endpoint Exceptions Handler', + type: 'siem:endpoint:exceptions-packager', + timeout: '1m', + createTaskRunner: () => { + return { + run: async () => { + await run(); + }, + cancel: async () => {}, + }; }, - }); - } + }, + }); - private async run() { - this.logger!.debug('HELLO WORLD'); - } + return { + getTaskRunner, + }; } diff --git a/x-pack/plugins/siem/server/lib/exceptions/task_runner.ts b/x-pack/plugins/siem/server/lib/exceptions/task_runner.ts deleted file mode 100644 index d30e581ac97a1..0000000000000 --- a/x-pack/plugins/siem/server/lib/exceptions/task_runner.ts +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { Logger, LoggerFactory } from '../../../../../../src/core/server'; -import { TaskManagerStartContract } from '../../../../../plugins/task_manager/server'; - -export class ExceptionsPackagerTaskRunner { - private readonly logger?: Logger; - private readonly taskManager?: TaskManagerStartContract; - - constructor(logger: LoggerFactory, taskManager: TaskManagerStartContract) { - this.logger! = logger.get('exceptions_packager_task_runner'); - this.taskManager! = taskManager; - } - - public async schedule() { - try { - await this.taskManager!.ensureScheduled({ - id: 'siem-endpoint:exceptions-packager', - taskType: 'endpoint:exceptions-packager', - scope: ['siem'], - schedule: { - interval: '5s', - }, - state: {}, - params: {}, - }); - } catch (e) { - this.logger!.debug(`Error scheduling task, received ${e.message}`); - } - } -} From 076fa7a7ec40a9d5ea996af228a698225c7c7e49 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Thu, 28 May 2020 13:56:27 -0400 Subject: [PATCH 004/106] Begin adding saved object and type definitions --- .../security_solution/server/plugin.ts | 16 ++++---- .../security_solution/server/saved_objects.ts | 10 ++++- .../server/lib/exceptions/saved_object.ts | 15 +++++++ .../lib/exceptions/saved_object_mappings.ts | 40 +++++++++++++++++++ .../siem/server/lib/exceptions/types.ts | 19 +++++++++ 5 files changed, 91 insertions(+), 9 deletions(-) create mode 100644 x-pack/plugins/siem/server/lib/exceptions/saved_object.ts create mode 100644 x-pack/plugins/siem/server/lib/exceptions/saved_object_mappings.ts create mode 100644 x-pack/plugins/siem/server/lib/exceptions/types.ts diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 4a36dc43ed3e1..a6a1edb99374a 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -216,13 +216,11 @@ export class Plugin implements IPlugin type.name); diff --git a/x-pack/plugins/siem/server/lib/exceptions/saved_object.ts b/x-pack/plugins/siem/server/lib/exceptions/saved_object.ts new file mode 100644 index 0000000000000..a4e4b1ee79f17 --- /dev/null +++ b/x-pack/plugins/siem/server/lib/exceptions/saved_object.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { FrameworkRequest } from '../framework'; +import { ExceptionsArtifactSavedObject } from '.types'; + +export interface ExceptionsArtifact { + getArtifact: ( + request: FrameworkRequest, + sha256: string + ) => Promise; +} diff --git a/x-pack/plugins/siem/server/lib/exceptions/saved_object_mappings.ts b/x-pack/plugins/siem/server/lib/exceptions/saved_object_mappings.ts new file mode 100644 index 0000000000000..c737b192e23b7 --- /dev/null +++ b/x-pack/plugins/siem/server/lib/exceptions/saved_object_mappings.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SavedObjectsType } from '../../../../../../src/core/server'; + +export const exceptionsArtifactSavedObjectType = 'siem-exceptions-artifact'; + +export const exceptionsArtifactSavedObjectMappings: SavedObjectsType['mappings'] = { + properties: { + // e.g. 'global-whitelist' + name: { + type: 'keyword', + }, + schemaVersion: { + type: 'keyword', + }, + sha256: { + type: 'keyword', + }, + encoding: { + type: 'keyword', + }, + created: { + type: 'date', + }, + body: { + type: 'binary', + }, + }, +}; + +export const type: SavedObjectsType = { + name: exceptionsArtifactSavedObjectType, + hidden: false, // TODO: should these be hidden? + namespaceType: 'agnostic', + mappings: exceptionsArtifactSavedObjectMappings, +}; diff --git a/x-pack/plugins/siem/server/lib/exceptions/types.ts b/x-pack/plugins/siem/server/lib/exceptions/types.ts new file mode 100644 index 0000000000000..5bd29531dbb55 --- /dev/null +++ b/x-pack/plugins/siem/server/lib/exceptions/types.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +type ArtifactName = 'global-whitelist'; +type ArtifactVersion = '1.0.0'; +type ArtifactEncoding = 'xz'; + +// TODO: define runtime types +export interface ExceptionsArtifactSavedObject { + name: ArtifactName; + schemaVersion: ArtifactVersion; + sha256: string; + encoding: ArtifactEncoding; + created: number; + body: string; +} From 1a0e24f0996a5d03e5dac17cbdab1092ed166074 Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Thu, 28 May 2020 14:31:01 -0400 Subject: [PATCH 005/106] Transforms to endpoint exceptions --- .../exceptions/get_endpoint_exception_list.ts | 71 +++++++++++++++++-- .../get_endpoint_exception_manifest.ts | 3 +- 2 files changed, 67 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list.ts b/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list.ts index e5acb838de25b..e744d86761da5 100644 --- a/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list.ts +++ b/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list.ts @@ -4,12 +4,33 @@ * you may not use this file except in compliance with the Elastic License. */ -import { IRouter, RequestHandlerContext, APICaller, SavedObjectsClient } from 'kibana/server'; +import { IRouter, SavedObjectsClient } from 'kibana/server'; +import { FoundExceptionListItemSchema } from '../../../../../../lists/common/schemas/response/found_exception_list_item_schema'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths import { findExceptionListItem } from '../../../../../../lists/server/services/exception_lists/find_exception_list_item'; const allowlistBaseRoute: string = '/api/endpoint/allowlist'; +export interface EndpointExceptionList { + exceptions_list: ExceptionsList[]; +} + +export interface ExceptionsList { + type: string; + entries: EntryElement[]; +} + +export interface EntryElement { + field: string; + operator: string; + entry: EntryEntry; +} + +export interface EntryEntry { + exact_caseless?: string; + exact_caseless_any?: string[]; +} + /** * Registers the exception list route to enable sensors to download a compressed exception list */ @@ -24,13 +45,14 @@ export function getEndpointExceptionList(router: IRouter) { ); } -async function getFullEndpointExceptionList(soClient: SavedObjectsClient) { - let exceptions = []; // TODO: type me +async function getFullEndpointExceptionList( + soClient: SavedObjectsClient +): Promise { + const exceptions: EndpointExceptionList = { exceptions_list: [] }; let numResponses = 0; let page = 1; do { - console.log(`Fetching page ${page}`); const response = await findExceptionListItem({ listId: 'endpoint_list', // TODO namespaceType: 'single', // ? @@ -43,9 +65,12 @@ async function getFullEndpointExceptionList(soClient: SavedObjectsClient) { }); if (response?.data !== undefined) { - console.log(`Found ${response.data.length} exceptions`); numResponses = response.data.length; - exceptions = exceptions.concat(response.data); + + exceptions.exceptions_list = exceptions.exceptions_list.concat( + translateToEndpointExceptions(response) + ); + page++; } else { break; @@ -55,6 +80,40 @@ async function getFullEndpointExceptionList(soClient: SavedObjectsClient) { return exceptions; } +/** + * Translates Exception list items to Exceptions the endpoint can understand + * @param exc + */ +function translateToEndpointExceptions(exc: FoundExceptionListItemSchema): ExceptionsList[] { + const translated: ExceptionsList[] = []; + // Transform to endpoint format + exc.data.forEach((item) => { + const endpointItem: ExceptionsList = { + type: item.type, + entries: [], + }; + item.entries.forEach((entry) => { + // TODO case sensitive? + const e: EntryEntry = {}; + if (entry.match) { + e.exact_caseless = entry.match; + } + + if (entry.match_any) { + e.exact_caseless_any = entry.match_any; + } + + endpointItem.entries.push({ + field: entry.field, + operator: entry.operator, + entry: e, + }); + }); + translated.push(endpointItem); + }); + return translated; +} + /** * Handles the GET request for downloading the whitelist */ diff --git a/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_manifest.ts b/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_manifest.ts index 5ca8d41cc247e..afa367bda2993 100644 --- a/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_manifest.ts +++ b/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_manifest.ts @@ -39,6 +39,7 @@ async function handleWhitelistManifest(context, req, res) { * Creates the manifest for the whitelist */ async function getWhitelistManifest(ctx) { + const whitelistArtifactCache = []; // TODO const hash = createHash('sha256') .update(whitelistArtifactCache.toString('utf8'), 'utf8') .digest('hex'); @@ -56,4 +57,4 @@ async function getWhitelistManifest(ctx) { }, }; return manifest; -} \ No newline at end of file +} From 82a1de148077e4d76f0769e77b0b9698da41e55f Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Thu, 28 May 2020 16:44:11 -0400 Subject: [PATCH 006/106] Get internal SO client --- x-pack/plugins/security_solution/kibana.json | 1 + .../plugins/security_solution/server/plugin.ts | 8 +++++++- .../plugins/siem/server/lib/exceptions/task.ts | 17 +++++++++++++---- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/security_solution/kibana.json b/x-pack/plugins/security_solution/kibana.json index f6f2d5171312c..d79c31ecafa16 100644 --- a/x-pack/plugins/security_solution/kibana.json +++ b/x-pack/plugins/security_solution/kibana.json @@ -13,6 +13,7 @@ "home", "ingestManager", "taskManager", + "lists", "inspector", "licensing", "maps", diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index a6a1edb99374a..52eebfa4e66f8 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -18,6 +18,7 @@ import { import { PluginSetupContract as AlertingSetup } from '../../alerts/server'; import { SecurityPluginSetup as SecuritySetup } from '../../security/server'; import { PluginSetupContract as FeaturesSetup } from '../../features/server'; +import { ListPluginSetup as ListsSetup } from '../../lists/server'; import { MlPluginSetup as MlSetup } from '../../ml/server'; import { ListPluginSetup } from '../../lists/server'; import { EncryptedSavedObjectsPluginSetup as EncryptedSavedObjectsSetup } from '../../encrypted_saved_objects/server'; @@ -220,6 +221,7 @@ export class Plugin implements IPlugin PackagerTaskRunner; + getTaskRunner: (context: PackagerTaskRunnerContext) => PackagerTaskRunner; } interface PackagerTaskRunner { @@ -22,18 +23,26 @@ interface PackagerTaskContext { core: CoreSetup; logger: Logger; taskManager: TaskManagerSetupContract; + lists: ListsSetup; +} + +interface PackagerTaskRunnerContext { + taskManager: TaskManagerStartContract; } export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { const run = async () => { context.logger.debug('HELLO WORLD'); + const [{ savedObjects }] = await context.core.getStartServices(); + const savedObjectsRepository = savedObjects.createInternalRepository(); + context.logger.debug(savedObjectsRepository.get); }; - const getTaskRunner = (taskManager: TaskManagerStartContract) => { + const getTaskRunner = (runnerContext: PackagerTaskRunnerContext) => { return { run: async () => { try { - await taskManager.ensureScheduled({ + await runnerContext.taskManager.ensureScheduled({ id: 'siem:endpoint:exceptions-packager', taskType: 'siem:endpoint:exceptions-packager', scope: ['siem'], From 559076587167e3567d70ab9ddd52604ca2383eca Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Thu, 28 May 2020 16:59:53 -0400 Subject: [PATCH 007/106] update messaging --- x-pack/plugins/siem/server/lib/exceptions/task.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/siem/server/lib/exceptions/task.ts b/x-pack/plugins/siem/server/lib/exceptions/task.ts index 3865549ac8e03..0297fa552c0ae 100644 --- a/x-pack/plugins/siem/server/lib/exceptions/task.ts +++ b/x-pack/plugins/siem/server/lib/exceptions/task.ts @@ -32,10 +32,14 @@ interface PackagerTaskRunnerContext { export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { const run = async () => { - context.logger.debug('HELLO WORLD'); const [{ savedObjects }] = await context.core.getStartServices(); const savedObjectsRepository = savedObjects.createInternalRepository(); - context.logger.debug(savedObjectsRepository.get); + // TODO: add logic here to: + // 1. pull entire exception list + // 2. compile endpoint message for all supported schemaVersions + // 3. compare hashes to the latest hashes that appear in the artifact manifest + // 4. write new artifact record and update manifest, if necessary + // 5. clean up old artifacts, if necessary }; const getTaskRunner = (runnerContext: PackagerTaskRunnerContext) => { From 81d9a8315cc99afa20ee3f46d38f327ba825d247 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Thu, 28 May 2020 17:16:22 -0400 Subject: [PATCH 008/106] cleanup --- x-pack/plugins/security_solution/kibana.json | 1 - x-pack/plugins/security_solution/server/plugin.ts | 14 ++++++++------ x-pack/plugins/siem/server/lib/exceptions/task.ts | 8 +++++--- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/security_solution/kibana.json b/x-pack/plugins/security_solution/kibana.json index d79c31ecafa16..f6f2d5171312c 100644 --- a/x-pack/plugins/security_solution/kibana.json +++ b/x-pack/plugins/security_solution/kibana.json @@ -13,7 +13,6 @@ "home", "ingestManager", "taskManager", - "lists", "inspector", "licensing", "maps", diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 52eebfa4e66f8..f0968f579f53f 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -217,12 +217,14 @@ export class Plugin implements IPlugin PackagerTaskRunner; @@ -23,7 +23,7 @@ interface PackagerTaskContext { core: CoreSetup; logger: Logger; taskManager: TaskManagerSetupContract; - lists: ListsSetup; + lists: ListPluginSetup; } interface PackagerTaskRunnerContext { @@ -32,6 +32,8 @@ interface PackagerTaskRunnerContext { export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { const run = async () => { + context.logger.debug('Running exception list packager task'); + const [{ savedObjects }] = await context.core.getStartServices(); const savedObjectsRepository = savedObjects.createInternalRepository(); // TODO: add logic here to: @@ -51,7 +53,7 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { taskType: 'siem:endpoint:exceptions-packager', scope: ['siem'], schedule: { - interval: '5s', + interval: '5s', // TODO: update this with real interval }, state: {}, params: {}, From 33eea0439ca943396ec665bde7fa29259a39e783 Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Fri, 29 May 2020 12:04:20 -0400 Subject: [PATCH 009/106] Integrating with task manager --- x-pack/plugins/security_solution/package.json | 3 +- .../exceptions/get_endpoint_exception_list.ts | 113 ++---------------- .../exceptions/fetch_endpoint_exceptions.ts | 102 ++++++++++++++++ .../siem/server/lib/exceptions/task.ts | 10 +- 4 files changed, 119 insertions(+), 109 deletions(-) create mode 100644 x-pack/plugins/siem/server/lib/exceptions/fetch_endpoint_exceptions.ts diff --git a/x-pack/plugins/security_solution/package.json b/x-pack/plugins/security_solution/package.json index 73347e00e6b34..312203854bd0f 100644 --- a/x-pack/plugins/security_solution/package.json +++ b/x-pack/plugins/security_solution/package.json @@ -17,8 +17,9 @@ }, "dependencies": { "lodash": "^4.17.15", + "lzma-native": "6.0.0", "querystring": "^0.2.0", "redux-devtools-extension": "^2.13.8", "@types/seedrandom": ">=2.0.0 <4.0.0" } -} +} \ No newline at end of file diff --git a/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list.ts b/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list.ts index e744d86761da5..37921b08fd3ec 100644 --- a/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list.ts +++ b/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list.ts @@ -5,32 +5,12 @@ */ import { IRouter, SavedObjectsClient } from 'kibana/server'; -import { FoundExceptionListItemSchema } from '../../../../../../lists/common/schemas/response/found_exception_list_item_schema'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths -import { findExceptionListItem } from '../../../../../../lists/server/services/exception_lists/find_exception_list_item'; +import { ExceptionListClient } from '../../../../../../lists/server/services/exception_lists/exception_list_client'; +import { GetFullEndpointExceptionList } from '../../../exceptions/fetch_endpoint_exceptions'; const allowlistBaseRoute: string = '/api/endpoint/allowlist'; -export interface EndpointExceptionList { - exceptions_list: ExceptionsList[]; -} - -export interface ExceptionsList { - type: string; - entries: EntryElement[]; -} - -export interface EntryElement { - field: string; - operator: string; - entry: EntryEntry; -} - -export interface EntryEntry { - exact_caseless?: string; - exact_caseless_any?: string[]; -} - /** * Registers the exception list route to enable sensors to download a compressed exception list */ @@ -45,96 +25,17 @@ export function getEndpointExceptionList(router: IRouter) { ); } -async function getFullEndpointExceptionList( - soClient: SavedObjectsClient -): Promise { - const exceptions: EndpointExceptionList = { exceptions_list: [] }; - let numResponses = 0; - let page = 1; - - do { - const response = await findExceptionListItem({ - listId: 'endpoint_list', // TODO - namespaceType: 'single', // ? - savedObjectsClient: soClient, - filter: undefined, - perPage: 100, - page, - sortField: undefined, - sortOrder: undefined, - }); - - if (response?.data !== undefined) { - numResponses = response.data.length; - - exceptions.exceptions_list = exceptions.exceptions_list.concat( - translateToEndpointExceptions(response) - ); - - page++; - } else { - break; - } - } while (numResponses > 0); - - return exceptions; -} - -/** - * Translates Exception list items to Exceptions the endpoint can understand - * @param exc - */ -function translateToEndpointExceptions(exc: FoundExceptionListItemSchema): ExceptionsList[] { - const translated: ExceptionsList[] = []; - // Transform to endpoint format - exc.data.forEach((item) => { - const endpointItem: ExceptionsList = { - type: item.type, - entries: [], - }; - item.entries.forEach((entry) => { - // TODO case sensitive? - const e: EntryEntry = {}; - if (entry.match) { - e.exact_caseless = entry.match; - } - - if (entry.match_any) { - e.exact_caseless_any = entry.match_any; - } - - endpointItem.entries.push({ - field: entry.field, - operator: entry.operator, - entry: e, - }); - }); - translated.push(endpointItem); - }); - return translated; -} - /** * Handles the GET request for downloading the whitelist */ async function handleEndpointExceptionDownload(context, req, res) { try { - // const whitelistHash: string = req.params.hash; - // const bufferHash = createHash('sha256') - // .update(whitelistArtifactCache.toString('utf8'), 'utf8') - // .digest('hex'); - // if (whitelistHash !== bufferHash) { - // return res.badRequest({ - // body: i18n.translate('allowlist.download.fail', { - // defaultMessage: - // 'The requested artifact with hash {whitelistHash} does not match current hash of {bufferHash}', - // values: { whitelistHash, bufferHash }, - // description: 'Allowlist download failure.', - // }), - // }); - // } const soClient: SavedObjectsClient = context.core.savedObjects.client; - const exceptions = await getFullEndpointExceptionList(soClient); + const eClient: ExceptionListClient = new ExceptionListClient({ + savedObjectsClient: soClient, + user: 'kibana', + }); + const exceptions = await GetFullEndpointExceptionList(eClient); return res.ok({ body: exceptions }); } catch (err) { return res.internalError({ body: err }); diff --git a/x-pack/plugins/siem/server/lib/exceptions/fetch_endpoint_exceptions.ts b/x-pack/plugins/siem/server/lib/exceptions/fetch_endpoint_exceptions.ts new file mode 100644 index 0000000000000..8c64e5af03863 --- /dev/null +++ b/x-pack/plugins/siem/server/lib/exceptions/fetch_endpoint_exceptions.ts @@ -0,0 +1,102 @@ +import lzma from 'lzma-native'; +import { FoundExceptionListItemSchema } from '../../../../lists/common/schemas/response/found_exception_list_item_schema'; +// eslint-disable-next-line @kbn/eslint/no-restricted-paths +import { ExceptionListClient } from '../../../../lists/server/services/exception_lists/exception_list_client'; + +export interface EndpointExceptionList { + exceptions_list: ExceptionsList[]; +} + +export interface ExceptionsList { + type: string; + entries: EntryElement[]; +} + +export interface EntryElement { + field: string; + operator: string; + entry: EntryEntry; +} + +export interface EntryEntry { + exact_caseless?: string; + exact_caseless_any?: string[]; +} + + +export async function GetFullEndpointExceptionList( + eClient: ExceptionListClient +): Promise { + const exceptions: EndpointExceptionList = { exceptions_list: [] }; + let numResponses = 0; + let page = 1; + + do { + const response = await eClient.findExceptionListItem({ + listId: 'endpoint_list', // TODO + namespaceType: 'single', // ? + filter: undefined, + perPage: 100, + page, + sortField: undefined, + sortOrder: undefined, + }); + + if (response?.data !== undefined) { + numResponses = response.data.length; + + exceptions.exceptions_list = exceptions.exceptions_list.concat( + translateToEndpointExceptions(response) + ); + + page++; + } else { + break; + } + } while (numResponses > 0); + + return exceptions; +} + +/** + * Translates Exception list items to Exceptions the endpoint can understand + * @param exc + */ +function translateToEndpointExceptions(exc: FoundExceptionListItemSchema): ExceptionsList[] { + const translated: ExceptionsList[] = []; + // Transform to endpoint format + exc.data.forEach((item) => { + const endpointItem: ExceptionsList = { + type: item.type, + entries: [], + }; + item.entries.forEach((entry) => { + // TODO case sensitive? + const e: EntryEntry = {}; + if (entry.match) { + e.exact_caseless = entry.match; + } + + if (entry.match_any) { + e.exact_caseless_any = entry.match_any; + } + + endpointItem.entries.push({ + field: entry.field, + operator: entry.operator, + entry: e, + }); + }); + translated.push(endpointItem); + }); + return translated; +} + +/** + * Compresses the exception list + */ +function compressExceptionList(exceptionList: EndpointExceptionList): Promise { + return lzma.compress(JSON.stringify(exceptionList), (res: Buffer) => { + return res; + }); +} \ No newline at end of file diff --git a/x-pack/plugins/siem/server/lib/exceptions/task.ts b/x-pack/plugins/siem/server/lib/exceptions/task.ts index b8cf231e3824b..bc8c08310c727 100644 --- a/x-pack/plugins/siem/server/lib/exceptions/task.ts +++ b/x-pack/plugins/siem/server/lib/exceptions/task.ts @@ -4,12 +4,13 @@ * you may not use this file except in compliance with the Elastic License. */ -import { CoreSetup, Logger, SavedObjectsRepository } from '../../../../../../src/core/server'; +import { CoreSetup, Logger, SavedObjectsClient } from '../../../../../../src/core/server'; import { TaskManagerSetupContract, TaskManagerStartContract, } from '../../../../../plugins/task_manager/server'; import { ListPluginSetup } from '../../../../lists/server'; +import { GetFullEndpointExceptionList } from './fetch_endpoint_exceptions'; export interface PackagerTask { getTaskRunner: (context: PackagerTaskRunnerContext) => PackagerTaskRunner; @@ -36,6 +37,11 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { const [{ savedObjects }] = await context.core.getStartServices(); const savedObjectsRepository = savedObjects.createInternalRepository(); + const soClient = new SavedObjectsClient(savedObjectsRepository); + const exceptionListClient = context.lists.getExceptionListClient(soClient, 'kibana'); + const exceptions = await GetFullEndpointExceptionList(exceptionListClient); + + await soClient.create('config', exceptions); // TODO is config the right type? // TODO: add logic here to: // 1. pull entire exception list // 2. compile endpoint message for all supported schemaVersions @@ -75,7 +81,7 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { run: async () => { await run(); }, - cancel: async () => {}, + cancel: async () => { }, }; }, }, From e8cef9afaf5e27f32dccb000d590a220cdd48339 Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Fri, 29 May 2020 13:56:41 -0400 Subject: [PATCH 010/106] Integrated with task manager properly --- .../exceptions/get_endpoint_exception_list.ts | 24 +++++++------ .../get_endpoint_exception_manifest.ts | 25 +++++-------- .../exceptions/fetch_endpoint_exceptions.ts | 11 ++++-- .../siem/server/lib/exceptions/task.ts | 35 +++++++++++++++++-- 4 files changed, 61 insertions(+), 34 deletions(-) diff --git a/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list.ts b/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list.ts index 37921b08fd3ec..4e93c98fabecf 100644 --- a/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list.ts +++ b/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list.ts @@ -5,20 +5,17 @@ */ import { IRouter, SavedObjectsClient } from 'kibana/server'; -// eslint-disable-next-line @kbn/eslint/no-restricted-paths -import { ExceptionListClient } from '../../../../../../lists/server/services/exception_lists/exception_list_client'; -import { GetFullEndpointExceptionList } from '../../../exceptions/fetch_endpoint_exceptions'; const allowlistBaseRoute: string = '/api/endpoint/allowlist'; /** - * Registers the exception list route to enable sensors to download a compressed exception list + * Registers the exception list route to enable sensors to download a compressed allowlist */ export function getEndpointExceptionList(router: IRouter) { router.get( { path: `${allowlistBaseRoute}`, - validate: {}, + validate: {}, // TODO: add param for hash options: { authRequired: true }, }, handleEndpointExceptionDownload @@ -26,17 +23,22 @@ export function getEndpointExceptionList(router: IRouter) { } /** - * Handles the GET request for downloading the whitelist + * Handles the GET request for downloading the allowlist */ async function handleEndpointExceptionDownload(context, req, res) { try { const soClient: SavedObjectsClient = context.core.savedObjects.client; - const eClient: ExceptionListClient = new ExceptionListClient({ - savedObjectsClient: soClient, - user: 'kibana', + const sha256Hash = '1825fb19fcc6dc391cae0bc4a2e96dd7f728a0c3ae9e1469251ada67f9e1b975'; // TODO get from req + const resp = await soClient.find({ + type: 'siem-exceptions-artifact', + search: sha256Hash, + searchFields: ['sha256'], }); - const exceptions = await GetFullEndpointExceptionList(eClient); - return res.ok({ body: exceptions }); + if (resp.total > 0) { + return res.ok({ body: resp.saved_objects[0] }); + } else { + return res.notFound({}); + } } catch (err) { return res.internalError({ body: err }); } diff --git a/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_manifest.ts b/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_manifest.ts index afa367bda2993..46c57ee5c9e2f 100644 --- a/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_manifest.ts +++ b/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_manifest.ts @@ -6,6 +6,7 @@ import { IRouter, RequestHandlerContext, APICaller } from 'kibana/server'; import { createHash } from 'crypto'; +import { SavedObjectsClient } from 'kibana/public'; const allowlistBaseRoute: string = '/api/endpoint/allowlist'; @@ -39,22 +40,12 @@ async function handleWhitelistManifest(context, req, res) { * Creates the manifest for the whitelist */ async function getWhitelistManifest(ctx) { - const whitelistArtifactCache = []; // TODO - const hash = createHash('sha256') - .update(whitelistArtifactCache.toString('utf8'), 'utf8') - .digest('hex'); + const soClient: SavedObjectsClient = ctx.core.savedObjects.client; - const manifest = { - schemaVersion: '1.0.0', - manifestVersion: '1.0.0', - artifacts: { - 'global-whitelist': { - url: `${allowlistBaseRoute}/download/${hash}`, - sha256: hash, - size: whitelistArtifactCache.byteLength, - encoding: 'xz', - }, - }, - }; - return manifest; + const manifestResp = soClient.find({ + type: 'siem-exceptions-artifact', + fields: ['name', 'schemaVersion', 'sha256', 'encoding', 'created'], + }); + + return manifestResp; } diff --git a/x-pack/plugins/siem/server/lib/exceptions/fetch_endpoint_exceptions.ts b/x-pack/plugins/siem/server/lib/exceptions/fetch_endpoint_exceptions.ts index 8c64e5af03863..4788874dff522 100644 --- a/x-pack/plugins/siem/server/lib/exceptions/fetch_endpoint_exceptions.ts +++ b/x-pack/plugins/siem/server/lib/exceptions/fetch_endpoint_exceptions.ts @@ -1,3 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + import lzma from 'lzma-native'; import { FoundExceptionListItemSchema } from '../../../../lists/common/schemas/response/found_exception_list_item_schema'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths @@ -23,7 +29,6 @@ export interface EntryEntry { exact_caseless_any?: string[]; } - export async function GetFullEndpointExceptionList( eClient: ExceptionListClient ): Promise { @@ -95,8 +100,8 @@ function translateToEndpointExceptions(exc: FoundExceptionListItemSchema): Excep /** * Compresses the exception list */ -function compressExceptionList(exceptionList: EndpointExceptionList): Promise { +export function CompressExceptionList(exceptionList: EndpointExceptionList): Promise { return lzma.compress(JSON.stringify(exceptionList), (res: Buffer) => { return res; }); -} \ No newline at end of file +} diff --git a/x-pack/plugins/siem/server/lib/exceptions/task.ts b/x-pack/plugins/siem/server/lib/exceptions/task.ts index bc8c08310c727..b44c1ed930401 100644 --- a/x-pack/plugins/siem/server/lib/exceptions/task.ts +++ b/x-pack/plugins/siem/server/lib/exceptions/task.ts @@ -4,13 +4,14 @@ * you may not use this file except in compliance with the Elastic License. */ +import { createHash } from 'crypto'; import { CoreSetup, Logger, SavedObjectsClient } from '../../../../../../src/core/server'; import { TaskManagerSetupContract, TaskManagerStartContract, } from '../../../../../plugins/task_manager/server'; import { ListPluginSetup } from '../../../../lists/server'; -import { GetFullEndpointExceptionList } from './fetch_endpoint_exceptions'; +import { GetFullEndpointExceptionList, CompressExceptionList } from './fetch_endpoint_exceptions'; export interface PackagerTask { getTaskRunner: (context: PackagerTaskRunnerContext) => PackagerTaskRunner; @@ -40,8 +41,36 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { const soClient = new SavedObjectsClient(savedObjectsRepository); const exceptionListClient = context.lists.getExceptionListClient(soClient, 'kibana'); const exceptions = await GetFullEndpointExceptionList(exceptionListClient); + const compressedExceptions: Buffer = await CompressExceptionList(exceptions); + + const sha256Hash = createHash('sha256') + .update(compressedExceptions.toString('utf8'), 'utf8') + .digest('hex'); + + const exceptionSO = { + name: 'global-whitelist', + schemaVersion: '1.0.0', + sha256: sha256Hash, + encoding: 'xz', + created: Date.now(), + body: compressedExceptions.toString(), + }; + + const resp = await soClient.find({ + type: 'siem-exceptions-artifact', + search: sha256Hash, + searchFields: ['sha256'], + fields: [], + }); + + // TODO clean this up and handle errors better + if (resp.total === 0) { + const soResponse = await soClient.create('siem-exceptions-artifact', exceptionSO); + context.logger.debug(JSON.stringify(soResponse)); + } else { + context.logger.debug('No update to Endpoint Exceptions, skipping.'); + } - await soClient.create('config', exceptions); // TODO is config the right type? // TODO: add logic here to: // 1. pull entire exception list // 2. compile endpoint message for all supported schemaVersions @@ -59,7 +88,7 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { taskType: 'siem:endpoint:exceptions-packager', scope: ['siem'], schedule: { - interval: '5s', // TODO: update this with real interval + interval: '30s', // TODO: update this with real interval }, state: {}, params: {}, From b9e478c729d95ca8a6fb2cf11ae4ed022c4af765 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Fri, 29 May 2020 15:20:38 -0400 Subject: [PATCH 011/106] Begin adding schemas --- .../security_solution/server/routes/index.ts | 8 +- .../lib/detection_engine/exceptions/index.ts | 5 + .../lib/detection_engine/exceptions/types.ts | 9 ++ ...ts => download_endpoint_exception_list.ts} | 24 +++-- ...> get_endpoint_exception_list_manifest.ts} | 19 ++-- .../routes/exceptions/validate.ts | 7 ++ .../schemas/download_exception_list_schema.ts | 11 +++ .../exception_list_manifest_schema.ts | 25 +++++ .../server/lib/exceptions/saved_object.ts | 2 +- yarn.lock | 91 ++++++++++++++++++- 10 files changed, 173 insertions(+), 28 deletions(-) create mode 100644 x-pack/plugins/siem/server/lib/detection_engine/exceptions/index.ts create mode 100644 x-pack/plugins/siem/server/lib/detection_engine/exceptions/types.ts rename x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/{get_endpoint_exception_list.ts => download_endpoint_exception_list.ts} (54%) rename x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/{get_endpoint_exception_manifest.ts => get_endpoint_exception_list_manifest.ts} (63%) create mode 100644 x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/validate.ts create mode 100644 x-pack/plugins/siem/server/lib/detection_engine/routes/schemas/download_exception_list_schema.ts create mode 100644 x-pack/plugins/siem/server/lib/detection_engine/routes/schemas/response/exception_list_manifest_schema.ts diff --git a/x-pack/plugins/security_solution/server/routes/index.ts b/x-pack/plugins/security_solution/server/routes/index.ts index dbc5b208b4a31..25f4a7a47783a 100644 --- a/x-pack/plugins/security_solution/server/routes/index.ts +++ b/x-pack/plugins/security_solution/server/routes/index.ts @@ -28,8 +28,8 @@ import { importRulesRoute } from '../lib/detection_engine/routes/rules/import_ru import { exportRulesRoute } from '../lib/detection_engine/routes/rules/export_rules_route'; import { findRulesStatusesRoute } from '../lib/detection_engine/routes/rules/find_rules_status_route'; import { getPrepackagedRulesStatusRoute } from '../lib/detection_engine/routes/rules/get_prepackaged_rules_status_route'; -import { getEndpointExceptionList } from '../lib/detection_engine/routes/exceptions/get_endpoint_exception_list'; -import { getEndpointExceptionManifest } from '../lib/detection_engine/routes/exceptions/get_endpoint_exception_manifest'; +import { downloadEndpointExceptionList } from '../lib/detection_engine/routes/exceptions/download_endpoint_exception_list'; +import { getEndpointExceptionListManifest } from '../lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest'; import { importTimelinesRoute } from '../lib/timeline/routes/import_timelines_route'; import { exportTimelinesRoute } from '../lib/timeline/routes/export_timelines_route'; import { createTimelinesRoute } from '../lib/timeline/routes/create_timelines_route'; @@ -67,8 +67,8 @@ export const initRoutes = ( importRulesRoute(router, config, ml); exportRulesRoute(router, config); - getEndpointExceptionList(router); - getEndpointExceptionManifest(router); + downloadEndpointExceptionList(router); + getEndpointExceptionListManifest(router); importTimelinesRoute(router, config, security); exportTimelinesRoute(router, config); diff --git a/x-pack/plugins/siem/server/lib/detection_engine/exceptions/index.ts b/x-pack/plugins/siem/server/lib/detection_engine/exceptions/index.ts new file mode 100644 index 0000000000000..41bc2aa258807 --- /dev/null +++ b/x-pack/plugins/siem/server/lib/detection_engine/exceptions/index.ts @@ -0,0 +1,5 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ diff --git a/x-pack/plugins/siem/server/lib/detection_engine/exceptions/types.ts b/x-pack/plugins/siem/server/lib/detection_engine/exceptions/types.ts new file mode 100644 index 0000000000000..53c406d2137d3 --- /dev/null +++ b/x-pack/plugins/siem/server/lib/detection_engine/exceptions/types.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export interface DownloadExceptionListRequestParams { + sha256: string; +} diff --git a/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list.ts b/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.ts similarity index 54% rename from x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list.ts rename to x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.ts index 4e93c98fabecf..e541c3009b5ee 100644 --- a/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list.ts +++ b/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.ts @@ -4,18 +4,25 @@ * you may not use this file except in compliance with the Elastic License. */ -import { IRouter, SavedObjectsClient } from 'kibana/server'; +import { IRouter } from '../../../../../../../../src/core/server'; +import { buildRouteValidation } from '../utils'; +import { downloadExceptionListSchema } from '../schemas/download_exception_list_schema'; +import { DownloadExceptionListRequestParams } from '../../exceptions/types'; const allowlistBaseRoute: string = '/api/endpoint/allowlist'; /** * Registers the exception list route to enable sensors to download a compressed allowlist */ -export function getEndpointExceptionList(router: IRouter) { +export function downloadEndpointExceptionList(router: IRouter) { router.get( { - path: `${allowlistBaseRoute}`, - validate: {}, // TODO: add param for hash + path: `${allowlistBaseRoute}/download/{sha256}`, + validate: { + params: buildRouteValidation( + downloadExceptionListSchema + ), + }, options: { authRequired: true }, }, handleEndpointExceptionDownload @@ -27,17 +34,18 @@ export function getEndpointExceptionList(router: IRouter) { */ async function handleEndpointExceptionDownload(context, req, res) { try { - const soClient: SavedObjectsClient = context.core.savedObjects.client; - const sha256Hash = '1825fb19fcc6dc391cae0bc4a2e96dd7f728a0c3ae9e1469251ada67f9e1b975'; // TODO get from req + const soClient = context.core.savedObjects.client; const resp = await soClient.find({ type: 'siem-exceptions-artifact', - search: sha256Hash, + search: req.params.sha256, searchFields: ['sha256'], }); if (resp.total > 0) { return res.ok({ body: resp.saved_objects[0] }); + } else if (resp.total > 1) { + context.logger.warn(`Duplicate allowlist entries found: ${req.params.sha256}`); } else { - return res.notFound({}); + return res.notFound(); } } catch (err) { return res.internalError({ body: err }); diff --git a/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_manifest.ts b/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts similarity index 63% rename from x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_manifest.ts rename to x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts index 46c57ee5c9e2f..0f9981f085268 100644 --- a/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_manifest.ts +++ b/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts @@ -4,32 +4,31 @@ * you may not use this file except in compliance with the Elastic License. */ -import { IRouter, RequestHandlerContext, APICaller } from 'kibana/server'; -import { createHash } from 'crypto'; -import { SavedObjectsClient } from 'kibana/public'; +import { IRouter } from '../../../../../../../../src/core/server'; const allowlistBaseRoute: string = '/api/endpoint/allowlist'; /** * Registers the exception list route to enable sensors to retrieve a manifest of available lists */ -export function getEndpointExceptionManifest(router: IRouter) { +export function getEndpointExceptionListManifest(router: IRouter) { router.get( { - path: '/api/endpoint/manifest', + path: `${allowlistBaseRoute}/manifest`, validate: {}, options: { authRequired: true }, }, - handleWhitelistManifest + handleAllowlistManifest ); } /** * Handles the GET request for whitelist manifest */ -async function handleWhitelistManifest(context, req, res) { +async function handleAllowlistManifest(context, req, res) { try { - const manifest = await getWhitelistManifest(context); + const manifest = await getAllowlistManifest(context); + // TODO: transform and validate response return res.ok({ body: manifest }); } catch (err) { return res.internalError({ body: err }); @@ -39,8 +38,8 @@ async function handleWhitelistManifest(context, req, res) { /** * Creates the manifest for the whitelist */ -async function getWhitelistManifest(ctx) { - const soClient: SavedObjectsClient = ctx.core.savedObjects.client; +async function getAllowlistManifest(ctx) { + const soClient = ctx.core.savedObjects.client; const manifestResp = soClient.find({ type: 'siem-exceptions-artifact', diff --git a/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/validate.ts b/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/validate.ts new file mode 100644 index 0000000000000..654341f5a6511 --- /dev/null +++ b/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/validate.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +// TODO diff --git a/x-pack/plugins/siem/server/lib/detection_engine/routes/schemas/download_exception_list_schema.ts b/x-pack/plugins/siem/server/lib/detection_engine/routes/schemas/download_exception_list_schema.ts new file mode 100644 index 0000000000000..73d6650d0f0f0 --- /dev/null +++ b/x-pack/plugins/siem/server/lib/detection_engine/routes/schemas/download_exception_list_schema.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import Joi from 'joi'; + +export const downloadExceptionListSchema = Joi.object({ + sha256: Joi.string(), +}); diff --git a/x-pack/plugins/siem/server/lib/detection_engine/routes/schemas/response/exception_list_manifest_schema.ts b/x-pack/plugins/siem/server/lib/detection_engine/routes/schemas/response/exception_list_manifest_schema.ts new file mode 100644 index 0000000000000..973f63699fa96 --- /dev/null +++ b/x-pack/plugins/siem/server/lib/detection_engine/routes/schemas/response/exception_list_manifest_schema.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; + +const exceptionListManifestEntrySchema = t.exact( + t.type({ + id: t.string, + name: t.string, + schemaVersion: t.string, + sha256: t.string, + created: t.number, + }) +); + +export const exceptionListManifestSchema = t.exact( + t.type({ + artifacts: t.array(exceptionListManifestEntrySchema), + }) +); + +export type ExceptionListManifestSchema = t.TypeOf; diff --git a/x-pack/plugins/siem/server/lib/exceptions/saved_object.ts b/x-pack/plugins/siem/server/lib/exceptions/saved_object.ts index a4e4b1ee79f17..2151219983f4c 100644 --- a/x-pack/plugins/siem/server/lib/exceptions/saved_object.ts +++ b/x-pack/plugins/siem/server/lib/exceptions/saved_object.ts @@ -5,7 +5,7 @@ */ import { FrameworkRequest } from '../framework'; -import { ExceptionsArtifactSavedObject } from '.types'; +import { ExceptionsArtifactSavedObject } from './types'; export interface ExceptionsArtifact { getArtifact: ( diff --git a/yarn.lock b/yarn.lock index 892fa1b5aa567..faec30177191f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11328,6 +11328,11 @@ detect-indent@^4.0.0: dependencies: repeating "^2.0.0" +detect-libc@^1.0.2: + version "1.0.3" + resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-1.0.3.tgz#fa137c4bd698edf55cd5cd02ac559f91a4c4ba9b" + integrity sha1-+hN8S9aY7fVc1c0CrFWfkaTEups= + detect-newline@2.X, detect-newline@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/detect-newline/-/detect-newline-2.1.0.tgz#f41f1c10be4b00e87b5f13da680759f2c5bfd3e2" @@ -16426,7 +16431,7 @@ icalendar@0.7.1: resolved "https://registry.yarnpkg.com/icalendar/-/icalendar-0.7.1.tgz#d0d3486795f8f1c5cf4f8cafac081b4b4e7a32ae" integrity sha1-0NNIZ5X48cXPT4yvrAgbS056Mq4= -iconv-lite@0.4, iconv-lite@0.4.24, iconv-lite@^0.4.17, iconv-lite@^0.4.22, iconv-lite@^0.4.24, iconv-lite@~0.4.13: +iconv-lite@0.4, iconv-lite@0.4.24, iconv-lite@^0.4.17, iconv-lite@^0.4.22, iconv-lite@^0.4.24, iconv-lite@^0.4.4, iconv-lite@~0.4.13: version "0.4.24" resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b" integrity sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA== @@ -16503,6 +16508,13 @@ ignore-by-default@^1.0.1: resolved "https://registry.yarnpkg.com/ignore-by-default/-/ignore-by-default-1.0.1.tgz#48ca6d72f6c6a3af00a9ad4ae6876be3889e2b09" integrity sha1-SMptcvbGo68Aqa1K5odr44ieKwk= +ignore-walk@^3.0.1: + version "3.0.3" + resolved "https://registry.yarnpkg.com/ignore-walk/-/ignore-walk-3.0.3.tgz#017e2447184bfeade7c238e4aefdd1e8f95b1e37" + integrity sha512-m7o6xuOaT1aqheYHKf8W6J5pYH85ZI9w077erOzLje3JsB1gkafkAhHHY19dqjulgIZHFm32Cp5uNZgcQqdJKw== + dependencies: + minimatch "^3.0.4" + ignore@^3.1.2, ignore@^3.3.5: version "3.3.10" resolved "https://registry.yarnpkg.com/ignore/-/ignore-3.3.10.tgz#0a97fb876986e8081c631160f8f9f389157f0043" @@ -20042,6 +20054,16 @@ lz-string@^1.4.4: resolved "https://registry.yarnpkg.com/lz-string/-/lz-string-1.4.4.tgz#c0d8eaf36059f705796e1e344811cf4c498d3a26" integrity sha1-wNjq82BZ9wV5bh40SBHPTEmNOiY= +lzma-native@6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/lzma-native/-/lzma-native-6.0.0.tgz#e49d57023c81c4e2a1cbe5ebcf223618457ce31d" + integrity sha512-rf5f4opPymsPHotgY2d0cUP3kbVxERSxWDGEbi2gnbnxuWGokFrBaQ02Oe9pssIwsgp0r0PnbSNg7VPY3AYe7w== + dependencies: + node-addon-api "^1.6.0" + node-pre-gyp "^0.11.0" + readable-stream "^2.3.5" + rimraf "^2.7.1" + macos-release@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/macos-release/-/macos-release-1.1.0.tgz#831945e29365b470aa8724b0ab36c8f8959d10fb" @@ -21259,6 +21281,15 @@ nearley@^2.7.10: randexp "0.4.6" semver "^5.4.1" +needle@^2.2.1: + version "2.5.0" + resolved "https://registry.yarnpkg.com/needle/-/needle-2.5.0.tgz#e6fc4b3cc6c25caed7554bd613a5cf0bac8c31c0" + integrity sha512-o/qITSDR0JCyCKEQ1/1bnUXMmznxabbwi/Y4WwJElf+evwJNFNwIDMCCt5IigFVxgeGBJESLohGtIS9gEzo1fA== + dependencies: + debug "^3.2.6" + iconv-lite "^0.4.4" + sax "^1.2.4" + negotiator@0.6.1: version "0.6.1" resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.1.tgz#2b327184e8992101177b28563fb5e7102acd0ca9" @@ -21372,6 +21403,11 @@ nock@12.0.3: lodash "^4.17.13" propagate "^2.0.0" +node-addon-api@^1.6.0: + version "1.7.1" + resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-1.7.1.tgz#cf813cd69bb8d9100f6bdca6755fc268f54ac492" + integrity sha512-2+DuKodWvwRTrCfKOeR24KIc5unKjOh8mz17NCzVnHWfjAdDqbfbjqh7gUT+BkXBRQM52+xCHciKWonJ3CbJMQ== + node-dir@^0.1.10: version "0.1.17" resolved "https://registry.yarnpkg.com/node-dir/-/node-dir-0.1.17.tgz#5f5665d93351335caabef8f1c554516cf5f1e4e5" @@ -21543,6 +21579,22 @@ node-notifier@^5.4.2: shellwords "^0.1.1" which "^1.3.0" +node-pre-gyp@^0.11.0: + version "0.11.0" + resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.11.0.tgz#db1f33215272f692cd38f03238e3e9b47c5dd054" + integrity sha512-TwWAOZb0j7e9eGaf9esRx3ZcLaE5tQ2lvYy1pb5IAaG1a2e2Kv5Lms1Y4hpj+ciXJRofIxxlt5haeQ/2ANeE0Q== + dependencies: + detect-libc "^1.0.2" + mkdirp "^0.5.1" + needle "^2.2.1" + nopt "^4.0.1" + npm-packlist "^1.1.6" + npmlog "^4.0.2" + rc "^1.2.7" + rimraf "^2.6.1" + semver "^5.3.0" + tar "^4" + node-preload@^0.2.1: version "0.2.1" resolved "https://registry.yarnpkg.com/node-preload/-/node-preload-0.2.1.tgz#c03043bb327f417a18fee7ab7ee57b408a144301" @@ -21642,6 +21694,14 @@ nopt@^2.2.0: dependencies: abbrev "1" +nopt@^4.0.1: + version "4.0.3" + resolved "https://registry.yarnpkg.com/nopt/-/nopt-4.0.3.tgz#a375cad9d02fd921278d954c2254d5aa57e15e48" + integrity sha512-CvaGwVMztSMJLOeXPrez7fyfObdZqNUK1cPAEzLHrTybIua9pMdmmPR5YwtfNftIOMv3DPUhFaxsZMNTQO20Kg== + dependencies: + abbrev "1" + osenv "^0.1.4" + nopt@~1.0.10: version "1.0.10" resolved "https://registry.yarnpkg.com/nopt/-/nopt-1.0.10.tgz#6ddd21bd2a31417b92727dd585f8a6f37608ebee" @@ -21722,6 +21782,13 @@ now-and-later@^2.0.0: dependencies: once "^1.3.2" +npm-bundled@^1.0.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/npm-bundled/-/npm-bundled-1.1.1.tgz#1edd570865a94cdb1bc8220775e29466c9fb234b" + integrity sha512-gqkfgGePhTpAEgUsGEgcq1rqPXA+tv/aVBlgEzfXwA1yiUJF7xtEt3CtVwOjNYQOVknDk0F20w58Fnm3EtG0fA== + dependencies: + npm-normalize-package-bin "^1.0.1" + npm-conf@^1.1.0, npm-conf@^1.1.3: version "1.1.3" resolved "https://registry.yarnpkg.com/npm-conf/-/npm-conf-1.1.3.tgz#256cc47bd0e218c259c4e9550bf413bc2192aff9" @@ -21738,6 +21805,20 @@ npm-keyword@^5.0.0: got "^7.1.0" registry-url "^3.0.3" +npm-normalize-package-bin@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/npm-normalize-package-bin/-/npm-normalize-package-bin-1.0.1.tgz#6e79a41f23fd235c0623218228da7d9c23b8f6e2" + integrity sha512-EPfafl6JL5/rU+ot6P3gRSCpPDW5VmIzX959Ob1+ySFUuuYHWHekXpwdUZcKP5C+DS4GEtdJluwBjnsNDl+fSA== + +npm-packlist@^1.1.6: + version "1.4.8" + resolved "https://registry.yarnpkg.com/npm-packlist/-/npm-packlist-1.4.8.tgz#56ee6cc135b9f98ad3d51c1c95da22bbb9b2ef3e" + integrity sha512-5+AZgwru5IevF5ZdnFglB5wNlHG1AOOuw28WhUq8/8emhBmLv6jX5by4WJCh7lW0uSYZYS6DXqIsyZVIXRZU9A== + dependencies: + ignore-walk "^3.0.1" + npm-bundled "^1.0.1" + npm-normalize-package-bin "^1.0.1" + npm-run-path@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/npm-run-path/-/npm-run-path-1.0.0.tgz#f5c32bf595fe81ae927daec52e82f8b000ac3c8f" @@ -21775,7 +21856,7 @@ npmconf@^2.1.3: semver "2 || 3 || 4" uid-number "0.0.5" -"npmlog@0 || 1 || 2 || 3 || 4", npmlog@^4.0.0, npmlog@^4.1.2: +"npmlog@0 || 1 || 2 || 3 || 4", npmlog@^4.0.0, npmlog@^4.0.2, npmlog@^4.1.2: version "4.1.2" resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-4.1.2.tgz#08a7f2a8bf734604779a9efa4ad5cc717abb954b" integrity sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg== @@ -22356,7 +22437,7 @@ os-tmpdir@^1.0.0, os-tmpdir@~1.0.1, os-tmpdir@~1.0.2: resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274" integrity sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ= -osenv@0, osenv@^0.1.0: +osenv@0, osenv@^0.1.0, osenv@^0.1.4: version "0.1.5" resolved "https://registry.yarnpkg.com/osenv/-/osenv-0.1.5.tgz#85cdfafaeb28e8677f416e287592b5f3f49ea410" integrity sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g== @@ -26630,7 +26711,7 @@ sass-resources-loader@^2.0.1: glob "^7.1.1" loader-utils "^1.0.4" -sax@>=0.6.0, sax@^1.2.1, sax@~1.2.4: +sax@>=0.6.0, sax@^1.2.1, sax@^1.2.4, sax@~1.2.4: version "1.2.4" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw== @@ -28675,7 +28756,7 @@ tar-stream@^2.1.0: inherits "^2.0.3" readable-stream "^3.1.1" -tar@4.4.13: +tar@4.4.13, tar@^4: version "4.4.13" resolved "https://registry.yarnpkg.com/tar/-/tar-4.4.13.tgz#43b364bc52888d555298637b10d60790254ab525" integrity sha512-w2VwSrBoHa5BsSyH+KxEqeQBAllHhccyMFVHtGtdMpF4W7IRWfZjFiQceJPChOeTsSDVUpER2T8FA93pr0L+QA== From f2e6b16880a61f965e33ee68b4c305d6517cd0b8 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Wed, 3 Jun 2020 13:04:13 -0400 Subject: [PATCH 012/106] Add multiple OS and schema version support --- .../download_endpoint_exception_list.ts | 7 +- .../get_endpoint_exception_list_manifest.ts | 32 ++++-- .../exceptions/fetch_endpoint_exceptions.ts | 5 +- .../lib/exceptions/saved_object_mappings.ts | 5 +- .../siem/server/lib/exceptions/task.ts | 97 +++++++++++-------- 5 files changed, 93 insertions(+), 53 deletions(-) diff --git a/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.ts b/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.ts index e541c3009b5ee..750142b7627b9 100644 --- a/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.ts +++ b/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.ts @@ -41,8 +41,11 @@ async function handleEndpointExceptionDownload(context, req, res) { searchFields: ['sha256'], }); if (resp.total > 0) { - return res.ok({ body: resp.saved_objects[0] }); - } else if (resp.total > 1) { + return res.ok({ + body: resp.saved_objects[0].body, + headers: { 'content-encoding': 'xz' }, + }); + } else if (res.total > 1) { context.logger.warn(`Duplicate allowlist entries found: ${req.params.sha256}`); } else { return res.notFound(); diff --git a/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts b/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts index 0f9981f085268..f22218c15c1a7 100644 --- a/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts +++ b/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts @@ -14,7 +14,7 @@ const allowlistBaseRoute: string = '/api/endpoint/allowlist'; export function getEndpointExceptionListManifest(router: IRouter) { router.get( { - path: `${allowlistBaseRoute}/manifest`, + path: `${allowlistBaseRoute}/manifest/{schemaVersion}`, validate: {}, options: { authRequired: true }, }, @@ -27,9 +27,23 @@ export function getEndpointExceptionListManifest(router: IRouter) { */ async function handleAllowlistManifest(context, req, res) { try { - const manifest = await getAllowlistManifest(context); - // TODO: transform and validate response - return res.ok({ body: manifest }); + const resp = await getAllowlistManifest(context, req.params.schemaVersion); + const manifestResp = {}; + + // transform and validate response + for (const manifest of resp.saved_objects) { + if (!manifestResp[manifest.name]) { + manifestResp[manifest.name] = { + [manifest.name]: { + url: `${allowlistBaseRoute}/download/${manifest.sha256}`, + sha256: manifest.sha256, + size: manifest.size, + }, + }; + } + } + + return res.ok({ body: manifestResp }); } catch (err) { return res.internalError({ body: err }); } @@ -38,12 +52,16 @@ async function handleAllowlistManifest(context, req, res) { /** * Creates the manifest for the whitelist */ -async function getAllowlistManifest(ctx) { +async function getAllowlistManifest(ctx, schemaVersion) { const soClient = ctx.core.savedObjects.client; const manifestResp = soClient.find({ - type: 'siem-exceptions-artifact', - fields: ['name', 'schemaVersion', 'sha256', 'encoding', 'created'], + type: 'siem-exceptions-artifact', // TODO: use exported const + fields: ['name', 'schemaVersion', 'sha256', 'encoding', 'size', 'created'], + search: schemaVersion, + searchFields: ['schemaVersion'], + sortField: 'updated_at', + sortOrder: 'desc', }); return manifestResp; diff --git a/x-pack/plugins/siem/server/lib/exceptions/fetch_endpoint_exceptions.ts b/x-pack/plugins/siem/server/lib/exceptions/fetch_endpoint_exceptions.ts index 4788874dff522..c9248a864f5e7 100644 --- a/x-pack/plugins/siem/server/lib/exceptions/fetch_endpoint_exceptions.ts +++ b/x-pack/plugins/siem/server/lib/exceptions/fetch_endpoint_exceptions.ts @@ -30,7 +30,8 @@ export interface EntryEntry { } export async function GetFullEndpointExceptionList( - eClient: ExceptionListClient + eClient: ExceptionListClient, + os: string // TODO: make type ): Promise { const exceptions: EndpointExceptionList = { exceptions_list: [] }; let numResponses = 0; @@ -40,7 +41,7 @@ export async function GetFullEndpointExceptionList( const response = await eClient.findExceptionListItem({ listId: 'endpoint_list', // TODO namespaceType: 'single', // ? - filter: undefined, + filter: `exception-list-item.attributes.sensor_os:${os}`, perPage: 100, page, sortField: undefined, diff --git a/x-pack/plugins/siem/server/lib/exceptions/saved_object_mappings.ts b/x-pack/plugins/siem/server/lib/exceptions/saved_object_mappings.ts index c737b192e23b7..24e701321bc64 100644 --- a/x-pack/plugins/siem/server/lib/exceptions/saved_object_mappings.ts +++ b/x-pack/plugins/siem/server/lib/exceptions/saved_object_mappings.ts @@ -10,7 +10,7 @@ export const exceptionsArtifactSavedObjectType = 'siem-exceptions-artifact'; export const exceptionsArtifactSavedObjectMappings: SavedObjectsType['mappings'] = { properties: { - // e.g. 'global-whitelist' + // e.g. 'global-whitelist-windows' name: { type: 'keyword', }, @@ -29,6 +29,9 @@ export const exceptionsArtifactSavedObjectMappings: SavedObjectsType['mappings'] body: { type: 'binary', }, + size: { + type: 'long', + }, }, }; diff --git a/x-pack/plugins/siem/server/lib/exceptions/task.ts b/x-pack/plugins/siem/server/lib/exceptions/task.ts index b44c1ed930401..7fb6d63b5345b 100644 --- a/x-pack/plugins/siem/server/lib/exceptions/task.ts +++ b/x-pack/plugins/siem/server/lib/exceptions/task.ts @@ -13,6 +13,19 @@ import { import { ListPluginSetup } from '../../../../lists/server'; import { GetFullEndpointExceptionList, CompressExceptionList } from './fetch_endpoint_exceptions'; +const PackagerTaskConstants = { + INTERVAL: '30s', + TIMEOUT: '1m', + TYPE: 'siem:endpoint:exceptions-packager', +}; + +export const ArtifactConstants = { + GLOBAL_ALLOWLIST_NAME: 'global-allowlist', + SAVED_OBJECT_TYPE: 'siem-exceptions-artifact', + SUPPORTED_OPERATING_SYSTEMS: ['windows'], + SUPPORTED_SCHEMA_VERSIONS: ['1.0.0'], +}; + export interface PackagerTask { getTaskRunner: (context: PackagerTaskRunnerContext) => PackagerTaskRunner; } @@ -40,43 +53,45 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { const savedObjectsRepository = savedObjects.createInternalRepository(); const soClient = new SavedObjectsClient(savedObjectsRepository); const exceptionListClient = context.lists.getExceptionListClient(soClient, 'kibana'); - const exceptions = await GetFullEndpointExceptionList(exceptionListClient); - const compressedExceptions: Buffer = await CompressExceptionList(exceptions); - - const sha256Hash = createHash('sha256') - .update(compressedExceptions.toString('utf8'), 'utf8') - .digest('hex'); - - const exceptionSO = { - name: 'global-whitelist', - schemaVersion: '1.0.0', - sha256: sha256Hash, - encoding: 'xz', - created: Date.now(), - body: compressedExceptions.toString(), - }; - const resp = await soClient.find({ - type: 'siem-exceptions-artifact', - search: sha256Hash, - searchFields: ['sha256'], - fields: [], - }); - - // TODO clean this up and handle errors better - if (resp.total === 0) { - const soResponse = await soClient.create('siem-exceptions-artifact', exceptionSO); - context.logger.debug(JSON.stringify(soResponse)); - } else { - context.logger.debug('No update to Endpoint Exceptions, skipping.'); - } + for (const os of ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS) { + const exceptions = await GetFullEndpointExceptionList(exceptionListClient, os); + const compressedExceptions: Buffer = await CompressExceptionList(exceptions); + + const sha256Hash = createHash('sha256') + .update(compressedExceptions.toString('utf8'), 'utf8') + .digest('hex'); - // TODO: add logic here to: - // 1. pull entire exception list - // 2. compile endpoint message for all supported schemaVersions - // 3. compare hashes to the latest hashes that appear in the artifact manifest - // 4. write new artifact record and update manifest, if necessary - // 5. clean up old artifacts, if necessary + for (const schemaVersion of ArtifactConstants.SUPPORTED_SCHEMA_VERSIONS) { + const exceptionSO = { + name: `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-${os}`, + schemaVersion, + sha256: sha256Hash, + encoding: 'xz', + created: Date.now(), + body: compressedExceptions.toString(), + size: Buffer.from(exceptions).byteLength, + }; + + const resp = await soClient.find({ + type: ArtifactConstants.SAVED_OBJECT_TYPE, + search: sha256Hash, + searchFields: ['sha256'], + fields: [], + }); + + // TODO clean this up and handle errors better + if (resp.total === 0) { + const soResponse = await soClient.create( + ArtifactConstants.SAVED_OBJECT_TYPE, + exceptionSO + ); + context.logger.debug(JSON.stringify(soResponse)); + } else { + context.logger.debug('No update to Endpoint Exceptions, skipping.'); + } + } + } }; const getTaskRunner = (runnerContext: PackagerTaskRunnerContext) => { @@ -84,11 +99,11 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { run: async () => { try { await runnerContext.taskManager.ensureScheduled({ - id: 'siem:endpoint:exceptions-packager', - taskType: 'siem:endpoint:exceptions-packager', + id: PackagerTaskConstants.TYPE, + taskType: PackagerTaskConstants.TYPE, scope: ['siem'], schedule: { - interval: '30s', // TODO: update this with real interval + interval: PackagerTaskConstants.INTERVAL, }, state: {}, params: {}, @@ -103,14 +118,14 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { context.taskManager.registerTaskDefinitions({ 'siem:endpoint:exceptions-packager': { title: 'SIEM Endpoint Exceptions Handler', - type: 'siem:endpoint:exceptions-packager', - timeout: '1m', + type: PackagerTaskConstants.TYPE, + timeout: PackagerTaskConstants.TIMEOUT, createTaskRunner: () => { return { run: async () => { await run(); }, - cancel: async () => { }, + cancel: async () => {}, }; }, }, From 184143e92bb2a69402aaa75fc75137a7925de0c8 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Wed, 3 Jun 2020 14:39:56 -0400 Subject: [PATCH 013/106] filter by OS --- .../exceptions/download_endpoint_exception_list.ts | 5 +++-- .../exceptions/get_endpoint_exception_list_manifest.ts | 5 +++-- .../server/lib/exceptions/fetch_endpoint_exceptions.ts | 10 +++++----- x-pack/plugins/siem/server/lib/exceptions/task.ts | 4 ++-- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.ts b/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.ts index 750142b7627b9..12658e936e64a 100644 --- a/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.ts +++ b/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.ts @@ -5,9 +5,10 @@ */ import { IRouter } from '../../../../../../../../src/core/server'; +import { ArtifactConstants } from '../../../exceptions'; +import { DownloadExceptionListRequestParams } from '../../exceptions/types'; import { buildRouteValidation } from '../utils'; import { downloadExceptionListSchema } from '../schemas/download_exception_list_schema'; -import { DownloadExceptionListRequestParams } from '../../exceptions/types'; const allowlistBaseRoute: string = '/api/endpoint/allowlist'; @@ -36,7 +37,7 @@ async function handleEndpointExceptionDownload(context, req, res) { try { const soClient = context.core.savedObjects.client; const resp = await soClient.find({ - type: 'siem-exceptions-artifact', + type: ArtifactConstants.SAVED_OBJECT_TYPE, search: req.params.sha256, searchFields: ['sha256'], }); diff --git a/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts b/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts index f22218c15c1a7..70ad7f4defe02 100644 --- a/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts +++ b/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts @@ -5,6 +5,7 @@ */ import { IRouter } from '../../../../../../../../src/core/server'; +import { ArtifactConstants } from '../../../exceptions'; const allowlistBaseRoute: string = '/api/endpoint/allowlist'; @@ -56,11 +57,11 @@ async function getAllowlistManifest(ctx, schemaVersion) { const soClient = ctx.core.savedObjects.client; const manifestResp = soClient.find({ - type: 'siem-exceptions-artifact', // TODO: use exported const + type: ArtifactConstants.SAVED_OBJECT_TYPE, fields: ['name', 'schemaVersion', 'sha256', 'encoding', 'size', 'created'], search: schemaVersion, searchFields: ['schemaVersion'], - sortField: 'updated_at', + sortField: 'created_at', sortOrder: 'desc', }); diff --git a/x-pack/plugins/siem/server/lib/exceptions/fetch_endpoint_exceptions.ts b/x-pack/plugins/siem/server/lib/exceptions/fetch_endpoint_exceptions.ts index c9248a864f5e7..c7100f722e233 100644 --- a/x-pack/plugins/siem/server/lib/exceptions/fetch_endpoint_exceptions.ts +++ b/x-pack/plugins/siem/server/lib/exceptions/fetch_endpoint_exceptions.ts @@ -39,13 +39,13 @@ export async function GetFullEndpointExceptionList( do { const response = await eClient.findExceptionListItem({ - listId: 'endpoint_list', // TODO - namespaceType: 'single', // ? - filter: `exception-list-item.attributes.sensor_os:${os}`, + listId: 'endpoint_list', + namespaceType: 'agnostic', + filter: `_tags:"sensor:${os}"`, perPage: 100, page, - sortField: undefined, - sortOrder: undefined, + sortField: 'created_at', + sortOrder: 'desc', }); if (response?.data !== undefined) { diff --git a/x-pack/plugins/siem/server/lib/exceptions/task.ts b/x-pack/plugins/siem/server/lib/exceptions/task.ts index 7fb6d63b5345b..3c1ddc47d2a2f 100644 --- a/x-pack/plugins/siem/server/lib/exceptions/task.ts +++ b/x-pack/plugins/siem/server/lib/exceptions/task.ts @@ -20,7 +20,7 @@ const PackagerTaskConstants = { }; export const ArtifactConstants = { - GLOBAL_ALLOWLIST_NAME: 'global-allowlist', + GLOBAL_ALLOWLIST_NAME: 'endpoint-allowlist', SAVED_OBJECT_TYPE: 'siem-exceptions-artifact', SUPPORTED_OPERATING_SYSTEMS: ['windows'], SUPPORTED_SCHEMA_VERSIONS: ['1.0.0'], @@ -70,7 +70,7 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { encoding: 'xz', created: Date.now(), body: compressedExceptions.toString(), - size: Buffer.from(exceptions).byteLength, + size: Buffer.from(JSON.stringify(exceptions)).byteLength, }; const resp = await soClient.find({ From ac70535d79b746ea6fa233e7943470d19d614d0c Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Wed, 3 Jun 2020 17:31:46 -0400 Subject: [PATCH 014/106] Fixing sort --- .../get_endpoint_exception_list_manifest.ts | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts b/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts index 70ad7f4defe02..776c3c705c251 100644 --- a/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts +++ b/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts @@ -33,15 +33,13 @@ async function handleAllowlistManifest(context, req, res) { // transform and validate response for (const manifest of resp.saved_objects) { - if (!manifestResp[manifest.name]) { - manifestResp[manifest.name] = { - [manifest.name]: { - url: `${allowlistBaseRoute}/download/${manifest.sha256}`, - sha256: manifest.sha256, - size: manifest.size, - }, - }; - } + manifestResp[manifest.attributes.name] = { + [manifest.attributes.name]: { + url: `${allowlistBaseRoute}/download/${manifest.attributes.sha256}`, + sha256: manifest.attributes.sha256, + size: manifest.attributes.size, // TODO add size + }, + }; } return res.ok({ body: manifestResp }); @@ -61,8 +59,8 @@ async function getAllowlistManifest(ctx, schemaVersion) { fields: ['name', 'schemaVersion', 'sha256', 'encoding', 'size', 'created'], search: schemaVersion, searchFields: ['schemaVersion'], - sortField: 'created_at', - sortOrder: 'desc', + sortField: 'created', + sortOrder: 'asc', }); return manifestResp; From 11e56d0729b2db1c4b7dbb2785d2e1276f7a0ba0 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Thu, 4 Jun 2020 11:52:55 -0400 Subject: [PATCH 015/106] Move to security_solutions --- .../server/lib/detection_engine/exceptions/index.ts | 0 .../server/lib/detection_engine/exceptions/types.ts | 0 .../routes/exceptions/download_endpoint_exception_list.ts | 0 .../routes/exceptions/get_endpoint_exception_list_manifest.ts | 0 .../server/lib/detection_engine/routes/exceptions/validate.ts | 0 .../routes/schemas/download_exception_list_schema.ts | 0 .../routes/schemas/response/exception_list_manifest_schema.ts | 0 .../server/lib/exceptions/fetch_endpoint_exceptions.ts | 0 .../{siem => security_solution}/server/lib/exceptions/index.ts | 0 .../server/lib/exceptions/saved_object.ts | 0 .../server/lib/exceptions/saved_object_mappings.ts | 0 .../{siem => security_solution}/server/lib/exceptions/task.ts | 0 .../{siem => security_solution}/server/lib/exceptions/types.ts | 0 13 files changed, 0 insertions(+), 0 deletions(-) rename x-pack/plugins/{siem => security_solution}/server/lib/detection_engine/exceptions/index.ts (100%) rename x-pack/plugins/{siem => security_solution}/server/lib/detection_engine/exceptions/types.ts (100%) rename x-pack/plugins/{siem => security_solution}/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.ts (100%) rename x-pack/plugins/{siem => security_solution}/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts (100%) rename x-pack/plugins/{siem => security_solution}/server/lib/detection_engine/routes/exceptions/validate.ts (100%) rename x-pack/plugins/{siem => security_solution}/server/lib/detection_engine/routes/schemas/download_exception_list_schema.ts (100%) rename x-pack/plugins/{siem => security_solution}/server/lib/detection_engine/routes/schemas/response/exception_list_manifest_schema.ts (100%) rename x-pack/plugins/{siem => security_solution}/server/lib/exceptions/fetch_endpoint_exceptions.ts (100%) rename x-pack/plugins/{siem => security_solution}/server/lib/exceptions/index.ts (100%) rename x-pack/plugins/{siem => security_solution}/server/lib/exceptions/saved_object.ts (100%) rename x-pack/plugins/{siem => security_solution}/server/lib/exceptions/saved_object_mappings.ts (100%) rename x-pack/plugins/{siem => security_solution}/server/lib/exceptions/task.ts (100%) rename x-pack/plugins/{siem => security_solution}/server/lib/exceptions/types.ts (100%) diff --git a/x-pack/plugins/siem/server/lib/detection_engine/exceptions/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/exceptions/index.ts similarity index 100% rename from x-pack/plugins/siem/server/lib/detection_engine/exceptions/index.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/exceptions/index.ts diff --git a/x-pack/plugins/siem/server/lib/detection_engine/exceptions/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/exceptions/types.ts similarity index 100% rename from x-pack/plugins/siem/server/lib/detection_engine/exceptions/types.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/exceptions/types.ts diff --git a/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.ts similarity index 100% rename from x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.ts diff --git a/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts similarity index 100% rename from x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts diff --git a/x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/validate.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/validate.ts similarity index 100% rename from x-pack/plugins/siem/server/lib/detection_engine/routes/exceptions/validate.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/validate.ts diff --git a/x-pack/plugins/siem/server/lib/detection_engine/routes/schemas/download_exception_list_schema.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/schemas/download_exception_list_schema.ts similarity index 100% rename from x-pack/plugins/siem/server/lib/detection_engine/routes/schemas/download_exception_list_schema.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/routes/schemas/download_exception_list_schema.ts diff --git a/x-pack/plugins/siem/server/lib/detection_engine/routes/schemas/response/exception_list_manifest_schema.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/schemas/response/exception_list_manifest_schema.ts similarity index 100% rename from x-pack/plugins/siem/server/lib/detection_engine/routes/schemas/response/exception_list_manifest_schema.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/routes/schemas/response/exception_list_manifest_schema.ts diff --git a/x-pack/plugins/siem/server/lib/exceptions/fetch_endpoint_exceptions.ts b/x-pack/plugins/security_solution/server/lib/exceptions/fetch_endpoint_exceptions.ts similarity index 100% rename from x-pack/plugins/siem/server/lib/exceptions/fetch_endpoint_exceptions.ts rename to x-pack/plugins/security_solution/server/lib/exceptions/fetch_endpoint_exceptions.ts diff --git a/x-pack/plugins/siem/server/lib/exceptions/index.ts b/x-pack/plugins/security_solution/server/lib/exceptions/index.ts similarity index 100% rename from x-pack/plugins/siem/server/lib/exceptions/index.ts rename to x-pack/plugins/security_solution/server/lib/exceptions/index.ts diff --git a/x-pack/plugins/siem/server/lib/exceptions/saved_object.ts b/x-pack/plugins/security_solution/server/lib/exceptions/saved_object.ts similarity index 100% rename from x-pack/plugins/siem/server/lib/exceptions/saved_object.ts rename to x-pack/plugins/security_solution/server/lib/exceptions/saved_object.ts diff --git a/x-pack/plugins/siem/server/lib/exceptions/saved_object_mappings.ts b/x-pack/plugins/security_solution/server/lib/exceptions/saved_object_mappings.ts similarity index 100% rename from x-pack/plugins/siem/server/lib/exceptions/saved_object_mappings.ts rename to x-pack/plugins/security_solution/server/lib/exceptions/saved_object_mappings.ts diff --git a/x-pack/plugins/siem/server/lib/exceptions/task.ts b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts similarity index 100% rename from x-pack/plugins/siem/server/lib/exceptions/task.ts rename to x-pack/plugins/security_solution/server/lib/exceptions/task.ts diff --git a/x-pack/plugins/siem/server/lib/exceptions/types.ts b/x-pack/plugins/security_solution/server/lib/exceptions/types.ts similarity index 100% rename from x-pack/plugins/siem/server/lib/exceptions/types.ts rename to x-pack/plugins/security_solution/server/lib/exceptions/types.ts From e304959f8fa269994cd8be50c26da418a5e198dd Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Thu, 4 Jun 2020 12:34:57 -0400 Subject: [PATCH 016/106] siem -> securitySolution --- .../server/lib/exceptions/saved_object_mappings.ts | 2 +- .../security_solution/server/lib/exceptions/task.ts | 10 +++++----- .../security_solution/server/lib/exceptions/types.ts | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/saved_object_mappings.ts b/x-pack/plugins/security_solution/server/lib/exceptions/saved_object_mappings.ts index 24e701321bc64..492c475979023 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/saved_object_mappings.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/saved_object_mappings.ts @@ -6,7 +6,7 @@ import { SavedObjectsType } from '../../../../../../src/core/server'; -export const exceptionsArtifactSavedObjectType = 'siem-exceptions-artifact'; +export const exceptionsArtifactSavedObjectType = 'securitySolution-exceptions-artifact'; export const exceptionsArtifactSavedObjectMappings: SavedObjectsType['mappings'] = { properties: { diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts index 3c1ddc47d2a2f..6f657c5f9420a 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts @@ -16,12 +16,12 @@ import { GetFullEndpointExceptionList, CompressExceptionList } from './fetch_end const PackagerTaskConstants = { INTERVAL: '30s', TIMEOUT: '1m', - TYPE: 'siem:endpoint:exceptions-packager', + TYPE: 'securitySolution:endpoint:exceptions-packager', }; export const ArtifactConstants = { GLOBAL_ALLOWLIST_NAME: 'endpoint-allowlist', - SAVED_OBJECT_TYPE: 'siem-exceptions-artifact', + SAVED_OBJECT_TYPE: 'securitySolution-exceptions-artifact', SUPPORTED_OPERATING_SYSTEMS: ['windows'], SUPPORTED_SCHEMA_VERSIONS: ['1.0.0'], }; @@ -101,7 +101,7 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { await runnerContext.taskManager.ensureScheduled({ id: PackagerTaskConstants.TYPE, taskType: PackagerTaskConstants.TYPE, - scope: ['siem'], + scope: ['securitySolution'], schedule: { interval: PackagerTaskConstants.INTERVAL, }, @@ -116,8 +116,8 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { }; context.taskManager.registerTaskDefinitions({ - 'siem:endpoint:exceptions-packager': { - title: 'SIEM Endpoint Exceptions Handler', + 'securitySolution:endpoint:exceptions-packager': { + title: 'Security Solution Endpoint Exceptions Handler', type: PackagerTaskConstants.TYPE, timeout: PackagerTaskConstants.TIMEOUT, createTaskRunner: () => { diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/types.ts b/x-pack/plugins/security_solution/server/lib/exceptions/types.ts index 5bd29531dbb55..2b27c5026750f 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/types.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/types.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -type ArtifactName = 'global-whitelist'; +type ArtifactName = 'endpoint-allowlist'; type ArtifactVersion = '1.0.0'; type ArtifactEncoding = 'xz'; From 3288bdaea6e633b3643f08195c395f031f14d7f7 Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Thu, 4 Jun 2020 15:26:24 -0400 Subject: [PATCH 017/106] Progress on downloads, cleanup --- .../lib/detection_engine/exceptions/types.ts | 4 ++ .../download_endpoint_exception_list.ts | 8 +++- .../get_endpoint_exception_list_manifest.ts | 45 +++++++++++++++---- .../get_endpoint_exception_manifest_schema.ts | 11 +++++ .../exceptions/fetch_endpoint_exceptions.ts | 4 +- .../server/lib/exceptions/task.ts | 6 +++ 6 files changed, 65 insertions(+), 13 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/routes/schemas/get_endpoint_exception_manifest_schema.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/exceptions/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/exceptions/types.ts index 53c406d2137d3..a4129e9cfa540 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/exceptions/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/exceptions/types.ts @@ -7,3 +7,7 @@ export interface DownloadExceptionListRequestParams { sha256: string; } + +export interface GetExceptionListManifestRequestParams { + schemaVersion: string; +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.ts index 12658e936e64a..73ef184600769 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.ts @@ -42,9 +42,13 @@ async function handleEndpointExceptionDownload(context, req, res) { searchFields: ['sha256'], }); if (resp.total > 0) { + const artifact = resp.saved_objects[0]; return res.ok({ - body: resp.saved_objects[0].body, - headers: { 'content-encoding': 'xz' }, + body: Buffer.from(artifact.attributes.body), + headers: { + 'content-encoding': 'xz', + 'content-disposition': `attachment; filename=${artifact.attributes.name}.xz`, + }, }); } else if (res.total > 1) { context.logger.warn(`Duplicate allowlist entries found: ${req.params.sha256}`); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts index 776c3c705c251..965fe598cbe45 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts @@ -6,9 +6,26 @@ import { IRouter } from '../../../../../../../../src/core/server'; import { ArtifactConstants } from '../../../exceptions'; +import { buildRouteValidation } from '../utils'; +import { GetExceptionListManifestRequestParams } from '../../exceptions/types'; +import { getExceptionListManifestSchema } from '../schemas/get_endpoint_exception_manifest_schema'; const allowlistBaseRoute: string = '/api/endpoint/allowlist'; +export interface Manifest { + schemaVersion: string; + manifestVersion: string; + artifacts: Artifacts; +} +export interface Artifacts { + [key: string]: Artifact; +} +export interface Artifact { + url: string; + sha256: string; + size: number; +} + /** * Registers the exception list route to enable sensors to retrieve a manifest of available lists */ @@ -16,7 +33,11 @@ export function getEndpointExceptionListManifest(router: IRouter) { router.get( { path: `${allowlistBaseRoute}/manifest/{schemaVersion}`, - validate: {}, + validate: { + params: buildRouteValidation( + getExceptionListManifestSchema + ), + }, options: { authRequired: true }, }, handleAllowlistManifest @@ -29,16 +50,21 @@ export function getEndpointExceptionListManifest(router: IRouter) { async function handleAllowlistManifest(context, req, res) { try { const resp = await getAllowlistManifest(context, req.params.schemaVersion); - const manifestResp = {}; + if (resp.saved_objects.length === 0) { + return res.notFound({ body: `No manifest found for version ${req.params.schemaVersion}` }); + } + const manifestResp: Manifest = { + schemaVersion: req.params.schemaVersion, + manifestVersion: '1.0.0', // TODO hardcode? + artifacts: {}, + }; // transform and validate response for (const manifest of resp.saved_objects) { - manifestResp[manifest.attributes.name] = { - [manifest.attributes.name]: { - url: `${allowlistBaseRoute}/download/${manifest.attributes.sha256}`, - sha256: manifest.attributes.sha256, - size: manifest.attributes.size, // TODO add size - }, + manifestResp.artifacts[manifest.attributes.name] = { + url: `${allowlistBaseRoute}/download/${manifest.attributes.sha256}`, + sha256: manifest.attributes.sha256, + size: manifest.attributes.size, }; } @@ -51,9 +77,10 @@ async function handleAllowlistManifest(context, req, res) { /** * Creates the manifest for the whitelist */ -async function getAllowlistManifest(ctx, schemaVersion) { +async function getAllowlistManifest(ctx, schemaVersion: string) { const soClient = ctx.core.savedObjects.client; + // TODO page const manifestResp = soClient.find({ type: ArtifactConstants.SAVED_OBJECT_TYPE, fields: ['name', 'schemaVersion', 'sha256', 'encoding', 'size', 'created'], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/schemas/get_endpoint_exception_manifest_schema.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/schemas/get_endpoint_exception_manifest_schema.ts new file mode 100644 index 0000000000000..12801533205ef --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/schemas/get_endpoint_exception_manifest_schema.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import Joi from 'joi'; + +export const getExceptionListManifestSchema = Joi.object({ + schemaVersion: Joi.string(), +}); diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/fetch_endpoint_exceptions.ts b/x-pack/plugins/security_solution/server/lib/exceptions/fetch_endpoint_exceptions.ts index c7100f722e233..e77e7fafcd7a4 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/fetch_endpoint_exceptions.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/fetch_endpoint_exceptions.ts @@ -40,8 +40,8 @@ export async function GetFullEndpointExceptionList( do { const response = await eClient.findExceptionListItem({ listId: 'endpoint_list', - namespaceType: 'agnostic', - filter: `_tags:"sensor:${os}"`, + namespaceType: 'single', + filter: undefined, // TODO this doesnt work `_tags:"sensor:${os}"`, perPage: 100, page, sortField: 'created_at', diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts index 6f657c5f9420a..a400caeb1ed97 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts @@ -56,6 +56,12 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { for (const os of ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS) { const exceptions = await GetFullEndpointExceptionList(exceptionListClient, os); + // Don't create an artifact if there are no exceptions + if (exceptions.exceptions_list.length === 0) { + context.logger.debug('No endpoint exceptions found.'); + return; + } + const compressedExceptions: Buffer = await CompressExceptionList(exceptions); const sha256Hash = createHash('sha256') From e33563fe58145313d1b13e66cef2f50bb3656ca4 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Sun, 7 Jun 2020 20:07:52 -0400 Subject: [PATCH 018/106] Add config, update artifact creation, add TODOs --- .../security_solution/server/config.ts | 5 +++ .../get_endpoint_exception_list_manifest.ts | 8 +++- .../schemas/download_exception_list_schema.ts | 3 ++ .../get_endpoint_exception_manifest_schema.ts | 4 ++ .../server/lib/exceptions/task.ts | 25 ++++++++--- .../security_solution/server/plugin.ts | 6 +-- .../scripts/exceptions/check_env_variables.sh | 41 +++++++++++++++++++ .../scripts/exceptions/get_artifact_by_id.sh | 12 ++++++ .../server/scripts/exceptions/get_manifest.sh | 12 ++++++ 9 files changed, 106 insertions(+), 10 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/scripts/exceptions/check_env_variables.sh create mode 100644 x-pack/plugins/security_solution/server/scripts/exceptions/get_artifact_by_id.sh create mode 100644 x-pack/plugins/security_solution/server/scripts/exceptions/get_manifest.sh diff --git a/x-pack/plugins/security_solution/server/config.ts b/x-pack/plugins/security_solution/server/config.ts index f7d961ae3ec5c..6868c7dde84c4 100644 --- a/x-pack/plugins/security_solution/server/config.ts +++ b/x-pack/plugins/security_solution/server/config.ts @@ -29,6 +29,11 @@ export const configSchema = schema.object({ from: schema.string({ defaultValue: 'now-15m' }), to: schema.string({ defaultValue: 'now' }), }), + + /** + * Artifact Configuration + */ + packagerTaskInterval: schema.number({ defaultValue: 60 }), }); export const createConfig$ = (context: PluginInitializerContext) => diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts index 965fe598cbe45..e1439abd43521 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts @@ -32,7 +32,7 @@ export interface Artifact { export function getEndpointExceptionListManifest(router: IRouter) { router.get( { - path: `${allowlistBaseRoute}/manifest/{schemaVersion}`, + path: `${allowlistBaseRoute}/manifest/{manifestVersion}/{schemaVersion}`, validate: { params: buildRouteValidation( getExceptionListManifestSchema @@ -49,6 +49,10 @@ export function getEndpointExceptionListManifest(router: IRouter) { */ async function handleAllowlistManifest(context, req, res) { try { + if (req.params.manifestVersion !== '1.0.0') { + return res.badRequest('invalid manifest version'); + } + const resp = await getAllowlistManifest(context, req.params.schemaVersion); if (resp.saved_objects.length === 0) { return res.notFound({ body: `No manifest found for version ${req.params.schemaVersion}` }); @@ -77,7 +81,7 @@ async function handleAllowlistManifest(context, req, res) { /** * Creates the manifest for the whitelist */ -async function getAllowlistManifest(ctx, schemaVersion: string) { +async function getAllowlistManifest(ctx, manifestVersion: string, schemaVersion: string) { const soClient = ctx.core.savedObjects.client; // TODO page diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/schemas/download_exception_list_schema.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/schemas/download_exception_list_schema.ts index 73d6650d0f0f0..557f4e847e699 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/schemas/download_exception_list_schema.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/schemas/download_exception_list_schema.ts @@ -6,6 +6,9 @@ import Joi from 'joi'; +// TODO: convert to io-tsk +// import * as t from 'io-ts'; + export const downloadExceptionListSchema = Joi.object({ sha256: Joi.string(), }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/schemas/get_endpoint_exception_manifest_schema.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/schemas/get_endpoint_exception_manifest_schema.ts index 12801533205ef..b73af1c1ad357 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/schemas/get_endpoint_exception_manifest_schema.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/schemas/get_endpoint_exception_manifest_schema.ts @@ -6,6 +6,10 @@ import Joi from 'joi'; +// TODO: convert to io-tsk +// import * as t from 'io-ts'; + export const getExceptionListManifestSchema = Joi.object({ + manifestVersion: Joi.string(), schemaVersion: Joi.string(), }); diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts index a400caeb1ed97..87688f70e683a 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts @@ -11,10 +11,10 @@ import { TaskManagerStartContract, } from '../../../../../plugins/task_manager/server'; import { ListPluginSetup } from '../../../../lists/server'; +import { ConfigType } from '../../config'; import { GetFullEndpointExceptionList, CompressExceptionList } from './fetch_endpoint_exceptions'; const PackagerTaskConstants = { - INTERVAL: '30s', TIMEOUT: '1m', TYPE: 'securitySolution:endpoint:exceptions-packager', }; @@ -36,6 +36,7 @@ interface PackagerTaskRunner { interface PackagerTaskContext { core: CoreSetup; + config: ConfigType; logger: Logger; taskManager: TaskManagerSetupContract; lists: ListPluginSetup; @@ -58,7 +59,7 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { const exceptions = await GetFullEndpointExceptionList(exceptionListClient, os); // Don't create an artifact if there are no exceptions if (exceptions.exceptions_list.length === 0) { - context.logger.debug('No endpoint exceptions found.'); + context.logger.debug(`No endpoint exceptions found for ${os}.`); return; } @@ -79,23 +80,37 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { size: Buffer.from(JSON.stringify(exceptions)).byteLength, }; + /* const resp = await soClient.find({ type: ArtifactConstants.SAVED_OBJECT_TYPE, search: sha256Hash, searchFields: ['sha256'], fields: [], }); + */ // TODO clean this up and handle errors better - if (resp.total === 0) { + // if (resp.total === 0) { + try { const soResponse = await soClient.create( ArtifactConstants.SAVED_OBJECT_TYPE, - exceptionSO + exceptionSO, + { id: sha256Hash } ); context.logger.debug(JSON.stringify(soResponse)); + } catch (error) { + if (error.statusCode === 409) { + context.logger.debug('No update to Endpoint Exceptions, skipping.'); + } else { + context.logger.error(error); + } + } + + /* } else { context.logger.debug('No update to Endpoint Exceptions, skipping.'); } + */ } } }; @@ -109,7 +124,7 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { taskType: PackagerTaskConstants.TYPE, scope: ['securitySolution'], schedule: { - interval: PackagerTaskConstants.INTERVAL, + interval: context.config.packagerTaskInterval, }, state: {}, params: {}, diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index f0968f579f53f..e3d9bd116c3d8 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -18,7 +18,6 @@ import { import { PluginSetupContract as AlertingSetup } from '../../alerts/server'; import { SecurityPluginSetup as SecuritySetup } from '../../security/server'; import { PluginSetupContract as FeaturesSetup } from '../../features/server'; -import { ListPluginSetup as ListsSetup } from '../../lists/server'; import { MlPluginSetup as MlSetup } from '../../ml/server'; import { ListPluginSetup } from '../../lists/server'; import { EncryptedSavedObjectsPluginSetup as EncryptedSavedObjectsSetup } from '../../encrypted_saved_objects/server'; @@ -66,9 +65,9 @@ export interface StartPlugins { } // eslint-disable-next-line @typescript-eslint/no-empty-interface -export interface PluginSetup { } +export interface PluginSetup {} // eslint-disable-next-line @typescript-eslint/no-empty-interface -export interface PluginStart { } +export interface PluginStart {} export class Plugin implements IPlugin { private readonly logger: Logger; @@ -220,6 +219,7 @@ export class Plugin implements IPlugin Date: Mon, 8 Jun 2020 12:32:06 -0400 Subject: [PATCH 019/106] Fixing buffer serialization problem --- .../routes/exceptions/download_endpoint_exception_list.ts | 4 +++- .../plugins/security_solution/server/lib/exceptions/task.ts | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.ts index 73ef184600769..ca82cca50b33b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.ts @@ -43,8 +43,10 @@ async function handleEndpointExceptionDownload(context, req, res) { }); if (resp.total > 0) { const artifact = resp.saved_objects[0]; + const outBuffer = Buffer.from(artifact.attributes.body, 'binary'); + return res.ok({ - body: Buffer.from(artifact.attributes.body), + body: outBuffer, headers: { 'content-encoding': 'xz', 'content-disposition': `attachment; filename=${artifact.attributes.name}.xz`, diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts index 87688f70e683a..3de2af9277717 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts @@ -76,7 +76,7 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { sha256: sha256Hash, encoding: 'xz', created: Date.now(), - body: compressedExceptions.toString(), + body: compressedExceptions.toString('binary'), size: Buffer.from(JSON.stringify(exceptions)).byteLength, }; From 6a440822e77b697b7dc389ec757f3b63a0a907d6 Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Mon, 8 Jun 2020 13:54:07 -0400 Subject: [PATCH 020/106] Adding cleanup to task --- .../server/lib/exceptions/task.ts | 40 ++++++++++--------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts index 3de2af9277717..0b1077e6020af 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts @@ -70,8 +70,9 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { .digest('hex'); for (const schemaVersion of ArtifactConstants.SUPPORTED_SCHEMA_VERSIONS) { + const artifactName = `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-${os}`; const exceptionSO = { - name: `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-${os}`, + name: artifactName, schemaVersion, sha256: sha256Hash, encoding: 'xz', @@ -80,24 +81,33 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { size: Buffer.from(JSON.stringify(exceptions)).byteLength, }; - /* - const resp = await soClient.find({ - type: ArtifactConstants.SAVED_OBJECT_TYPE, - search: sha256Hash, - searchFields: ['sha256'], - fields: [], - }); - */ - - // TODO clean this up and handle errors better - // if (resp.total === 0) { try { + // Create the new artifact const soResponse = await soClient.create( ArtifactConstants.SAVED_OBJECT_TYPE, exceptionSO, { id: sha256Hash } ); context.logger.debug(JSON.stringify(soResponse)); + + // Clean up old artifacts + const otherArtifacts = await soClient.find({ + type: ArtifactConstants.SAVED_OBJECT_TYPE, + search: artifactName, + searchFields: ['name'], + sortField: 'created', + sortOrder: 'desc', + }); + + // Remove all but the latest artifact + const toDelete = otherArtifacts.saved_objects.slice( + 1, + otherArtifacts.saved_objects.length + ); + for (const delObj of toDelete) { + context.logger.debug(`REMOVING ${delObj.id}`); + await soClient.delete(ArtifactConstants.SAVED_OBJECT_TYPE, delObj.id); + } } catch (error) { if (error.statusCode === 409) { context.logger.debug('No update to Endpoint Exceptions, skipping.'); @@ -105,12 +115,6 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { context.logger.error(error); } } - - /* - } else { - context.logger.debug('No update to Endpoint Exceptions, skipping.'); - } - */ } } }; From 62079c71615006b78db2e34282c096f3ec1b0ba6 Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Mon, 8 Jun 2020 15:16:42 -0400 Subject: [PATCH 021/106] Handle HEAD req --- .../get_endpoint_exception_list_manifest.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts index e1439abd43521..bebd79a72b50a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import { createHash } from 'crypto'; import { IRouter } from '../../../../../../../../src/core/server'; import { ArtifactConstants } from '../../../exceptions'; import { buildRouteValidation } from '../utils'; @@ -53,7 +54,7 @@ async function handleAllowlistManifest(context, req, res) { return res.badRequest('invalid manifest version'); } - const resp = await getAllowlistManifest(context, req.params.schemaVersion); + const resp = await getAllowlistManifest(context, '1.0.0', req.params.schemaVersion); if (resp.saved_objects.length === 0) { return res.notFound({ body: `No manifest found for version ${req.params.schemaVersion}` }); } @@ -63,6 +64,15 @@ async function handleAllowlistManifest(context, req, res) { artifacts: {}, }; + // Handle a HEAD request + if (req.route.method === 'head') { + const manifestHash = createHash('sha256') + .update(JSON.stringify(manifestResp), 'utf8') + .digest('hex'); + + return res.ok({ headers: { 'e-tag': manifestHash } }); + } + // transform and validate response for (const manifest of resp.saved_objects) { manifestResp.artifacts[manifest.attributes.name] = { From 502fd5e14e858c6f8f969850f0d8eead757dd765 Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Mon, 8 Jun 2020 15:17:55 -0400 Subject: [PATCH 022/106] proper header --- .../routes/exceptions/get_endpoint_exception_list_manifest.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts index bebd79a72b50a..edc6072f454c1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts @@ -70,7 +70,7 @@ async function handleAllowlistManifest(context, req, res) { .update(JSON.stringify(manifestResp), 'utf8') .digest('hex'); - return res.ok({ headers: { 'e-tag': manifestHash } }); + return res.ok({ headers: { ETag: manifestHash } }); } // transform and validate response From 4156900892326e5268e3dd40c780e0d74bb37f26 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Mon, 8 Jun 2020 15:55:54 -0400 Subject: [PATCH 023/106] More robust task management --- .../server/lib/exceptions/task.ts | 48 ++++++++++++------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts index 0b1077e6020af..001e686589190 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts @@ -7,6 +7,7 @@ import { createHash } from 'crypto'; import { CoreSetup, Logger, SavedObjectsClient } from '../../../../../../src/core/server'; import { + ConcreteTaskInstance, TaskManagerSetupContract, TaskManagerStartContract, } from '../../../../../plugins/task_manager/server'; @@ -17,6 +18,7 @@ import { GetFullEndpointExceptionList, CompressExceptionList } from './fetch_end const PackagerTaskConstants = { TIMEOUT: '1m', TYPE: 'securitySolution:endpoint:exceptions-packager', + VERSION: '1.0.0', }; export const ArtifactConstants = { @@ -47,7 +49,17 @@ interface PackagerTaskRunnerContext { } export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { - const run = async () => { + const getTaskId = (): string => { + return `${PackagerTaskConstants.TYPE}:${PackagerTaskConstants.VERSION}`; + }; + + const run = async (taskId: string) => { + if (taskId !== getTaskId()) { + // old task, return + context.logger.debug(`Outdated task running: ${taskId}`); + return; + } + context.logger.debug('Running exception list packager task'); const [{ savedObjects }] = await context.core.getStartServices(); @@ -57,12 +69,6 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { for (const os of ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS) { const exceptions = await GetFullEndpointExceptionList(exceptionListClient, os); - // Don't create an artifact if there are no exceptions - if (exceptions.exceptions_list.length === 0) { - context.logger.debug(`No endpoint exceptions found for ${os}.`); - return; - } - const compressedExceptions: Buffer = await CompressExceptionList(exceptions); const sha256Hash = createHash('sha256') @@ -117,22 +123,22 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { } } } + return true; }; - const getTaskRunner = (runnerContext: PackagerTaskRunnerContext) => { + const getTaskRunner = (runnerContext: PackagerTaskRunnerContext): PackagerTaskRunner => { return { run: async () => { try { - await runnerContext.taskManager.ensureScheduled({ - id: PackagerTaskConstants.TYPE, + const taskId = getTaskId(); + const taskInstance = await runnerContext.taskManager.ensureScheduled({ + id: taskId, taskType: PackagerTaskConstants.TYPE, scope: ['securitySolution'], - schedule: { - interval: context.config.packagerTaskInterval, - }, state: {}, - params: {}, + params: { version: PackagerTaskConstants.VERSION }, }); + await runnerContext.taskManager.runNow(taskInstance.id); } catch (e) { context.logger.debug(`Error scheduling task, received ${e.message}`); } @@ -141,14 +147,22 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { }; context.taskManager.registerTaskDefinitions({ - 'securitySolution:endpoint:exceptions-packager': { + [PackagerTaskConstants.TYPE]: { title: 'Security Solution Endpoint Exceptions Handler', type: PackagerTaskConstants.TYPE, timeout: PackagerTaskConstants.TIMEOUT, - createTaskRunner: () => { + createTaskRunner: ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => { return { run: async () => { - await run(); + await run(taskInstance.id); + + const nextRun = new Date(); + nextRun.setSeconds(nextRun.getSeconds() + context.config.packagerTaskInterval); + + return { + state: {}, + runAt: nextRun, + }; }, cancel: async () => {}, }; From 2d2172330283bf97e3b35253ff39d8dccf61650d Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Mon, 8 Jun 2020 16:13:30 -0400 Subject: [PATCH 024/106] single -> agnostic --- .../server/lib/exceptions/fetch_endpoint_exceptions.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/fetch_endpoint_exceptions.ts b/x-pack/plugins/security_solution/server/lib/exceptions/fetch_endpoint_exceptions.ts index e77e7fafcd7a4..19d0fc16860f8 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/fetch_endpoint_exceptions.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/fetch_endpoint_exceptions.ts @@ -40,7 +40,7 @@ export async function GetFullEndpointExceptionList( do { const response = await eClient.findExceptionListItem({ listId: 'endpoint_list', - namespaceType: 'single', + namespaceType: 'agnostic', filter: undefined, // TODO this doesnt work `_tags:"sensor:${os}"`, perPage: 100, page, From 86732e2f99b533b18e2c77acf7120a91ae0e9725 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Mon, 8 Jun 2020 17:54:52 -0400 Subject: [PATCH 025/106] Fix OS filtering --- .../lib/exceptions/fetch_endpoint_exceptions.ts | 2 +- .../security_solution/server/lib/exceptions/task.ts | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/fetch_endpoint_exceptions.ts b/x-pack/plugins/security_solution/server/lib/exceptions/fetch_endpoint_exceptions.ts index 19d0fc16860f8..0c2eba6d515f8 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/fetch_endpoint_exceptions.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/fetch_endpoint_exceptions.ts @@ -41,7 +41,7 @@ export async function GetFullEndpointExceptionList( const response = await eClient.findExceptionListItem({ listId: 'endpoint_list', namespaceType: 'agnostic', - filter: undefined, // TODO this doesnt work `_tags:"sensor:${os}"`, + filter: `exception-list-agnostic.attributes._tags:\"os:${os}\"`, perPage: 100, page, sortField: 'created_at', diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts index 001e686589190..49f236b973f49 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts @@ -24,7 +24,7 @@ const PackagerTaskConstants = { export const ArtifactConstants = { GLOBAL_ALLOWLIST_NAME: 'endpoint-allowlist', SAVED_OBJECT_TYPE: 'securitySolution-exceptions-artifact', - SUPPORTED_OPERATING_SYSTEMS: ['windows'], + SUPPORTED_OPERATING_SYSTEMS: ['linux', 'windows'], SUPPORTED_SCHEMA_VERSIONS: ['1.0.0'], }; @@ -92,7 +92,7 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { const soResponse = await soClient.create( ArtifactConstants.SAVED_OBJECT_TYPE, exceptionSO, - { id: sha256Hash } + { id: `${artifactName}-${sha256Hash}` } ); context.logger.debug(JSON.stringify(soResponse)); @@ -129,19 +129,20 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { const getTaskRunner = (runnerContext: PackagerTaskRunnerContext): PackagerTaskRunner => { return { run: async () => { + const taskId = getTaskId(); try { - const taskId = getTaskId(); - const taskInstance = await runnerContext.taskManager.ensureScheduled({ + await runnerContext.taskManager.ensureScheduled({ id: taskId, taskType: PackagerTaskConstants.TYPE, scope: ['securitySolution'], state: {}, params: { version: PackagerTaskConstants.VERSION }, }); - await runnerContext.taskManager.runNow(taskInstance.id); } catch (e) { context.logger.debug(`Error scheduling task, received ${e.message}`); } + + await runnerContext.taskManager.runNow(taskId); }, }; }; From a15566e0580055a5745641c409b6c63ff36b640c Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Mon, 8 Jun 2020 23:21:04 -0400 Subject: [PATCH 026/106] Scaffolding digital signatures / tests --- .../lib/detection_engine/exceptions/types.ts | 2 ++ .../download_endpoint_exception_list.test.ts | 7 +++++ ...t_endpoint_exception_list_manifest.test.ts | 7 +++++ .../get_endpoint_exception_list_manifest.ts | 5 ++++ .../server/lib/exceptions/exceptions.test.ts | 7 +++++ .../server/lib/exceptions/task.ts | 1 + .../apis/security_solution/exceptions.ts | 27 +++++++++++++++++++ 7 files changed, 56 insertions(+) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/exceptions/exceptions.test.ts create mode 100644 x-pack/test/api_integration/apis/security_solution/exceptions.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/exceptions/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/exceptions/types.ts index a4129e9cfa540..345c0a1058344 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/exceptions/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/exceptions/types.ts @@ -5,9 +5,11 @@ */ export interface DownloadExceptionListRequestParams { + // TODO: this should probably be id? sha256: string; } export interface GetExceptionListManifestRequestParams { + manifestVersion: string; schemaVersion: string; } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.test.ts new file mode 100644 index 0000000000000..654341f5a6511 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.test.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +// TODO diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.test.ts new file mode 100644 index 0000000000000..654341f5a6511 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.test.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +// TODO diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts index edc6072f454c1..3851438421c7f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts @@ -17,6 +17,7 @@ export interface Manifest { schemaVersion: string; manifestVersion: string; artifacts: Artifacts; + signature: string; // TODO: stronger type? } export interface Artifacts { [key: string]: Artifact; @@ -58,6 +59,7 @@ async function handleAllowlistManifest(context, req, res) { if (resp.saved_objects.length === 0) { return res.notFound({ body: `No manifest found for version ${req.params.schemaVersion}` }); } + const manifestResp: Manifest = { schemaVersion: req.params.schemaVersion, manifestVersion: '1.0.0', // TODO hardcode? @@ -82,6 +84,9 @@ async function handleAllowlistManifest(context, req, res) { }; } + // TODO: digitally sign + manifestResp.signature = 'abcd'; + return res.ok({ body: manifestResp }); } catch (err) { return res.internalError({ body: err }); diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/exceptions.test.ts b/x-pack/plugins/security_solution/server/lib/exceptions/exceptions.test.ts new file mode 100644 index 0000000000000..654341f5a6511 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/exceptions/exceptions.test.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +// TODO diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts index 49f236b973f49..dbca1e32849b4 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts @@ -89,6 +89,7 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { try { // Create the new artifact + // TODO: let id be auto-generated... revert to previous algorithm, doesn't need to be atomic. const soResponse = await soClient.create( ArtifactConstants.SAVED_OBJECT_TYPE, exceptionSO, diff --git a/x-pack/test/api_integration/apis/security_solution/exceptions.ts b/x-pack/test/api_integration/apis/security_solution/exceptions.ts new file mode 100644 index 0000000000000..30aca7df83e58 --- /dev/null +++ b/x-pack/test/api_integration/apis/security_solution/exceptions.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; + +import { FtrProviderContext } from '../../ftr_provider_context'; + +export default function ({ getService }: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + + describe('artifact manifest', () => { + before(() => esArchiver.load('TODO')); + after(() => esArchiver.unload('TODO')); + + it('Do a manifest test', () => {}); + }); + + describe('artifact download', () => { + before(() => esArchiver.load('TODO')); + after(() => esArchiver.unload('TODO')); + + it('Do a download test', () => {}); + }); +} From 0dbb443ba5899c374f9b25f75e710f4c1c160650 Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Wed, 10 Jun 2020 14:19:01 -0400 Subject: [PATCH 027/106] Adds rotue for creating endpoint user --- .../download_endpoint_exception_list.ts | 2 - .../server/lib/exceptions/task.ts | 2 +- .../server/lib/setup/index.ts | 89 +++++++++++++++++++ .../security_solution/server/routes/index.ts | 3 + 4 files changed, 93 insertions(+), 3 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/setup/index.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.ts index ca82cca50b33b..437366228b3e4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.ts @@ -52,8 +52,6 @@ async function handleEndpointExceptionDownload(context, req, res) { 'content-disposition': `attachment; filename=${artifact.attributes.name}.xz`, }, }); - } else if (res.total > 1) { - context.logger.warn(`Duplicate allowlist entries found: ${req.params.sha256}`); } else { return res.notFound(); } diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts index dbca1e32849b4..4bfcfab9d2737 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts @@ -117,7 +117,7 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { } } catch (error) { if (error.statusCode === 409) { - context.logger.debug('No update to Endpoint Exceptions, skipping.'); + context.logger.debug(`No update to Endpoint Exceptions (${artifactName}), skipping.`); } else { context.logger.error(error); } diff --git a/x-pack/plugins/security_solution/server/lib/setup/index.ts b/x-pack/plugins/security_solution/server/lib/setup/index.ts new file mode 100644 index 0000000000000..a36df404ad2e4 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/setup/index.ts @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import uuid from 'uuid'; +import { FakeRequest, IRouter, KibanaRequest, SavedObjectsClientContract } from 'src/core/server'; +import { SetupPlugins } from '../../plugin'; + +const ENDPOINT_MANAGER_ROLE = 'endpoint_manager_role'; +const ENDPOINT_MANAGER_USERNAME = 'endpoint_admin'; + +/** + * Registers the setup route that enables the creation of an endpoint user + */ +export function postEndpointSetup(router: IRouter, security: SetupPlugins['security']) { + router.post( + { + path: '/api/endpoint/setup', // TODO + validate: {}, + options: { authRequired: true }, + }, + async (context, req, res) => { + try { + const soClient = context.core.savedObjects.client; + const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser; + await setupEndpointUser(soClient, security, callCluster); + return res.ok(); + } catch (err) { + return res.internalError({ body: err }); + } + } + ); +} + +/** + * Creates the endpoint user and generates an API key to be used by endpoints while communicating with Kibana + */ +async function setupEndpointUser( + soClient: SavedObjectsClientContract, + security: SetupPlugins['security'], + callCluster: any // todo +) { + const res = await callCluster('transport.request', { + method: 'PUT', + path: `/_security/role/${ENDPOINT_MANAGER_ROLE}`, + body: { + cluster: ['monitor', 'manage_api_key'], + indices: [ + { + names: ['logs-*', 'metrics-*', 'events-*'], + privileges: ['write', 'create_index'], + }, + ], + }, + }); + + const password = Buffer.from(uuid.v4()).toString('base64'); + + const resp = await callCluster('transport.request', { + method: 'PUT', + path: `/_security/user/${ENDPOINT_MANAGER_USERNAME}`, + body: { + password, + roles: [ENDPOINT_MANAGER_ROLE], + metadata: { + updated_at: new Date().toISOString(), + }, + }, + }); + + const request: FakeRequest = { + headers: { + authorization: `Basic ${Buffer.from(`${ENDPOINT_MANAGER_USERNAME}:${password}`).toString( + 'base64' + )}`, + }, + }; + + if (!security) { + throw new Error('Missing security plugin'); + } + + const apikey = await security.authc.createAPIKey(request as KibanaRequest, { + name: 'test-api2', + role_descriptors: {}, // TODO + }); +} diff --git a/x-pack/plugins/security_solution/server/routes/index.ts b/x-pack/plugins/security_solution/server/routes/index.ts index 25f4a7a47783a..9d02706d4feec 100644 --- a/x-pack/plugins/security_solution/server/routes/index.ts +++ b/x-pack/plugins/security_solution/server/routes/index.ts @@ -36,6 +36,7 @@ import { createTimelinesRoute } from '../lib/timeline/routes/create_timelines_ro import { updateTimelinesRoute } from '../lib/timeline/routes/update_timelines_route'; import { getDraftTimelinesRoute } from '../lib/timeline/routes/get_draft_timelines_route'; import { cleanDraftTimelinesRoute } from '../lib/timeline/routes/clean_draft_timelines_route'; +import { postEndpointSetup } from '../lib/setup'; import { SetupPlugins } from '../plugin'; import { ConfigType } from '../config'; @@ -77,6 +78,8 @@ export const initRoutes = ( findRulesStatusesRoute(router); + postEndpointSetup(router, security); + // Detection Engine Signals routes that have the REST endpoints of /api/detection_engine/signals // POST /api/detection_engine/signals/status // Example usage can be found in security_solution/server/lib/detection_engine/scripts/signals From 47ccc35ed93f77de3836e88d3aa614a8dad06d0a Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Wed, 10 Jun 2020 14:19:40 -0400 Subject: [PATCH 028/106] Cleanup --- .../server/lib/setup/index.ts | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/setup/index.ts b/x-pack/plugins/security_solution/server/lib/setup/index.ts index a36df404ad2e4..0f05748e091d5 100644 --- a/x-pack/plugins/security_solution/server/lib/setup/index.ts +++ b/x-pack/plugins/security_solution/server/lib/setup/index.ts @@ -5,7 +5,13 @@ */ import uuid from 'uuid'; -import { FakeRequest, IRouter, KibanaRequest, SavedObjectsClientContract } from 'src/core/server'; +import { + FakeRequest, + IRouter, + KibanaRequest, + SavedObjectsClientContract, + IScopedClusterClient, +} from 'src/core/server'; import { SetupPlugins } from '../../plugin'; const ENDPOINT_MANAGER_ROLE = 'endpoint_manager_role'; @@ -24,8 +30,8 @@ export function postEndpointSetup(router: IRouter, security: SetupPlugins['secur async (context, req, res) => { try { const soClient = context.core.savedObjects.client; - const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser; - await setupEndpointUser(soClient, security, callCluster); + const client = context.core.elasticsearch.legacy.client; + await setupEndpointUser(soClient, security, client); return res.ok(); } catch (err) { return res.internalError({ body: err }); @@ -40,9 +46,9 @@ export function postEndpointSetup(router: IRouter, security: SetupPlugins['secur async function setupEndpointUser( soClient: SavedObjectsClientContract, security: SetupPlugins['security'], - callCluster: any // todo + client: IScopedClusterClient // todo ) { - const res = await callCluster('transport.request', { + const res = await client.callAsCurrentUser('transport.request', { method: 'PUT', path: `/_security/role/${ENDPOINT_MANAGER_ROLE}`, body: { @@ -58,7 +64,7 @@ async function setupEndpointUser( const password = Buffer.from(uuid.v4()).toString('base64'); - const resp = await callCluster('transport.request', { + const resp = await client.callAsCurrentUser('transport.request', { method: 'PUT', path: `/_security/user/${ENDPOINT_MANAGER_USERNAME}`, body: { From 4b8aa0c7bcbbc0287735e8b58614bb170e6c4697 Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Wed, 10 Jun 2020 17:28:00 -0400 Subject: [PATCH 029/106] persisting user --- .../server/lib/setup/index.ts | 21 +++++++++- .../server/lib/setup/saved_object_mappings.ts | 40 +++++++++++++++++++ .../security_solution/server/saved_objects.ts | 2 + 3 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugins/security_solution/server/lib/setup/saved_object_mappings.ts diff --git a/x-pack/plugins/security_solution/server/lib/setup/index.ts b/x-pack/plugins/security_solution/server/lib/setup/index.ts index 0f05748e091d5..810c3aad7015d 100644 --- a/x-pack/plugins/security_solution/server/lib/setup/index.ts +++ b/x-pack/plugins/security_solution/server/lib/setup/index.ts @@ -13,6 +13,7 @@ import { IScopedClusterClient, } from 'src/core/server'; import { SetupPlugins } from '../../plugin'; +import { endpointUserSavedObjectType, EndpointUser } from './saved_object_mappings'; const ENDPOINT_MANAGER_ROLE = 'endpoint_manager_role'; const ENDPOINT_MANAGER_USERNAME = 'endpoint_admin'; @@ -34,6 +35,7 @@ export function postEndpointSetup(router: IRouter, security: SetupPlugins['secur await setupEndpointUser(soClient, security, client); return res.ok(); } catch (err) { + console.log(err); return res.internalError({ body: err }); } } @@ -88,8 +90,25 @@ async function setupEndpointUser( throw new Error('Missing security plugin'); } - const apikey = await security.authc.createAPIKey(request as KibanaRequest, { + const apikeyResponse = await security.authc.createAPIKey(request as KibanaRequest, { name: 'test-api2', role_descriptors: {}, // TODO }); + + if (apikeyResponse !== null) { + const endpointUser: EndpointUser = { + username: ENDPOINT_MANAGER_USERNAME, + password, + apikey: apikeyResponse.api_key, + created: Date.now(), + }; + await persistEndpointUser(soClient, endpointUser); + } else { + throw new Error('Unable to persist endpoint user'); + } +} + +async function persistEndpointUser(soClient: SavedObjectsClientContract, user: EndpointUser) { + const soResponse = await soClient.create(endpointUserSavedObjectType, user, {}); + console.log(soResponse); } diff --git a/x-pack/plugins/security_solution/server/lib/setup/saved_object_mappings.ts b/x-pack/plugins/security_solution/server/lib/setup/saved_object_mappings.ts new file mode 100644 index 0000000000000..80605a0573ad7 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/setup/saved_object_mappings.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SavedObjectsType } from '../../../../../../src/core/server'; + +export const endpointUserSavedObjectType = 'securitySolution-endpoint-user'; + +export const endpointUserSavedObjectMappings: SavedObjectsType['mappings'] = { + properties: { + username: { + type: 'keyword', + }, + password: { + type: 'keyword', + }, + apikey: { + type: 'keyword', + }, + created: { + type: 'date', + }, + }, +}; + +export const type: SavedObjectsType = { + name: endpointUserSavedObjectType, + hidden: true, + namespaceType: 'agnostic', + mappings: endpointUserSavedObjectMappings, +}; + +export interface EndpointUser { + username: string; + password: string; + apikey: string; + created: number; +} diff --git a/x-pack/plugins/security_solution/server/saved_objects.ts b/x-pack/plugins/security_solution/server/saved_objects.ts index 2a8c03369eebb..c5941c2d0b2a0 100644 --- a/x-pack/plugins/security_solution/server/saved_objects.ts +++ b/x-pack/plugins/security_solution/server/saved_objects.ts @@ -12,6 +12,7 @@ import { type as timelineType } from './lib/timeline/saved_object_mappings'; import { type as ruleStatusType } from './lib/detection_engine/rules/saved_object_mappings'; import { type as ruleActionsType } from './lib/detection_engine/rule_actions/saved_object_mappings'; import { type as exceptionsArtifactType } from './lib/exceptions/saved_object_mappings'; +import { type as endpointUserType } from './lib/setup/saved_object_mappings'; const types = [ noteType, @@ -20,6 +21,7 @@ const types = [ ruleStatusType, timelineType, exceptionsArtifactType, + endpointUserType, ]; export const savedObjectTypes = types.map((type) => type.name); From 056f377b54f9be197aabfa20254a748edb7cecf5 Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Thu, 11 Jun 2020 14:12:54 -0400 Subject: [PATCH 030/106] Adding route to fetch created user --- .../server/lib/setup/index.ts | 36 +++++++++++++++++-- .../server/lib/setup/saved_object_mappings.ts | 2 +- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/setup/index.ts b/x-pack/plugins/security_solution/server/lib/setup/index.ts index 810c3aad7015d..52e77007c1078 100644 --- a/x-pack/plugins/security_solution/server/lib/setup/index.ts +++ b/x-pack/plugins/security_solution/server/lib/setup/index.ts @@ -17,6 +17,7 @@ import { endpointUserSavedObjectType, EndpointUser } from './saved_object_mappin const ENDPOINT_MANAGER_ROLE = 'endpoint_manager_role'; const ENDPOINT_MANAGER_USERNAME = 'endpoint_admin'; +const ENDPOINT_SETUP_ROUTE = '/api/endpoint/setup'; // TODO /** * Registers the setup route that enables the creation of an endpoint user @@ -24,7 +25,7 @@ const ENDPOINT_MANAGER_USERNAME = 'endpoint_admin'; export function postEndpointSetup(router: IRouter, security: SetupPlugins['security']) { router.post( { - path: '/api/endpoint/setup', // TODO + path: ENDPOINT_SETUP_ROUTE, validate: {}, options: { authRequired: true }, }, @@ -35,7 +36,22 @@ export function postEndpointSetup(router: IRouter, security: SetupPlugins['secur await setupEndpointUser(soClient, security, client); return res.ok(); } catch (err) { - console.log(err); + return res.internalError({ body: err }); + } + } + ); + router.get( + { + path: ENDPOINT_SETUP_ROUTE, + validate: {}, + options: { authRequired: true }, + }, + async (context, req, res) => { + try { + const soClient = context.core.savedObjects.client; + const userResp = await getEndpointUserKey(soClient); + return res.ok({ body: userResp }); + } catch (err) { return res.internalError({ body: err }); } } @@ -110,5 +126,19 @@ async function setupEndpointUser( async function persistEndpointUser(soClient: SavedObjectsClientContract, user: EndpointUser) { const soResponse = await soClient.create(endpointUserSavedObjectType, user, {}); - console.log(soResponse); +} + +async function getEndpointUserKey(soClient: SavedObjectsClientContract) { + const resp = await soClient.find({ + type: endpointUserSavedObjectType, + search: ENDPOINT_MANAGER_USERNAME, + searchFields: ['username'], + sortField: 'created', + sortOrder: 'desc', + }); + if (resp.saved_objects.length > 0) { + return resp.saved_objects[0]; + } else { + throw new Error('No Endpoint user created.'); + } } diff --git a/x-pack/plugins/security_solution/server/lib/setup/saved_object_mappings.ts b/x-pack/plugins/security_solution/server/lib/setup/saved_object_mappings.ts index 80605a0573ad7..89b6c19c6e6e4 100644 --- a/x-pack/plugins/security_solution/server/lib/setup/saved_object_mappings.ts +++ b/x-pack/plugins/security_solution/server/lib/setup/saved_object_mappings.ts @@ -27,7 +27,7 @@ export const endpointUserSavedObjectMappings: SavedObjectsType['mappings'] = { export const type: SavedObjectsType = { name: endpointUserSavedObjectType, - hidden: true, + hidden: false, namespaceType: 'agnostic', mappings: endpointUserSavedObjectMappings, }; From 628d6a831d311930dd61fbd458436b628057c9e4 Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Fri, 12 Jun 2020 15:46:52 -0400 Subject: [PATCH 031/106] Addings tests for translating exceptions --- .../server/lib/exceptions/exceptions.test.ts | 62 ++++++++++++++++++- .../exceptions/fetch_endpoint_exceptions.ts | 2 +- 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/exceptions.test.ts b/x-pack/plugins/security_solution/server/lib/exceptions/exceptions.test.ts index 654341f5a6511..ddf1e179e245e 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/exceptions.test.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/exceptions.test.ts @@ -4,4 +4,64 @@ * you may not use this file except in compliance with the Elastic License. */ -// TODO +import { ExceptionListClient } from '../../../../lists/server/services/exception_lists/exception_list_client'; +import { listMock } from '../../../../lists/server/mocks'; +import { GetFullEndpointExceptionList } from './fetch_endpoint_exceptions'; +import { getFoundExceptionListItemSchemaMock } from '../../../../lists/common/schemas/response/found_exception_list_item_schema.mock'; +import { getExceptionListItemSchemaMock } from '../../../../lists/common/schemas/response/exception_list_item_schema.mock'; + +describe('buildEventTypeSignal', () => { + let mockExceptionClient: ExceptionListClient; + + beforeEach(() => { + jest.clearAllMocks(); + mockExceptionClient = listMock.getExceptionList(); + }); + + test('it should convert the exception lists response to the proper endpoint format', async () => { + const expectedEndpointExceptions = { + exceptions_list: [ + { + entries: [ + { + entry: { exact_caseless: 'Elastic, N.V.' }, + field: 'actingProcess.file.signer', + operator: 'included', + }, + { + entry: { exact_caseless_any: ['process', 'malware'] }, + field: 'event.category', + operator: 'included', + }, + ], + type: 'simple', + }, + ], + }; + + const first = getFoundExceptionListItemSchemaMock(); + mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); + const resp = await GetFullEndpointExceptionList(mockExceptionClient, 'linux'); + expect(resp).toEqual(expectedEndpointExceptions); + }); + + test('it should convert the exception lists response to the proper endpoint format while paging', async () => { + // The first call returns one exception + const first = getFoundExceptionListItemSchemaMock(); + + // The second call returns two exceptions + const second = getFoundExceptionListItemSchemaMock(); + second.data.push(getExceptionListItemSchemaMock()); + + // The third call returns no exceptions, paging stops + const third = getFoundExceptionListItemSchemaMock(); + third.data = []; + mockExceptionClient.findExceptionListItem = jest + .fn() + .mockReturnValueOnce(first) + .mockReturnValueOnce(second) + .mockReturnValueOnce(third); + const resp = await GetFullEndpointExceptionList(mockExceptionClient, 'linux'); + expect(resp.exceptions_list.length).toEqual(3); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/fetch_endpoint_exceptions.ts b/x-pack/plugins/security_solution/server/lib/exceptions/fetch_endpoint_exceptions.ts index 0c2eba6d515f8..638fbfbd956eb 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/fetch_endpoint_exceptions.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/fetch_endpoint_exceptions.ts @@ -68,7 +68,7 @@ export async function GetFullEndpointExceptionList( * Translates Exception list items to Exceptions the endpoint can understand * @param exc */ -function translateToEndpointExceptions(exc: FoundExceptionListItemSchema): ExceptionsList[] { +export function translateToEndpointExceptions(exc: FoundExceptionListItemSchema): ExceptionsList[] { const translated: ExceptionsList[] = []; // Transform to endpoint format exc.data.forEach((item) => { From ab7e326305790952657c44d572c2d6cc956a58ec Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Mon, 15 Jun 2020 14:01:31 -0400 Subject: [PATCH 032/106] Adding test for download API --- .../download_endpoint_exception_list.test.ts | 120 +++++++++++++++++- 1 file changed, 119 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.test.ts index 654341f5a6511..f4737055369a4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.test.ts @@ -3,5 +3,123 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ +import { + IClusterClient, + IRouter, + SavedObjectsClientContract, + IScopedClusterClient, + RouteConfig, + RequestHandler, + KibanaResponseFactory, + RequestHandlerContext, + SavedObjectsFindResponse, +} from 'kibana/server'; +import { + elasticsearchServiceMock, + savedObjectsClientMock, + httpServiceMock, + httpServerMock, +} from 'src/core/server/mocks'; +import { downloadEndpointExceptionList } from './download_endpoint_exception_list'; +import { CompressExceptionList } from '../../../exceptions/fetch_endpoint_exceptions'; -// TODO +const mockArtifactName = 'test-artifact-windows'; +const expectedEndpointExceptions = { + exceptions_list: [ + { + entries: [ + { + entry: { exact_caseless: 'Elastic, N.V.' }, + field: 'actingProcess.file.signer', + operator: 'included', + }, + { + entry: { exact_caseless_any: ['process', 'malware'] }, + field: 'event.category', + operator: 'included', + }, + ], + type: 'simple', + }, + ], +}; + +describe('test alerts route', () => { + let routerMock: jest.Mocked; + let mockClusterClient: jest.Mocked; + let mockScopedClient: jest.Mocked; + let mockSavedObjectClient: jest.Mocked; + let mockResponse: jest.Mocked; + let routeConfig: RouteConfig; + let routeHandler: RequestHandler; + + beforeEach(() => { + mockClusterClient = elasticsearchServiceMock.createClusterClient(); + mockScopedClient = elasticsearchServiceMock.createScopedClusterClient(); + mockSavedObjectClient = savedObjectsClientMock.create(); + mockResponse = httpServerMock.createResponseFactory(); + mockClusterClient.asScoped.mockReturnValue(mockScopedClient); + routerMock = httpServiceMock.createRouter(); + + downloadEndpointExceptionList(routerMock); + }); + + it('should serve the compressed artifact to download', async () => { + const mockRequest = httpServerMock.createKibanaRequest({ + path: '/api/endpoint/allowlist/download/123456', + method: 'get', + }); + + const mockCompressedArtifact = await CompressExceptionList(expectedEndpointExceptions); + + const mockArtifact = { + id: '2468', + type: 'test', + references: [], + attributes: { + name: mockArtifactName, + schemaVersion: '1.0.0', + sha256: '123456', + encoding: 'xz', + created: Date.now(), + body: mockCompressedArtifact, + size: 100, + }, + }; + + const soFindResp: SavedObjectsFindResponse = { + page: 1, + per_page: 1, + saved_objects: [mockArtifact], + total: 1, + }; + + mockSavedObjectClient.find.mockImplementationOnce(() => Promise.resolve(soFindResp)); + + [routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) => + path.startsWith('/api/endpoint/allowlist/download') + )!; + + await routeHandler( + ({ + core: { + savedObjects: { + client: mockSavedObjectClient, + }, + }, + } as unknown) as RequestHandlerContext, + mockRequest, + mockResponse + ); + + const expectedHeaders = { + 'content-encoding': 'xz', + 'content-disposition': `attachment; filename=${mockArtifactName}.xz`, + }; + + expect(mockResponse.ok).toBeCalled(); + expect(mockResponse.ok.mock.calls[0][0]?.headers).toEqual(expectedHeaders); + const compressedArtifact = mockResponse.ok.mock.calls[0][0]?.body; + expect(compressedArtifact).toEqual(mockCompressedArtifact); + }); +}); From e1cba52d172c6be57fb2d8c4b4e74eb4d0361a08 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Wed, 17 Jun 2020 00:51:39 -0400 Subject: [PATCH 033/106] Download tweaks + artifact generation fixes --- .../lib/detection_engine/exceptions/types.ts | 7 +-- .../download_endpoint_exception_list.test.ts | 15 ++--- .../download_endpoint_exception_list.ts | 31 +++++----- .../exceptions/fetch_endpoint_exceptions.ts | 61 +++++++++++-------- .../server/lib/exceptions/index.ts | 1 + .../server/lib/exceptions/task.ts | 41 +++++++------ 6 files changed, 80 insertions(+), 76 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/exceptions/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/exceptions/types.ts index 345c0a1058344..828c432f26283 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/exceptions/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/exceptions/types.ts @@ -5,11 +5,6 @@ */ export interface DownloadExceptionListRequestParams { - // TODO: this should probably be id? + artifactName: string; sha256: string; } - -export interface GetExceptionListManifestRequestParams { - manifestVersion: string; - schemaVersion: string; -} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.test.ts index f4737055369a4..b6d7517bba33f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.test.ts @@ -12,7 +12,7 @@ import { RequestHandler, KibanaResponseFactory, RequestHandlerContext, - SavedObjectsFindResponse, + SavedObject, } from 'kibana/server'; import { elasticsearchServiceMock, @@ -20,10 +20,10 @@ import { httpServiceMock, httpServerMock, } from 'src/core/server/mocks'; +import { ArtifactConstants, CompressExceptionList } from '../../../exceptions'; import { downloadEndpointExceptionList } from './download_endpoint_exception_list'; -import { CompressExceptionList } from '../../../exceptions/fetch_endpoint_exceptions'; -const mockArtifactName = 'test-artifact-windows'; +const mockArtifactName = `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-windows-1.0.0`; const expectedEndpointExceptions = { exceptions_list: [ { @@ -66,7 +66,7 @@ describe('test alerts route', () => { it('should serve the compressed artifact to download', async () => { const mockRequest = httpServerMock.createKibanaRequest({ - path: '/api/endpoint/allowlist/download/123456', + path: `/api/endpoint/allowlist/download/${mockArtifactName}/123456`, method: 'get', }); @@ -87,11 +87,8 @@ describe('test alerts route', () => { }, }; - const soFindResp: SavedObjectsFindResponse = { - page: 1, - per_page: 1, - saved_objects: [mockArtifact], - total: 1, + const soFindResp: SavedObject = { + ...mockArtifact, }; mockSavedObjectClient.find.mockImplementationOnce(() => Promise.resolve(soFindResp)); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.ts index 437366228b3e4..a47de44ecd3ec 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.ts @@ -18,13 +18,13 @@ const allowlistBaseRoute: string = '/api/endpoint/allowlist'; export function downloadEndpointExceptionList(router: IRouter) { router.get( { - path: `${allowlistBaseRoute}/download/{sha256}`, + path: `${allowlistBaseRoute}/download/{artifactName}/{sha256}`, validate: { params: buildRouteValidation( downloadExceptionListSchema ), }, - options: { authRequired: true }, + options: { tags: [] }, }, handleEndpointExceptionDownload ); @@ -34,17 +34,18 @@ export function downloadEndpointExceptionList(router: IRouter) { * Handles the GET request for downloading the allowlist */ async function handleEndpointExceptionDownload(context, req, res) { - try { - const soClient = context.core.savedObjects.client; - const resp = await soClient.find({ + // TODO: api key validation + const soClient = context.core.savedObjects.client; + + soClient + .get({ type: ArtifactConstants.SAVED_OBJECT_TYPE, - search: req.params.sha256, - searchFields: ['sha256'], - }); - if (resp.total > 0) { - const artifact = resp.saved_objects[0]; + id: `${req.params.artifactName}-${req.params.sha256}`, + }) + .then((artifact) => { const outBuffer = Buffer.from(artifact.attributes.body, 'binary'); + // TODO: validate response before returning return res.ok({ body: outBuffer, headers: { @@ -52,10 +53,8 @@ async function handleEndpointExceptionDownload(context, req, res) { 'content-disposition': `attachment; filename=${artifact.attributes.name}.xz`, }, }); - } else { - return res.notFound(); - } - } catch (err) { - return res.internalError({ body: err }); - } + }) + .catch((err) => { + return res.internalError({ body: err }); + }); } diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/fetch_endpoint_exceptions.ts b/x-pack/plugins/security_solution/server/lib/exceptions/fetch_endpoint_exceptions.ts index 638fbfbd956eb..7fce105582450 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/fetch_endpoint_exceptions.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/fetch_endpoint_exceptions.ts @@ -31,7 +31,8 @@ export interface EntryEntry { export async function GetFullEndpointExceptionList( eClient: ExceptionListClient, - os: string // TODO: make type + os: string, // TODO: make type + schemaVersion: string ): Promise { const exceptions: EndpointExceptionList = { exceptions_list: [] }; let numResponses = 0; @@ -52,7 +53,7 @@ export async function GetFullEndpointExceptionList( numResponses = response.data.length; exceptions.exceptions_list = exceptions.exceptions_list.concat( - translateToEndpointExceptions(response) + translateToEndpointExceptions(response, schemaVersion) ); page++; @@ -68,33 +69,41 @@ export async function GetFullEndpointExceptionList( * Translates Exception list items to Exceptions the endpoint can understand * @param exc */ -export function translateToEndpointExceptions(exc: FoundExceptionListItemSchema): ExceptionsList[] { +export function translateToEndpointExceptions( + exc: FoundExceptionListItemSchema, + schemaVersion: string +): ExceptionsList[] { const translated: ExceptionsList[] = []; - // Transform to endpoint format - exc.data.forEach((item) => { - const endpointItem: ExceptionsList = { - type: item.type, - entries: [], - }; - item.entries.forEach((entry) => { - // TODO case sensitive? - const e: EntryEntry = {}; - if (entry.match) { - e.exact_caseless = entry.match; - } - - if (entry.match_any) { - e.exact_caseless_any = entry.match_any; - } - - endpointItem.entries.push({ - field: entry.field, - operator: entry.operator, - entry: e, + + if (schemaVersion === '1.0.0') { + // Transform to endpoint format + exc.data.forEach((item) => { + const endpointItem: ExceptionsList = { + type: item.type, + entries: [], + }; + item.entries.forEach((entry) => { + // TODO case sensitive? + const e: EntryEntry = {}; + if (entry.match) { + e.exact_caseless = entry.match; + } + + if (entry.match_any) { + e.exact_caseless_any = entry.match_any; + } + + endpointItem.entries.push({ + field: entry.field, + operator: entry.operator, + entry: e, + }); }); + translated.push(endpointItem); }); - translated.push(endpointItem); - }); + } else { + throw new Error('unsupported schemaVersion'); + } return translated; } diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/index.ts b/x-pack/plugins/security_solution/server/lib/exceptions/index.ts index 1869540c0254c..b265314862ca1 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/index.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/index.ts @@ -4,4 +4,5 @@ * you may not use this file except in compliance with the Elastic License. */ +export * from './fetch_endpoint_exceptions'; export * from './task'; diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts index 4bfcfab9d2737..2913809a31b8c 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts @@ -68,34 +68,37 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { const exceptionListClient = context.lists.getExceptionListClient(soClient, 'kibana'); for (const os of ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS) { - const exceptions = await GetFullEndpointExceptionList(exceptionListClient, os); - const compressedExceptions: Buffer = await CompressExceptionList(exceptions); - - const sha256Hash = createHash('sha256') - .update(compressedExceptions.toString('utf8'), 'utf8') - .digest('hex'); - for (const schemaVersion of ArtifactConstants.SUPPORTED_SCHEMA_VERSIONS) { - const artifactName = `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-${os}`; - const exceptionSO = { - name: artifactName, - schemaVersion, - sha256: sha256Hash, - encoding: 'xz', - created: Date.now(), - body: compressedExceptions.toString('binary'), - size: Buffer.from(JSON.stringify(exceptions)).byteLength, - }; + const artifactName = `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-${os}-${schemaVersion}`; try { + const exceptions = await GetFullEndpointExceptionList( + exceptionListClient, + os, + schemaVersion + ); + const compressedExceptions: Buffer = await CompressExceptionList(exceptions); + + const sha256Hash = createHash('sha256') + .update(compressedExceptions.toString('utf8'), 'utf8') + .digest('hex'); + + const exceptionSO = { + name: artifactName, + schemaVersion, + sha256: sha256Hash, + encoding: 'xz', + created: Date.now(), + body: compressedExceptions.toString('binary'), + size: Buffer.from(JSON.stringify(exceptions)).byteLength, + }; + // Create the new artifact - // TODO: let id be auto-generated... revert to previous algorithm, doesn't need to be atomic. const soResponse = await soClient.create( ArtifactConstants.SAVED_OBJECT_TYPE, exceptionSO, { id: `${artifactName}-${sha256Hash}` } ); - context.logger.debug(JSON.stringify(soResponse)); // Clean up old artifacts const otherArtifacts = await soClient.find({ From 2e8d58ba75151777ae6e4101ab5d7caa88e94f60 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Wed, 17 Jun 2020 10:39:18 -0400 Subject: [PATCH 034/106] reorganize --- .../lib/detection_engine/exceptions/types.ts | 10 -- ...t_endpoint_exception_list_manifest.test.ts | 7 -- .../get_endpoint_exception_list_manifest.ts | 113 ------------------ .../routes/exceptions/validate.ts | 7 -- .../schemas/download_exception_list_schema.ts | 14 --- .../get_endpoint_exception_manifest_schema.ts | 15 --- .../exception_list_manifest_schema.ts | 25 ---- .../download_endpoint_exception_list.test.ts | 0 .../download_endpoint_exception_list.ts | 14 +-- .../schemas/download_artifact_schema.ts | 19 +++ .../schemas}/index.ts | 2 + 11 files changed, 28 insertions(+), 198 deletions(-) delete mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/exceptions/types.ts delete mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.test.ts delete mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts delete mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/validate.ts delete mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/routes/schemas/download_exception_list_schema.ts delete mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/routes/schemas/get_endpoint_exception_manifest_schema.ts delete mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/routes/schemas/response/exception_list_manifest_schema.ts rename x-pack/plugins/security_solution/server/lib/{detection_engine/routes/exceptions => exceptions/routes}/download_endpoint_exception_list.test.ts (100%) rename x-pack/plugins/security_solution/server/lib/{detection_engine/routes/exceptions => exceptions/routes}/download_endpoint_exception_list.ts (79%) create mode 100644 x-pack/plugins/security_solution/server/lib/exceptions/schemas/download_artifact_schema.ts rename x-pack/plugins/security_solution/server/lib/{detection_engine/exceptions => exceptions/schemas}/index.ts (84%) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/exceptions/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/exceptions/types.ts deleted file mode 100644 index 828c432f26283..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/exceptions/types.ts +++ /dev/null @@ -1,10 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -export interface DownloadExceptionListRequestParams { - artifactName: string; - sha256: string; -} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.test.ts deleted file mode 100644 index 654341f5a6511..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.test.ts +++ /dev/null @@ -1,7 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -// TODO diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts deleted file mode 100644 index 3851438421c7f..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest.ts +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { createHash } from 'crypto'; -import { IRouter } from '../../../../../../../../src/core/server'; -import { ArtifactConstants } from '../../../exceptions'; -import { buildRouteValidation } from '../utils'; -import { GetExceptionListManifestRequestParams } from '../../exceptions/types'; -import { getExceptionListManifestSchema } from '../schemas/get_endpoint_exception_manifest_schema'; - -const allowlistBaseRoute: string = '/api/endpoint/allowlist'; - -export interface Manifest { - schemaVersion: string; - manifestVersion: string; - artifacts: Artifacts; - signature: string; // TODO: stronger type? -} -export interface Artifacts { - [key: string]: Artifact; -} -export interface Artifact { - url: string; - sha256: string; - size: number; -} - -/** - * Registers the exception list route to enable sensors to retrieve a manifest of available lists - */ -export function getEndpointExceptionListManifest(router: IRouter) { - router.get( - { - path: `${allowlistBaseRoute}/manifest/{manifestVersion}/{schemaVersion}`, - validate: { - params: buildRouteValidation( - getExceptionListManifestSchema - ), - }, - options: { authRequired: true }, - }, - handleAllowlistManifest - ); -} - -/** - * Handles the GET request for whitelist manifest - */ -async function handleAllowlistManifest(context, req, res) { - try { - if (req.params.manifestVersion !== '1.0.0') { - return res.badRequest('invalid manifest version'); - } - - const resp = await getAllowlistManifest(context, '1.0.0', req.params.schemaVersion); - if (resp.saved_objects.length === 0) { - return res.notFound({ body: `No manifest found for version ${req.params.schemaVersion}` }); - } - - const manifestResp: Manifest = { - schemaVersion: req.params.schemaVersion, - manifestVersion: '1.0.0', // TODO hardcode? - artifacts: {}, - }; - - // Handle a HEAD request - if (req.route.method === 'head') { - const manifestHash = createHash('sha256') - .update(JSON.stringify(manifestResp), 'utf8') - .digest('hex'); - - return res.ok({ headers: { ETag: manifestHash } }); - } - - // transform and validate response - for (const manifest of resp.saved_objects) { - manifestResp.artifacts[manifest.attributes.name] = { - url: `${allowlistBaseRoute}/download/${manifest.attributes.sha256}`, - sha256: manifest.attributes.sha256, - size: manifest.attributes.size, - }; - } - - // TODO: digitally sign - manifestResp.signature = 'abcd'; - - return res.ok({ body: manifestResp }); - } catch (err) { - return res.internalError({ body: err }); - } -} - -/** - * Creates the manifest for the whitelist - */ -async function getAllowlistManifest(ctx, manifestVersion: string, schemaVersion: string) { - const soClient = ctx.core.savedObjects.client; - - // TODO page - const manifestResp = soClient.find({ - type: ArtifactConstants.SAVED_OBJECT_TYPE, - fields: ['name', 'schemaVersion', 'sha256', 'encoding', 'size', 'created'], - search: schemaVersion, - searchFields: ['schemaVersion'], - sortField: 'created', - sortOrder: 'asc', - }); - - return manifestResp; -} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/validate.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/validate.ts deleted file mode 100644 index 654341f5a6511..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/validate.ts +++ /dev/null @@ -1,7 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -// TODO diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/schemas/download_exception_list_schema.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/schemas/download_exception_list_schema.ts deleted file mode 100644 index 557f4e847e699..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/schemas/download_exception_list_schema.ts +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import Joi from 'joi'; - -// TODO: convert to io-tsk -// import * as t from 'io-ts'; - -export const downloadExceptionListSchema = Joi.object({ - sha256: Joi.string(), -}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/schemas/get_endpoint_exception_manifest_schema.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/schemas/get_endpoint_exception_manifest_schema.ts deleted file mode 100644 index b73af1c1ad357..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/schemas/get_endpoint_exception_manifest_schema.ts +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import Joi from 'joi'; - -// TODO: convert to io-tsk -// import * as t from 'io-ts'; - -export const getExceptionListManifestSchema = Joi.object({ - manifestVersion: Joi.string(), - schemaVersion: Joi.string(), -}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/schemas/response/exception_list_manifest_schema.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/schemas/response/exception_list_manifest_schema.ts deleted file mode 100644 index 973f63699fa96..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/schemas/response/exception_list_manifest_schema.ts +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import * as t from 'io-ts'; - -const exceptionListManifestEntrySchema = t.exact( - t.type({ - id: t.string, - name: t.string, - schemaVersion: t.string, - sha256: t.string, - created: t.number, - }) -); - -export const exceptionListManifestSchema = t.exact( - t.type({ - artifacts: t.array(exceptionListManifestEntrySchema), - }) -); - -export type ExceptionListManifestSchema = t.TypeOf; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.test.ts b/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.test.ts similarity index 100% rename from x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.test.ts rename to x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.test.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.ts b/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.ts similarity index 79% rename from x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.ts rename to x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.ts index a47de44ecd3ec..480a1e9c316ca 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/exceptions/download_endpoint_exception_list.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.ts @@ -5,10 +5,9 @@ */ import { IRouter } from '../../../../../../../../src/core/server'; -import { ArtifactConstants } from '../../../exceptions'; -import { DownloadExceptionListRequestParams } from '../../exceptions/types'; -import { buildRouteValidation } from '../utils'; -import { downloadExceptionListSchema } from '../schemas/download_exception_list_schema'; +import { buildRouteValidation } from '../../../utils'; +import { ArtifactConstants } from '../task'; +import { DownloadArtifactReqParamsSchema, downloadArtifactReqParamsSchema } from '../schemas'; const allowlistBaseRoute: string = '/api/endpoint/allowlist'; @@ -20,9 +19,10 @@ export function downloadEndpointExceptionList(router: IRouter) { { path: `${allowlistBaseRoute}/download/{artifactName}/{sha256}`, validate: { - params: buildRouteValidation( - downloadExceptionListSchema - ), + params: buildRouteValidation< + typeof downloadArtifactReqParamsSchema, + DownloadArtifactReqParamsSchema + >(downloadArtifactReqParamsSchema), }, options: { tags: [] }, }, diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/schemas/download_artifact_schema.ts b/x-pack/plugins/security_solution/server/lib/exceptions/schemas/download_artifact_schema.ts new file mode 100644 index 0000000000000..16dde0d76b845 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/exceptions/schemas/download_artifact_schema.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; + +export const artifactName = t.string; +export const sha256 = t.string; + +export const downloadArtifactReqParamsSchema = t.exact( + t.type({ + artifactName, + sha256, + }) +); + +export type DownloadArtifactReqParamsSchema = t.TypeOf; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/exceptions/index.ts b/x-pack/plugins/security_solution/server/lib/exceptions/schemas/index.ts similarity index 84% rename from x-pack/plugins/security_solution/server/lib/detection_engine/exceptions/index.ts rename to x-pack/plugins/security_solution/server/lib/exceptions/schemas/index.ts index 41bc2aa258807..13e4165eb5f16 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/exceptions/index.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/schemas/index.ts @@ -3,3 +3,5 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ + +export * from './download_artifact_schema'; From 9cb08f1a23fa170a3dff4b0458228a480245f972 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Wed, 17 Jun 2020 10:53:51 -0400 Subject: [PATCH 035/106] fix imports --- .../routes/download_endpoint_exception_list.test.ts | 4 ++-- .../exceptions/routes/download_endpoint_exception_list.ts | 4 ++-- .../server/lib/exceptions/routes/index.ts | 7 +++++++ x-pack/plugins/security_solution/server/routes/index.ts | 6 ++---- 4 files changed, 13 insertions(+), 8 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/exceptions/routes/index.ts diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.test.ts b/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.test.ts index b6d7517bba33f..0a75b1af5c33a 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.test.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.test.ts @@ -21,7 +21,7 @@ import { httpServerMock, } from 'src/core/server/mocks'; import { ArtifactConstants, CompressExceptionList } from '../../../exceptions'; -import { downloadEndpointExceptionList } from './download_endpoint_exception_list'; +import { downloadEndpointExceptionListRoute } from './download_endpoint_exception_list'; const mockArtifactName = `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-windows-1.0.0`; const expectedEndpointExceptions = { @@ -61,7 +61,7 @@ describe('test alerts route', () => { mockClusterClient.asScoped.mockReturnValue(mockScopedClient); routerMock = httpServiceMock.createRouter(); - downloadEndpointExceptionList(routerMock); + downloadEndpointExceptionListRoute(routerMock); }); it('should serve the compressed artifact to download', async () => { diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.ts b/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.ts index 480a1e9c316ca..aa77a2f7d107e 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.ts @@ -5,7 +5,7 @@ */ import { IRouter } from '../../../../../../../../src/core/server'; -import { buildRouteValidation } from '../../../utils'; +import { buildRouteValidation } from '../../../utils/build_validation/route_validation'; import { ArtifactConstants } from '../task'; import { DownloadArtifactReqParamsSchema, downloadArtifactReqParamsSchema } from '../schemas'; @@ -14,7 +14,7 @@ const allowlistBaseRoute: string = '/api/endpoint/allowlist'; /** * Registers the exception list route to enable sensors to download a compressed allowlist */ -export function downloadEndpointExceptionList(router: IRouter) { +export function downloadEndpointExceptionListRoute(router: IRouter) { router.get( { path: `${allowlistBaseRoute}/download/{artifactName}/{sha256}`, diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/routes/index.ts b/x-pack/plugins/security_solution/server/lib/exceptions/routes/index.ts new file mode 100644 index 0000000000000..741702f37d047 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/exceptions/routes/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './download_endpoint_exception_list'; diff --git a/x-pack/plugins/security_solution/server/routes/index.ts b/x-pack/plugins/security_solution/server/routes/index.ts index 9d02706d4feec..702d89f2951ba 100644 --- a/x-pack/plugins/security_solution/server/routes/index.ts +++ b/x-pack/plugins/security_solution/server/routes/index.ts @@ -28,8 +28,7 @@ import { importRulesRoute } from '../lib/detection_engine/routes/rules/import_ru import { exportRulesRoute } from '../lib/detection_engine/routes/rules/export_rules_route'; import { findRulesStatusesRoute } from '../lib/detection_engine/routes/rules/find_rules_status_route'; import { getPrepackagedRulesStatusRoute } from '../lib/detection_engine/routes/rules/get_prepackaged_rules_status_route'; -import { downloadEndpointExceptionList } from '../lib/detection_engine/routes/exceptions/download_endpoint_exception_list'; -import { getEndpointExceptionListManifest } from '../lib/detection_engine/routes/exceptions/get_endpoint_exception_list_manifest'; +import { downloadEndpointExceptionListRoute } from '../lib/exceptions/routes/download_endpoint_exception_list'; import { importTimelinesRoute } from '../lib/timeline/routes/import_timelines_route'; import { exportTimelinesRoute } from '../lib/timeline/routes/export_timelines_route'; import { createTimelinesRoute } from '../lib/timeline/routes/create_timelines_route'; @@ -68,8 +67,7 @@ export const initRoutes = ( importRulesRoute(router, config, ml); exportRulesRoute(router, config); - downloadEndpointExceptionList(router); - getEndpointExceptionListManifest(router); + downloadEndpointExceptionListRoute(router); importTimelinesRoute(router, config, security); exportTimelinesRoute(router, config); From 978df5a15a8773b151f49a18cbc11c57b2fac1ac Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Wed, 17 Jun 2020 11:42:39 -0400 Subject: [PATCH 036/106] Fixing test --- .../server/lib/exceptions/exceptions.test.ts | 4 ++-- .../routes/download_endpoint_exception_list.test.ts | 5 +++-- .../exceptions/routes/download_endpoint_exception_list.ts | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/exceptions.test.ts b/x-pack/plugins/security_solution/server/lib/exceptions/exceptions.test.ts index ddf1e179e245e..7b60f1dcc8744 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/exceptions.test.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/exceptions.test.ts @@ -41,7 +41,7 @@ describe('buildEventTypeSignal', () => { const first = getFoundExceptionListItemSchemaMock(); mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); - const resp = await GetFullEndpointExceptionList(mockExceptionClient, 'linux'); + const resp = await GetFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); expect(resp).toEqual(expectedEndpointExceptions); }); @@ -61,7 +61,7 @@ describe('buildEventTypeSignal', () => { .mockReturnValueOnce(first) .mockReturnValueOnce(second) .mockReturnValueOnce(third); - const resp = await GetFullEndpointExceptionList(mockExceptionClient, 'linux'); + const resp = await GetFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); expect(resp.exceptions_list.length).toEqual(3); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.test.ts b/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.test.ts index 0a75b1af5c33a..f305c3dc2ccec 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.test.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.test.ts @@ -20,7 +20,8 @@ import { httpServiceMock, httpServerMock, } from 'src/core/server/mocks'; -import { ArtifactConstants, CompressExceptionList } from '../../../exceptions'; +import { CompressExceptionList } from '../fetch_endpoint_exceptions'; +import { ArtifactConstants } from '../task'; import { downloadEndpointExceptionListRoute } from './download_endpoint_exception_list'; const mockArtifactName = `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-windows-1.0.0`; @@ -91,7 +92,7 @@ describe('test alerts route', () => { ...mockArtifact, }; - mockSavedObjectClient.find.mockImplementationOnce(() => Promise.resolve(soFindResp)); + mockSavedObjectClient.get.mockImplementationOnce(() => Promise.resolve(soFindResp)); [routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) => path.startsWith('/api/endpoint/allowlist/download') diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.ts b/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.ts index aa77a2f7d107e..0487580333962 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { IRouter } from '../../../../../../../../src/core/server'; +import { IRouter } from 'src/core/server'; import { buildRouteValidation } from '../../../utils/build_validation/route_validation'; import { ArtifactConstants } from '../task'; import { DownloadArtifactReqParamsSchema, downloadArtifactReqParamsSchema } from '../schemas'; From 288407e3fe22ede42c215dc766107e786297e186 Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Wed, 17 Jun 2020 13:12:16 -0400 Subject: [PATCH 037/106] Changes id of SO --- .../download_endpoint_exception_list.test.ts | 49 +++++++++++++++++++ .../download_endpoint_exception_list.ts | 13 +++-- .../server/lib/exceptions/task.ts | 40 ++++++++------- 3 files changed, 78 insertions(+), 24 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.test.ts b/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.test.ts index f305c3dc2ccec..d5c31600e0a8b 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.test.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.test.ts @@ -69,6 +69,7 @@ describe('test alerts route', () => { const mockRequest = httpServerMock.createKibanaRequest({ path: `/api/endpoint/allowlist/download/${mockArtifactName}/123456`, method: 'get', + params: { sha256: '123456' }, }); const mockCompressedArtifact = await CompressExceptionList(expectedEndpointExceptions); @@ -120,4 +121,52 @@ describe('test alerts route', () => { const compressedArtifact = mockResponse.ok.mock.calls[0][0]?.body; expect(compressedArtifact).toEqual(mockCompressedArtifact); }); + + it('should handle a sha256 mismatch', async () => { + const mockRequest = httpServerMock.createKibanaRequest({ + path: `/api/endpoint/allowlist/download/${mockArtifactName}/123456`, + method: 'get', + params: { sha256: '789' }, + }); + + const mockCompressedArtifact = await CompressExceptionList(expectedEndpointExceptions); + + const mockArtifact = { + id: '2468', + type: 'test', + references: [], + attributes: { + name: mockArtifactName, + schemaVersion: '1.0.0', + sha256: '123456', + encoding: 'xz', + created: Date.now(), + body: mockCompressedArtifact, + size: 100, + }, + }; + + const soFindResp: SavedObject = { + ...mockArtifact, + }; + + mockSavedObjectClient.get.mockImplementationOnce(() => Promise.resolve(soFindResp)); + + [routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) => + path.startsWith('/api/endpoint/allowlist/download') + )!; + + await routeHandler( + ({ + core: { + savedObjects: { + client: mockSavedObjectClient, + }, + }, + } as unknown) as RequestHandlerContext, + mockRequest, + mockResponse + ); + expect(mockResponse.notFound).toBeCalled(); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.ts b/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.ts index 0487580333962..052d67a8f2e99 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.ts @@ -37,14 +37,17 @@ async function handleEndpointExceptionDownload(context, req, res) { // TODO: api key validation const soClient = context.core.savedObjects.client; - soClient - .get({ - type: ArtifactConstants.SAVED_OBJECT_TYPE, - id: `${req.params.artifactName}-${req.params.sha256}`, - }) + return soClient + .get(ArtifactConstants.SAVED_OBJECT_TYPE, `${req.params.artifactName}`) .then((artifact) => { const outBuffer = Buffer.from(artifact.attributes.body, 'binary'); + if (artifact.attributes.sha256 !== req.params.sha256) { + return res.notFound( + `No artifact matching sha256: ${req.params.sha256} for type ${req.params.artifactName}` + ); + } + // TODO: validate response before returning return res.ok({ body: outBuffer, diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts index 2913809a31b8c..5aacf0cc0aa2b 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts @@ -97,27 +97,29 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { const soResponse = await soClient.create( ArtifactConstants.SAVED_OBJECT_TYPE, exceptionSO, - { id: `${artifactName}-${sha256Hash}` } + { id: `${artifactName}`, overwrite: true } ); - // Clean up old artifacts - const otherArtifacts = await soClient.find({ - type: ArtifactConstants.SAVED_OBJECT_TYPE, - search: artifactName, - searchFields: ['name'], - sortField: 'created', - sortOrder: 'desc', - }); - - // Remove all but the latest artifact - const toDelete = otherArtifacts.saved_objects.slice( - 1, - otherArtifacts.saved_objects.length - ); - for (const delObj of toDelete) { - context.logger.debug(`REMOVING ${delObj.id}`); - await soClient.delete(ArtifactConstants.SAVED_OBJECT_TYPE, delObj.id); - } + context.logger.debug(`Current artifact ${artifactName} with hash ${sha256Hash}`); + + // // Clean up old artifacts + // const otherArtifacts = await soClient.find({ + // type: ArtifactConstants.SAVED_OBJECT_TYPE, + // search: artifactName, + // searchFields: ['name'], + // sortField: 'created', + // sortOrder: 'desc', + // }); + + // // Remove all but the latest artifact + // const toDelete = otherArtifacts.saved_objects.slice( + // 1, + // otherArtifacts.saved_objects.length + // ); + // for (const delObj of toDelete) { + // context.logger.debug(`REMOVING ${delObj.id}`); + // await soClient.delete(ArtifactConstants.SAVED_OBJECT_TYPE, delObj.id); + // } } catch (error) { if (error.statusCode === 409) { context.logger.debug(`No update to Endpoint Exceptions (${artifactName}), skipping.`); From 2459a3e607bc543281ca2c10aa696594099c1d94 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Wed, 17 Jun 2020 13:32:13 -0400 Subject: [PATCH 038/106] integration tests setup --- .../apis/security_solution/exceptions.ts | 11 +---- .../api_feature/exception_list/data.json | 45 +++++++++++++++++++ 2 files changed, 47 insertions(+), 9 deletions(-) create mode 100644 x-pack/test/functional/es_archives/security_solution/exceptions/api_feature/exception_list/data.json diff --git a/x-pack/test/api_integration/apis/security_solution/exceptions.ts b/x-pack/test/api_integration/apis/security_solution/exceptions.ts index 30aca7df83e58..20568013f5054 100644 --- a/x-pack/test/api_integration/apis/security_solution/exceptions.ts +++ b/x-pack/test/api_integration/apis/security_solution/exceptions.ts @@ -11,16 +11,9 @@ import { FtrProviderContext } from '../../ftr_provider_context'; export default function ({ getService }: FtrProviderContext) { const esArchiver = getService('esArchiver'); - describe('artifact manifest', () => { - before(() => esArchiver.load('TODO')); - after(() => esArchiver.unload('TODO')); - - it('Do a manifest test', () => {}); - }); - describe('artifact download', () => { - before(() => esArchiver.load('TODO')); - after(() => esArchiver.unload('TODO')); + before(() => esArchiver.load('security_solution/exceptions/api_feature/exception_list')); + after(() => esArchiver.unload('security_solution/exceptions/api_feature/exception_list')); it('Do a download test', () => {}); }); diff --git a/x-pack/test/functional/es_archives/security_solution/exceptions/api_feature/exception_list/data.json b/x-pack/test/functional/es_archives/security_solution/exceptions/api_feature/exception_list/data.json new file mode 100644 index 0000000000000..8f03dead04798 --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/exceptions/api_feature/exception_list/data.json @@ -0,0 +1,45 @@ +{ + "type": "doc", + "value": { + "id": "securitySolution-exceptions-artifact:endpoint-allowlist-linux-1.0.0-0fa0ba277627f7cc116e198bac3e524e9a8befdca1591a1dd4830dc7952b2103", + "index": ".kibana_1", + "source": { + "references": [ + ], + "securitySolution-exceptions-artifact": { + "body": "ý7zXZ\u0000\u0000\u0001i\"Þ6\u0002\u0000!\u0001\u0016\u0000\u0000\u0000t/å£à\u0000ÿ\u0000©]\u0000=ˆˆ§Ã{\u0000Š&W“{Wï¶=\f‚‚söÊ\u001eæõ\u0014WI1dƒÛtÂ\u00185ý\u0012[.?=ÃB#­òW€ì¦\u0006)ä˜ÕVkÌâ\b´Ï; ãÍ¥Á’‚Ê“èÕ£ÅuŸÜ0ê\t\u0001\u0002ÿmQ\u001b«Ï_°+Ræ­H¦,x¶½OÂ\u0004P\u001f*YP#ƒÐ.0\r‹\u0002\u0000\u0000\u0000\u0000\u0001YZ", + "created": 1592414128607, + "encoding": "xz", + "name": "endpoint-allowlist-linux-1.0.0", + "schemaVersion": "1.0.0", + "sha256": "0fa0ba277627f7cc116e198bac3e524e9a8befdca1591a1dd4830dc7952b2103", + "size": 256 + }, + "type": "securitySolution-exceptions-artifact", + "updated_at": "2020-06-17T17:15:28.607Z" + } + } +} + +{ + "type": "doc", + "value": { + "id": "securitySolution-exceptions-artifact:endpoint-allowlist-windows-1.0.0-1825fb19fcc6dc391cae0bc4a2e96dd7f728a0c3ae9e1469251ada67f9e1b975", + "index": ".kibana_1", + "source": { + "references": [ + ], + "securitySolution-exceptions-artifact": { + "body": "ý7zXZ\u0000\u0000\u0001i\"Þ6\u0002\u0000!\u0001\u0016\u0000\u0000\u0000t/å£\u0001\u0000\u0015{\"exceptions_list\":[]}\u0000\u0000\u000052¤\u0000\u0001*\u0016RÌ9»B™\r\u0001\u0000\u0000\u0000\u0000\u0001YZ", + "created": 1592414129763, + "encoding": "xz", + "name": "endpoint-allowlist-windows-1.0.0", + "schemaVersion": "1.0.0", + "sha256": "1825fb19fcc6dc391cae0bc4a2e96dd7f728a0c3ae9e1469251ada67f9e1b975", + "size": 22 + }, + "type": "securitySolution-exceptions-artifact", + "updated_at": "2020-06-17T17:15:29.763Z" + } + } +} From 83ab60928c7fa1b231b04fc1b9d00fd802df5fee Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Wed, 17 Jun 2020 13:59:43 -0400 Subject: [PATCH 039/106] Add first integration tests --- .../apis/security_solution/exceptions.ts | 17 ++++++++++++++++- .../api_feature/exception_list/data.json | 4 ++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/x-pack/test/api_integration/apis/security_solution/exceptions.ts b/x-pack/test/api_integration/apis/security_solution/exceptions.ts index 20568013f5054..79b05ce1c6eda 100644 --- a/x-pack/test/api_integration/apis/security_solution/exceptions.ts +++ b/x-pack/test/api_integration/apis/security_solution/exceptions.ts @@ -10,11 +10,26 @@ import { FtrProviderContext } from '../../ftr_provider_context'; export default function ({ getService }: FtrProviderContext) { const esArchiver = getService('esArchiver'); + const supertest = getService('supertest'); describe('artifact download', () => { before(() => esArchiver.load('security_solution/exceptions/api_feature/exception_list')); after(() => esArchiver.unload('security_solution/exceptions/api_feature/exception_list')); - it('Do a download test', () => {}); + it('should fail to find artifact with invalid hash', async () => { + const { body } = await supertest + .get('/api/endpoint/allowlist/download/endpoint-allowlist-windows-1.0.0/abcd') + .send() + .expect(404); + }); + + it('should download an artifact with correct hash', async () => { + const { body } = await supertest + .get( + '/api/endpoint/allowlist/download/endpoint-allowlist-windows-1.0.0/1825fb19fcc6dc391cae0bc4a2e96dd7f728a0c3ae9e1469251ada67f9e1b975' + ) + .send() + .expect(200); + }); }); } diff --git a/x-pack/test/functional/es_archives/security_solution/exceptions/api_feature/exception_list/data.json b/x-pack/test/functional/es_archives/security_solution/exceptions/api_feature/exception_list/data.json index 8f03dead04798..3991e1286b9f0 100644 --- a/x-pack/test/functional/es_archives/security_solution/exceptions/api_feature/exception_list/data.json +++ b/x-pack/test/functional/es_archives/security_solution/exceptions/api_feature/exception_list/data.json @@ -1,7 +1,7 @@ { "type": "doc", "value": { - "id": "securitySolution-exceptions-artifact:endpoint-allowlist-linux-1.0.0-0fa0ba277627f7cc116e198bac3e524e9a8befdca1591a1dd4830dc7952b2103", + "id": "securitySolution-exceptions-artifact:endpoint-allowlist-linux-1.0.0", "index": ".kibana_1", "source": { "references": [ @@ -24,7 +24,7 @@ { "type": "doc", "value": { - "id": "securitySolution-exceptions-artifact:endpoint-allowlist-windows-1.0.0-1825fb19fcc6dc391cae0bc4a2e96dd7f728a0c3ae9e1469251ada67f9e1b975", + "id": "securitySolution-exceptions-artifact:endpoint-allowlist-windows-1.0.0", "index": ".kibana_1", "source": { "references": [ From 067201df64a3165c36cea89c4228bcd721dd6644 Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Wed, 17 Jun 2020 15:45:22 -0400 Subject: [PATCH 040/106] Cache layer --- .../server/lib/exceptions/cache.ts | 39 ++++++++++ .../server/lib/exceptions/index.ts | 1 + .../download_endpoint_exception_list.test.ts | 38 +++++++++- .../download_endpoint_exception_list.ts | 71 +++++++++++-------- .../server/lib/exceptions/task.ts | 5 ++ .../security_solution/server/plugin.ts | 7 +- .../security_solution/server/routes/index.ts | 3 - 7 files changed, 128 insertions(+), 36 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/exceptions/cache.ts diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/cache.ts b/x-pack/plugins/security_solution/server/lib/exceptions/cache.ts new file mode 100644 index 0000000000000..8f84fac37b0b6 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/exceptions/cache.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export class ExceptionsCache { + private cache: Map; + private requested: string[]; + private ttl: number; + + constructor(ttl: number) { + this.cache = new Map(); + this.requested = []; + this.ttl = ttl; + this.startClean(); + } + + private startClean() { + setInterval(() => this.clean, this.ttl); + } + + private clean() { + for (const v of this.cache.values()) { + if (!this.requested.includes(v)) { + this.cache.delete(v); + } + } + } + + set(id: string, body: string) { + this.cache.set(id, body); + } + + get(id: string): string | undefined { + this.requested.push(id); + return this.cache.get(id); + } +} diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/index.ts b/x-pack/plugins/security_solution/server/lib/exceptions/index.ts index b265314862ca1..f27f511a198ad 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/index.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/index.ts @@ -6,3 +6,4 @@ export * from './fetch_endpoint_exceptions'; export * from './task'; +export * from './cache'; diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.test.ts b/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.test.ts index d5c31600e0a8b..70d23eddf7a67 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.test.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.test.ts @@ -23,6 +23,7 @@ import { import { CompressExceptionList } from '../fetch_endpoint_exceptions'; import { ArtifactConstants } from '../task'; import { downloadEndpointExceptionListRoute } from './download_endpoint_exception_list'; +import { ExceptionsCache } from '../cache'; const mockArtifactName = `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-windows-1.0.0`; const expectedEndpointExceptions = { @@ -53,6 +54,7 @@ describe('test alerts route', () => { let mockResponse: jest.Mocked; let routeConfig: RouteConfig; let routeHandler: RequestHandler; + let cache: ExceptionsCache; beforeEach(() => { mockClusterClient = elasticsearchServiceMock.createClusterClient(); @@ -61,8 +63,9 @@ describe('test alerts route', () => { mockResponse = httpServerMock.createResponseFactory(); mockClusterClient.asScoped.mockReturnValue(mockScopedClient); routerMock = httpServiceMock.createRouter(); + cache = new ExceptionsCache(10000); // TODO - downloadEndpointExceptionListRoute(routerMock); + downloadEndpointExceptionListRoute(routerMock, cache); }); it('should serve the compressed artifact to download', async () => { @@ -169,4 +172,37 @@ describe('test alerts route', () => { ); expect(mockResponse.notFound).toBeCalled(); }); + + it('should utilize the cache', async () => { + const mockSha = '123456789'; + const mockRequest = httpServerMock.createKibanaRequest({ + path: `/api/endpoint/allowlist/download/${mockArtifactName}/${mockSha}`, + method: 'get', + params: { sha256: mockSha, artifactName: mockArtifactName }, + }); + + // Add to the download cache + const mockCompressedArtifact = await CompressExceptionList(expectedEndpointExceptions); + const cacheKey = `${mockArtifactName}-${mockSha}`; + cache.set(cacheKey, mockCompressedArtifact.toString('binary')); + + [routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) => + path.startsWith('/api/endpoint/allowlist/download') + )!; + + await routeHandler( + ({ + core: { + savedObjects: { + client: mockSavedObjectClient, + }, + }, + } as unknown) as RequestHandlerContext, + mockRequest, + mockResponse + ); + expect(mockResponse.ok).toBeCalled(); + // The saved objects client should be bypassed as the cache will contain the download + expect(mockSavedObjectClient.get.mock.calls.length).toEqual(0); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.ts b/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.ts index 052d67a8f2e99..e8720582ab490 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.ts @@ -7,6 +7,7 @@ import { IRouter } from 'src/core/server'; import { buildRouteValidation } from '../../../utils/build_validation/route_validation'; import { ArtifactConstants } from '../task'; +import { ExceptionsCache } from '../cache'; import { DownloadArtifactReqParamsSchema, downloadArtifactReqParamsSchema } from '../schemas'; const allowlistBaseRoute: string = '/api/endpoint/allowlist'; @@ -14,7 +15,7 @@ const allowlistBaseRoute: string = '/api/endpoint/allowlist'; /** * Registers the exception list route to enable sensors to download a compressed allowlist */ -export function downloadEndpointExceptionListRoute(router: IRouter) { +export function downloadEndpointExceptionListRoute(router: IRouter, cache: ExceptionsCache) { router.get( { path: `${allowlistBaseRoute}/download/{artifactName}/{sha256}`, @@ -26,38 +27,46 @@ export function downloadEndpointExceptionListRoute(router: IRouter) { }, options: { tags: [] }, }, - handleEndpointExceptionDownload - ); -} + async (context, req, res) => { + const soClient = context.core.savedObjects.client; -/** - * Handles the GET request for downloading the allowlist - */ -async function handleEndpointExceptionDownload(context, req, res) { - // TODO: api key validation - const soClient = context.core.savedObjects.client; + const cacheKey = `${req.params.artifactName}-${req.params.sha256}`; + const cacheResp = cache.get(cacheKey); + if (cacheResp) { + // CACHE HIT + return res.ok({ + body: Buffer.from(cacheResp, 'binary'), + headers: { + 'content-encoding': 'xz', + 'content-disposition': `attachment; filename=${req.params.artifactName}.xz`, + }, + }); + } else { + // CACHE MISS + return soClient + .get(ArtifactConstants.SAVED_OBJECT_TYPE, `${req.params.artifactName}`) + .then((artifact) => { + const outBuffer = Buffer.from(artifact.attributes.body, 'binary'); - return soClient - .get(ArtifactConstants.SAVED_OBJECT_TYPE, `${req.params.artifactName}`) - .then((artifact) => { - const outBuffer = Buffer.from(artifact.attributes.body, 'binary'); + if (artifact.attributes.sha256 !== req.params.sha256) { + return res.notFound({ + body: `No artifact matching sha256: ${req.params.sha256} for type ${req.params.artifactName}`, + }); + } - if (artifact.attributes.sha256 !== req.params.sha256) { - return res.notFound( - `No artifact matching sha256: ${req.params.sha256} for type ${req.params.artifactName}` - ); + // TODO: factor this response out here and above + return res.ok({ + body: outBuffer, + headers: { + 'content-encoding': 'xz', + 'content-disposition': `attachment; filename=${artifact.attributes.name}.xz`, + }, + }); + }) + .catch((err) => { + return res.internalError({ body: err }); + }); } - - // TODO: validate response before returning - return res.ok({ - body: outBuffer, - headers: { - 'content-encoding': 'xz', - 'content-disposition': `attachment; filename=${artifact.attributes.name}.xz`, - }, - }); - }) - .catch((err) => { - return res.internalError({ body: err }); - }); + } + ); } diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts index 5aacf0cc0aa2b..2ffdab9399917 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts @@ -14,6 +14,7 @@ import { import { ListPluginSetup } from '../../../../lists/server'; import { ConfigType } from '../../config'; import { GetFullEndpointExceptionList, CompressExceptionList } from './fetch_endpoint_exceptions'; +import { ExceptionsCache } from './cache'; const PackagerTaskConstants = { TIMEOUT: '1m', @@ -42,6 +43,7 @@ interface PackagerTaskContext { logger: Logger; taskManager: TaskManagerSetupContract; lists: ListPluginSetup; + cache: ExceptionsCache; } interface PackagerTaskRunnerContext { @@ -100,6 +102,9 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { { id: `${artifactName}`, overwrite: true } ); + const cacheKey = `${artifactName}-${sha256Hash}`; + context.cache.set(cacheKey, compressedExceptions.toString('binary')); + context.logger.debug(`Current artifact ${artifactName} with hash ${sha256Hash}`); // // Clean up old artifacts diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index f730f15f42441..16cb77a6f21cb 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -33,7 +33,7 @@ import { signalRulesAlertType } from './lib/detection_engine/signals/signal_rule import { rulesNotificationAlertType } from './lib/detection_engine/notifications/rules_notification_alert_type'; import { isNotificationAlertExecutor } from './lib/detection_engine/notifications/types'; import { hasListsFeature, listsEnvFeatureFlagName } from './lib/detection_engine/feature_flags'; -import { PackagerTask, setupPackagerTask } from './lib/exceptions'; +import { PackagerTask, setupPackagerTask, ExceptionsCache } from './lib/exceptions'; import { initSavedObjects, savedObjectTypes } from './saved_objects'; import { AppClientFactory } from './client'; import { createConfig$, ConfigType } from './config'; @@ -45,6 +45,7 @@ import { registerAlertRoutes } from './endpoint/alerts/routes'; import { registerPolicyRoutes } from './endpoint/routes/policy'; import { EndpointAppContextService } from './endpoint/endpoint_app_context_services'; import { EndpointAppContext } from './endpoint/types'; +import { downloadEndpointExceptionListRoute } from './lib/exceptions/routes/download_endpoint_exception_list'; export interface SetupPlugins { alerts: AlertingSetup; @@ -75,12 +76,14 @@ export class Plugin implements IPlugin Date: Wed, 17 Jun 2020 15:48:44 -0400 Subject: [PATCH 041/106] more schema validation --- .../download_endpoint_exception_list.ts | 28 +++++++++++++------ .../server/lib/exceptions/saved_object.ts | 15 ---------- .../lib/exceptions/schemas/artifact_schema.ts | 22 +++++++++++++++ .../server/lib/exceptions/schemas/common.ts | 25 +++++++++++++++++ .../server/lib/exceptions/schemas/index.ts | 5 +++- .../{ => request}/download_artifact_schema.ts | 10 +++---- .../lib/exceptions/schemas/request/index.ts | 7 +++++ .../response/download_artifact_schema.ts | 25 +++++++++++++++++ .../lib/exceptions/schemas/response/index.ts | 7 +++++ .../server/lib/exceptions/task.ts | 22 ++------------- .../server/lib/exceptions/types.ts | 19 ------------- 11 files changed, 117 insertions(+), 68 deletions(-) delete mode 100644 x-pack/plugins/security_solution/server/lib/exceptions/saved_object.ts create mode 100644 x-pack/plugins/security_solution/server/lib/exceptions/schemas/artifact_schema.ts create mode 100644 x-pack/plugins/security_solution/server/lib/exceptions/schemas/common.ts rename x-pack/plugins/security_solution/server/lib/exceptions/schemas/{ => request}/download_artifact_schema.ts (59%) create mode 100644 x-pack/plugins/security_solution/server/lib/exceptions/schemas/request/index.ts create mode 100644 x-pack/plugins/security_solution/server/lib/exceptions/schemas/response/download_artifact_schema.ts create mode 100644 x-pack/plugins/security_solution/server/lib/exceptions/schemas/response/index.ts delete mode 100644 x-pack/plugins/security_solution/server/lib/exceptions/types.ts diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.ts b/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.ts index 052d67a8f2e99..1e898cebcd8c0 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.ts @@ -5,9 +5,15 @@ */ import { IRouter } from 'src/core/server'; +import { validate } from '../../../../common/validate'; import { buildRouteValidation } from '../../../utils/build_validation/route_validation'; import { ArtifactConstants } from '../task'; -import { DownloadArtifactReqParamsSchema, downloadArtifactReqParamsSchema } from '../schemas'; +import { + ArtifactDownloadSchema, + DownloadArtifactRequestParamsSchema, + downloadArtifactRequestParamsSchema, + downloadArtifactResponseSchema, +} from '../schemas'; const allowlistBaseRoute: string = '/api/endpoint/allowlist'; @@ -20,9 +26,9 @@ export function downloadEndpointExceptionListRoute(router: IRouter) { path: `${allowlistBaseRoute}/download/{artifactName}/{sha256}`, validate: { params: buildRouteValidation< - typeof downloadArtifactReqParamsSchema, - DownloadArtifactReqParamsSchema - >(downloadArtifactReqParamsSchema), + typeof downloadArtifactRequestParamsSchema, + DownloadArtifactRequestParamsSchema + >(downloadArtifactRequestParamsSchema), }, options: { tags: [] }, }, @@ -38,7 +44,7 @@ async function handleEndpointExceptionDownload(context, req, res) { const soClient = context.core.savedObjects.client; return soClient - .get(ArtifactConstants.SAVED_OBJECT_TYPE, `${req.params.artifactName}`) + .get(ArtifactConstants.SAVED_OBJECT_TYPE, `${req.params.artifactName}`) .then((artifact) => { const outBuffer = Buffer.from(artifact.attributes.body, 'binary'); @@ -48,14 +54,20 @@ async function handleEndpointExceptionDownload(context, req, res) { ); } - // TODO: validate response before returning - return res.ok({ + const downloadResponse = { body: outBuffer, headers: { 'content-encoding': 'xz', 'content-disposition': `attachment; filename=${artifact.attributes.name}.xz`, }, - }); + }; + + const [validated, errors] = validate(downloadResponse, downloadArtifactResponseSchema); + if (errors != null) { + return res.internalError({ body: errors }); + } else { + return res.ok(validated); + } }) .catch((err) => { return res.internalError({ body: err }); diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/saved_object.ts b/x-pack/plugins/security_solution/server/lib/exceptions/saved_object.ts deleted file mode 100644 index 2151219983f4c..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/exceptions/saved_object.ts +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { FrameworkRequest } from '../framework'; -import { ExceptionsArtifactSavedObject } from './types'; - -export interface ExceptionsArtifact { - getArtifact: ( - request: FrameworkRequest, - sha256: string - ) => Promise; -} diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/schemas/artifact_schema.ts b/x-pack/plugins/security_solution/server/lib/exceptions/schemas/artifact_schema.ts new file mode 100644 index 0000000000000..5f3d126d84087 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/exceptions/schemas/artifact_schema.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; +import { artifactName, body, created, encoding, schemaVersion, sha256, size } from './common'; + +export const artifactSoSchema = t.exact( + t.type({ + artifactName, + schemaVersion, + sha256, + encoding, + created, + body, + size, + }) +); + +export type ArtifactSoSchema = t.TypeOf; diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/schemas/common.ts b/x-pack/plugins/security_solution/server/lib/exceptions/schemas/common.ts new file mode 100644 index 0000000000000..a760ee750f449 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/exceptions/schemas/common.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; + +export const artifactName = t.string; + +export const body = t.string; + +export const created = t.string; // TODO: Make this into an ISO Date string check + +export const encoding = t.keyof({ + xz: null, +}); + +export const schemaVersion = t.keyof({ + '1.0.0': null, +}); + +export const sha256 = t.string; + +export const size = t.number; diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/schemas/index.ts b/x-pack/plugins/security_solution/server/lib/exceptions/schemas/index.ts index 13e4165eb5f16..23d65a29b52ab 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/schemas/index.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/schemas/index.ts @@ -4,4 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -export * from './download_artifact_schema'; +export * from './artifact_schema'; +export * from './common'; +export * from './request'; +export * from './response'; diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/schemas/download_artifact_schema.ts b/x-pack/plugins/security_solution/server/lib/exceptions/schemas/request/download_artifact_schema.ts similarity index 59% rename from x-pack/plugins/security_solution/server/lib/exceptions/schemas/download_artifact_schema.ts rename to x-pack/plugins/security_solution/server/lib/exceptions/schemas/request/download_artifact_schema.ts index 16dde0d76b845..5d311afcb93e8 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/schemas/download_artifact_schema.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/schemas/request/download_artifact_schema.ts @@ -5,15 +5,15 @@ */ import * as t from 'io-ts'; +import { artifactName, sha256 } from '../common'; -export const artifactName = t.string; -export const sha256 = t.string; - -export const downloadArtifactReqParamsSchema = t.exact( +export const downloadArtifactRequestParamsSchema = t.exact( t.type({ artifactName, sha256, }) ); -export type DownloadArtifactReqParamsSchema = t.TypeOf; +export type DownloadArtifactRequestParamsSchema = t.TypeOf< + typeof downloadArtifactRequestParamsSchema +>; diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/schemas/request/index.ts b/x-pack/plugins/security_solution/server/lib/exceptions/schemas/request/index.ts new file mode 100644 index 0000000000000..13e4165eb5f16 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/exceptions/schemas/request/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './download_artifact_schema'; diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/schemas/response/download_artifact_schema.ts b/x-pack/plugins/security_solution/server/lib/exceptions/schemas/response/download_artifact_schema.ts new file mode 100644 index 0000000000000..1323c67d1d029 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/exceptions/schemas/response/download_artifact_schema.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; +import { encoding } from '../common'; + +const body = t.unknown; // TODO: create Buffer type +const headers = t.exact( + t.type({ + 'content-encoding': encoding, + 'content-disposition': t.string, + }) +); + +export const downloadArtifactResponseSchema = t.exact( + t.type({ + body, + headers, + }) +); + +export type DownloadArtifactResponseSchema = t.TypeOf; diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/schemas/response/index.ts b/x-pack/plugins/security_solution/server/lib/exceptions/schemas/response/index.ts new file mode 100644 index 0000000000000..13e4165eb5f16 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/exceptions/schemas/response/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './download_artifact_schema'; diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts index 5aacf0cc0aa2b..23f769270fa77 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts @@ -14,6 +14,7 @@ import { import { ListPluginSetup } from '../../../../lists/server'; import { ConfigType } from '../../config'; import { GetFullEndpointExceptionList, CompressExceptionList } from './fetch_endpoint_exceptions'; +import { ArtifactSoSchema } from './schemas'; const PackagerTaskConstants = { TIMEOUT: '1m', @@ -94,32 +95,13 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { }; // Create the new artifact - const soResponse = await soClient.create( + const soResponse = await soClient.create( ArtifactConstants.SAVED_OBJECT_TYPE, exceptionSO, { id: `${artifactName}`, overwrite: true } ); context.logger.debug(`Current artifact ${artifactName} with hash ${sha256Hash}`); - - // // Clean up old artifacts - // const otherArtifacts = await soClient.find({ - // type: ArtifactConstants.SAVED_OBJECT_TYPE, - // search: artifactName, - // searchFields: ['name'], - // sortField: 'created', - // sortOrder: 'desc', - // }); - - // // Remove all but the latest artifact - // const toDelete = otherArtifacts.saved_objects.slice( - // 1, - // otherArtifacts.saved_objects.length - // ); - // for (const delObj of toDelete) { - // context.logger.debug(`REMOVING ${delObj.id}`); - // await soClient.delete(ArtifactConstants.SAVED_OBJECT_TYPE, delObj.id); - // } } catch (error) { if (error.statusCode === 409) { context.logger.debug(`No update to Endpoint Exceptions (${artifactName}), skipping.`); diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/types.ts b/x-pack/plugins/security_solution/server/lib/exceptions/types.ts deleted file mode 100644 index 2b27c5026750f..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/exceptions/types.ts +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -type ArtifactName = 'endpoint-allowlist'; -type ArtifactVersion = '1.0.0'; -type ArtifactEncoding = 'xz'; - -// TODO: define runtime types -export interface ExceptionsArtifactSavedObject { - name: ArtifactName; - schemaVersion: ArtifactVersion; - sha256: string; - encoding: ArtifactEncoding; - created: number; - body: string; -} From 849336c270930659ec2a80f4686703a950f2b247 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Wed, 17 Jun 2020 22:55:12 -0400 Subject: [PATCH 042/106] Set up for manifest update --- .../download_endpoint_exception_list.ts | 6 ++ .../server/lib/exceptions/task.ts | 69 +++++++++++++++---- 2 files changed, 61 insertions(+), 14 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.ts b/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.ts index 69046f2265d2c..8fb9e4d55dd24 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.ts @@ -36,6 +36,9 @@ export function downloadEndpointExceptionListRoute(router: IRouter, cache: Excep async (context, req, res) => { const soClient = context.core.savedObjects.client; + // TODO: authenticate api key + // https://github.com/elastic/kibana/issues/69329 + const validateResponse = (resp: object): KibanaResponse => { const [validated, errors] = validate(resp, downloadArtifactResponseSchema); if (errors != null) { @@ -74,6 +77,9 @@ export function downloadEndpointExceptionListRoute(router: IRouter, cache: Excep }); } + // Hashes match... populate cache + cache.set(cacheKey, artifact.attributes.body); + const downloadResponse = { body: outBuffer, headers: { diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts index e319bdc863beb..3260aaaf90bd6 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts @@ -56,25 +56,49 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { return `${PackagerTaskConstants.TYPE}:${PackagerTaskConstants.VERSION}`; }; - const run = async (taskId: string) => { + const run = async (taskId: int, state: Record) => { + // Check that this task is current if (taskId !== getTaskId()) { // old task, return context.logger.debug(`Outdated task running: ${taskId}`); return; } - context.logger.debug('Running exception list packager task'); - + // Get clients const [{ savedObjects }] = await context.core.getStartServices(); const savedObjectsRepository = savedObjects.createInternalRepository(); const soClient = new SavedObjectsClient(savedObjectsRepository); const exceptionListClient = context.lists.getExceptionListClient(soClient, 'kibana'); + // Main loop + let updated = false; + for (const os of ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS) { for (const schemaVersion of ArtifactConstants.SUPPORTED_SCHEMA_VERSIONS) { const artifactName = `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-${os}-${schemaVersion}`; + // Initialize state and prime cache + if (state[artifactName] === undefined) { + try { + const soGetResponse = await soClient.get( + ArtifactConstants.SAVED_OBJECT_TYPE, + artifactName + ); + + const cacheKey = `${artifactName}-${soGetResponse.attributes.sha256}`; + context.cache.set(cacheKey, soGetResponse.attributes.body); + + // eslint-disable-next-line require-atomic-updates + state[artifactName] = soGetResponse.attributes.sha256; + + updated = true; + } catch (err) { + context.logger.debug(`No artifact found ${artifactName} -- cache not primed`); + } + } + try { + // Retrieve exceptions, compute hash const exceptions = await GetFullEndpointExceptionList( exceptionListClient, os, @@ -96,27 +120,40 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { size: Buffer.from(JSON.stringify(exceptions)).byteLength, }; - // Create the new artifact + // Create/update the artifact const soResponse = await soClient.create( ArtifactConstants.SAVED_OBJECT_TYPE, exceptionSO, { id: `${artifactName}`, overwrite: true } ); + // If new, update state + if (state[artifactName] !== soResponse.attributes.sha256) { + context.logger.info( + `Change to artifact[${artifactName}] detected hash[${soResponse.attributes.sha256}]` + ); + + // eslint-disable-next-line require-atomic-updates + state[artifactName] = soResponse.attributes.sha256; + + updated = true; + } + + // Update the cache const cacheKey = `${artifactName}-${sha256Hash}`; + // TODO: does this reset the ttl? context.cache.set(cacheKey, compressedExceptions.toString('binary')); - - context.logger.debug(`Current artifact ${artifactName} with hash ${sha256Hash}`); } catch (error) { - if (error.statusCode === 409) { - context.logger.debug(`No update to Endpoint Exceptions (${artifactName}), skipping.`); - } else { - context.logger.error(error); - } + context.logger.error(error); } } } - return true; + + // Update manifest if there are changes + if (updated) { + context.logger.debug('One or more changes detected. Updating manifest...'); + // TODO: update manifest + } }; const getTaskRunner = (runnerContext: PackagerTaskRunnerContext): PackagerTaskRunner => { @@ -148,13 +185,17 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { createTaskRunner: ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => { return { run: async () => { - await run(taskInstance.id); + const state = Object.assign({}, ...taskInstance.state); + await run(taskInstance.id, state); + + // eslint-disable-next-line require-atomic-updates + taskInstance.state = state; const nextRun = new Date(); nextRun.setSeconds(nextRun.getSeconds() + context.config.packagerTaskInterval); return { - state: {}, + state: taskInstance.state, runAt: nextRun, }; }, From 40501cdcb540796455c7967d0a1931c57a50b16c Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Wed, 17 Jun 2020 23:01:13 -0400 Subject: [PATCH 043/106] minor change --- .../plugins/security_solution/server/lib/exceptions/task.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts index 3260aaaf90bd6..4272c8a8b9138 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts @@ -185,17 +185,15 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { createTaskRunner: ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => { return { run: async () => { + const state = Object.assign({}, ...taskInstance.state); await run(taskInstance.id, state); - // eslint-disable-next-line require-atomic-updates - taskInstance.state = state; - const nextRun = new Date(); nextRun.setSeconds(nextRun.getSeconds() + context.config.packagerTaskInterval); return { - state: taskInstance.state, + state, runAt: nextRun, }; }, From 680e425b018e665e94d400bd2edb17d05f7a819e Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Wed, 17 Jun 2020 23:04:42 -0400 Subject: [PATCH 044/106] remove setup code --- .../server/lib/exceptions/task.ts | 1 - .../server/lib/setup/index.ts | 144 ------------------ .../server/lib/setup/saved_object_mappings.ts | 40 ----- .../security_solution/server/routes/index.ts | 3 - .../security_solution/server/saved_objects.ts | 2 - 5 files changed, 190 deletions(-) delete mode 100644 x-pack/plugins/security_solution/server/lib/setup/index.ts delete mode 100644 x-pack/plugins/security_solution/server/lib/setup/saved_object_mappings.ts diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts index 4272c8a8b9138..4eea01e151cfc 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts @@ -185,7 +185,6 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { createTaskRunner: ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => { return { run: async () => { - const state = Object.assign({}, ...taskInstance.state); await run(taskInstance.id, state); diff --git a/x-pack/plugins/security_solution/server/lib/setup/index.ts b/x-pack/plugins/security_solution/server/lib/setup/index.ts deleted file mode 100644 index 52e77007c1078..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/setup/index.ts +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import uuid from 'uuid'; -import { - FakeRequest, - IRouter, - KibanaRequest, - SavedObjectsClientContract, - IScopedClusterClient, -} from 'src/core/server'; -import { SetupPlugins } from '../../plugin'; -import { endpointUserSavedObjectType, EndpointUser } from './saved_object_mappings'; - -const ENDPOINT_MANAGER_ROLE = 'endpoint_manager_role'; -const ENDPOINT_MANAGER_USERNAME = 'endpoint_admin'; -const ENDPOINT_SETUP_ROUTE = '/api/endpoint/setup'; // TODO - -/** - * Registers the setup route that enables the creation of an endpoint user - */ -export function postEndpointSetup(router: IRouter, security: SetupPlugins['security']) { - router.post( - { - path: ENDPOINT_SETUP_ROUTE, - validate: {}, - options: { authRequired: true }, - }, - async (context, req, res) => { - try { - const soClient = context.core.savedObjects.client; - const client = context.core.elasticsearch.legacy.client; - await setupEndpointUser(soClient, security, client); - return res.ok(); - } catch (err) { - return res.internalError({ body: err }); - } - } - ); - router.get( - { - path: ENDPOINT_SETUP_ROUTE, - validate: {}, - options: { authRequired: true }, - }, - async (context, req, res) => { - try { - const soClient = context.core.savedObjects.client; - const userResp = await getEndpointUserKey(soClient); - return res.ok({ body: userResp }); - } catch (err) { - return res.internalError({ body: err }); - } - } - ); -} - -/** - * Creates the endpoint user and generates an API key to be used by endpoints while communicating with Kibana - */ -async function setupEndpointUser( - soClient: SavedObjectsClientContract, - security: SetupPlugins['security'], - client: IScopedClusterClient // todo -) { - const res = await client.callAsCurrentUser('transport.request', { - method: 'PUT', - path: `/_security/role/${ENDPOINT_MANAGER_ROLE}`, - body: { - cluster: ['monitor', 'manage_api_key'], - indices: [ - { - names: ['logs-*', 'metrics-*', 'events-*'], - privileges: ['write', 'create_index'], - }, - ], - }, - }); - - const password = Buffer.from(uuid.v4()).toString('base64'); - - const resp = await client.callAsCurrentUser('transport.request', { - method: 'PUT', - path: `/_security/user/${ENDPOINT_MANAGER_USERNAME}`, - body: { - password, - roles: [ENDPOINT_MANAGER_ROLE], - metadata: { - updated_at: new Date().toISOString(), - }, - }, - }); - - const request: FakeRequest = { - headers: { - authorization: `Basic ${Buffer.from(`${ENDPOINT_MANAGER_USERNAME}:${password}`).toString( - 'base64' - )}`, - }, - }; - - if (!security) { - throw new Error('Missing security plugin'); - } - - const apikeyResponse = await security.authc.createAPIKey(request as KibanaRequest, { - name: 'test-api2', - role_descriptors: {}, // TODO - }); - - if (apikeyResponse !== null) { - const endpointUser: EndpointUser = { - username: ENDPOINT_MANAGER_USERNAME, - password, - apikey: apikeyResponse.api_key, - created: Date.now(), - }; - await persistEndpointUser(soClient, endpointUser); - } else { - throw new Error('Unable to persist endpoint user'); - } -} - -async function persistEndpointUser(soClient: SavedObjectsClientContract, user: EndpointUser) { - const soResponse = await soClient.create(endpointUserSavedObjectType, user, {}); -} - -async function getEndpointUserKey(soClient: SavedObjectsClientContract) { - const resp = await soClient.find({ - type: endpointUserSavedObjectType, - search: ENDPOINT_MANAGER_USERNAME, - searchFields: ['username'], - sortField: 'created', - sortOrder: 'desc', - }); - if (resp.saved_objects.length > 0) { - return resp.saved_objects[0]; - } else { - throw new Error('No Endpoint user created.'); - } -} diff --git a/x-pack/plugins/security_solution/server/lib/setup/saved_object_mappings.ts b/x-pack/plugins/security_solution/server/lib/setup/saved_object_mappings.ts deleted file mode 100644 index 89b6c19c6e6e4..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/setup/saved_object_mappings.ts +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { SavedObjectsType } from '../../../../../../src/core/server'; - -export const endpointUserSavedObjectType = 'securitySolution-endpoint-user'; - -export const endpointUserSavedObjectMappings: SavedObjectsType['mappings'] = { - properties: { - username: { - type: 'keyword', - }, - password: { - type: 'keyword', - }, - apikey: { - type: 'keyword', - }, - created: { - type: 'date', - }, - }, -}; - -export const type: SavedObjectsType = { - name: endpointUserSavedObjectType, - hidden: false, - namespaceType: 'agnostic', - mappings: endpointUserSavedObjectMappings, -}; - -export interface EndpointUser { - username: string; - password: string; - apikey: string; - created: number; -} diff --git a/x-pack/plugins/security_solution/server/routes/index.ts b/x-pack/plugins/security_solution/server/routes/index.ts index 6c42858be8ce4..54d7dcccba815 100644 --- a/x-pack/plugins/security_solution/server/routes/index.ts +++ b/x-pack/plugins/security_solution/server/routes/index.ts @@ -34,7 +34,6 @@ import { createTimelinesRoute } from '../lib/timeline/routes/create_timelines_ro import { updateTimelinesRoute } from '../lib/timeline/routes/update_timelines_route'; import { getDraftTimelinesRoute } from '../lib/timeline/routes/get_draft_timelines_route'; import { cleanDraftTimelinesRoute } from '../lib/timeline/routes/clean_draft_timelines_route'; -import { postEndpointSetup } from '../lib/setup'; import { SetupPlugins } from '../plugin'; import { ConfigType } from '../config'; @@ -73,8 +72,6 @@ export const initRoutes = ( findRulesStatusesRoute(router); - postEndpointSetup(router, security); - // Detection Engine Signals routes that have the REST endpoints of /api/detection_engine/signals // POST /api/detection_engine/signals/status // Example usage can be found in security_solution/server/lib/detection_engine/scripts/signals diff --git a/x-pack/plugins/security_solution/server/saved_objects.ts b/x-pack/plugins/security_solution/server/saved_objects.ts index c5941c2d0b2a0..2a8c03369eebb 100644 --- a/x-pack/plugins/security_solution/server/saved_objects.ts +++ b/x-pack/plugins/security_solution/server/saved_objects.ts @@ -12,7 +12,6 @@ import { type as timelineType } from './lib/timeline/saved_object_mappings'; import { type as ruleStatusType } from './lib/detection_engine/rules/saved_object_mappings'; import { type as ruleActionsType } from './lib/detection_engine/rule_actions/saved_object_mappings'; import { type as exceptionsArtifactType } from './lib/exceptions/saved_object_mappings'; -import { type as endpointUserType } from './lib/setup/saved_object_mappings'; const types = [ noteType, @@ -21,7 +20,6 @@ const types = [ ruleStatusType, timelineType, exceptionsArtifactType, - endpointUserType, ]; export const savedObjectTypes = types.map((type) => type.name); From 0eae6d8b0013898051a2ba41bdce45487bca839b Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Wed, 17 Jun 2020 23:24:19 -0400 Subject: [PATCH 045/106] add manifest schema --- .../server/lib/exceptions/schemas/common.ts | 8 ++++++ .../lib/exceptions/schemas/manifest_schema.ts | 27 +++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 x-pack/plugins/security_solution/server/lib/exceptions/schemas/manifest_schema.ts diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/schemas/common.ts b/x-pack/plugins/security_solution/server/lib/exceptions/schemas/common.ts index a760ee750f449..cb55c7a61ee57 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/schemas/common.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/schemas/common.ts @@ -16,6 +16,12 @@ export const encoding = t.keyof({ xz: null, }); +export const manifestVersion = t.string; + +export const manifestSchemaVersion = t.keyof({ + '1.0.0': null, +}); + export const schemaVersion = t.keyof({ '1.0.0': null, }); @@ -23,3 +29,5 @@ export const schemaVersion = t.keyof({ export const sha256 = t.string; export const size = t.number; + +export const url = t.string; diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/schemas/manifest_schema.ts b/x-pack/plugins/security_solution/server/lib/exceptions/schemas/manifest_schema.ts new file mode 100644 index 0000000000000..0333c79121710 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/exceptions/schemas/manifest_schema.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; +import { artifactName, manifestSchemaVersion, manifestVersion, sha256, size, url } from './common'; + +export const manifestSchema = t.exact( + t.type({ + manifestVersion, + manifestSchemaVersion, + artifacts: t.record(artifactName, manifestEntrySchema), + }) +); + +export const manifestEntrySchema = t.exact( + t.type({ + url, + sha256, + size, + }) +); + +export type ManifestEntrySchema = t.TypeOf; +export type ManifestSchema = t.TypeOf; From 80c0b4fd83952178f0d80430556995549e6a4e97 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Thu, 18 Jun 2020 10:22:28 -0400 Subject: [PATCH 046/106] refactoring --- .../server/lib/exceptions/cache.test.ts | 5 +++++ .../server/lib/exceptions/index.ts | 3 ++- .../{exceptions.test.ts => lists.test.ts} | 2 +- .../{fetch_endpoint_exceptions.ts => lists.ts} | 0 .../server/lib/exceptions/manifest.test.ts | 5 +++++ .../server/lib/exceptions/manifest.ts | 5 +++++ .../download_endpoint_exception_list.test.ts | 4 ++-- .../server/lib/exceptions/task.ts | 18 ++++++++++-------- .../plugins/security_solution/server/plugin.ts | 5 +++++ 9 files changed, 35 insertions(+), 12 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/exceptions/cache.test.ts rename x-pack/plugins/security_solution/server/lib/exceptions/{exceptions.test.ts => lists.test.ts} (97%) rename x-pack/plugins/security_solution/server/lib/exceptions/{fetch_endpoint_exceptions.ts => lists.ts} (100%) create mode 100644 x-pack/plugins/security_solution/server/lib/exceptions/manifest.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/exceptions/manifest.ts diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/cache.test.ts b/x-pack/plugins/security_solution/server/lib/exceptions/cache.test.ts new file mode 100644 index 0000000000000..41bc2aa258807 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/exceptions/cache.test.ts @@ -0,0 +1,5 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/index.ts b/x-pack/plugins/security_solution/server/lib/exceptions/index.ts index f27f511a198ad..6c6566528cbba 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/index.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/index.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -export * from './fetch_endpoint_exceptions'; +export * from './lists'; +export * from './manifest'; export * from './task'; export * from './cache'; diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/exceptions.test.ts b/x-pack/plugins/security_solution/server/lib/exceptions/lists.test.ts similarity index 97% rename from x-pack/plugins/security_solution/server/lib/exceptions/exceptions.test.ts rename to x-pack/plugins/security_solution/server/lib/exceptions/lists.test.ts index 7b60f1dcc8744..285cdcbf5942a 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/exceptions.test.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/lists.test.ts @@ -6,9 +6,9 @@ import { ExceptionListClient } from '../../../../lists/server/services/exception_lists/exception_list_client'; import { listMock } from '../../../../lists/server/mocks'; -import { GetFullEndpointExceptionList } from './fetch_endpoint_exceptions'; import { getFoundExceptionListItemSchemaMock } from '../../../../lists/common/schemas/response/found_exception_list_item_schema.mock'; import { getExceptionListItemSchemaMock } from '../../../../lists/common/schemas/response/exception_list_item_schema.mock'; +import { GetFullEndpointExceptionList } from './lists'; describe('buildEventTypeSignal', () => { let mockExceptionClient: ExceptionListClient; diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/fetch_endpoint_exceptions.ts b/x-pack/plugins/security_solution/server/lib/exceptions/lists.ts similarity index 100% rename from x-pack/plugins/security_solution/server/lib/exceptions/fetch_endpoint_exceptions.ts rename to x-pack/plugins/security_solution/server/lib/exceptions/lists.ts diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/manifest.test.ts b/x-pack/plugins/security_solution/server/lib/exceptions/manifest.test.ts new file mode 100644 index 0000000000000..41bc2aa258807 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/exceptions/manifest.test.ts @@ -0,0 +1,5 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/manifest.ts b/x-pack/plugins/security_solution/server/lib/exceptions/manifest.ts new file mode 100644 index 0000000000000..41bc2aa258807 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/exceptions/manifest.ts @@ -0,0 +1,5 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.test.ts b/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.test.ts index 70d23eddf7a67..1dcd1c738e834 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.test.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.test.ts @@ -20,10 +20,10 @@ import { httpServiceMock, httpServerMock, } from 'src/core/server/mocks'; -import { CompressExceptionList } from '../fetch_endpoint_exceptions'; +import { ExceptionsCache } from '../cache'; +import { CompressExceptionList } from '../lists'; import { ArtifactConstants } from '../task'; import { downloadEndpointExceptionListRoute } from './download_endpoint_exception_list'; -import { ExceptionsCache } from '../cache'; const mockArtifactName = `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-windows-1.0.0`; const expectedEndpointExceptions = { diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts index 4eea01e151cfc..ad1515cb1b004 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts @@ -13,9 +13,9 @@ import { } from '../../../../../plugins/task_manager/server'; import { ListPluginSetup } from '../../../../lists/server'; import { ConfigType } from '../../config'; -import { GetFullEndpointExceptionList, CompressExceptionList } from './fetch_endpoint_exceptions'; -import { ArtifactSoSchema } from './schemas'; import { ExceptionsCache } from './cache'; +import { GetFullEndpointExceptionList, CompressExceptionList } from './lists'; +import { ArtifactSoSchema } from './schemas'; const PackagerTaskConstants = { TIMEOUT: '1m', @@ -56,12 +56,14 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { return `${PackagerTaskConstants.TYPE}:${PackagerTaskConstants.VERSION}`; }; - const run = async (taskId: int, state: Record) => { + const run = async (taskId: int, taskState: Record): Record => { + const state = Object.assign({}, ...taskState); + // Check that this task is current if (taskId !== getTaskId()) { // old task, return context.logger.debug(`Outdated task running: ${taskId}`); - return; + return state; } // Get clients @@ -88,7 +90,6 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { const cacheKey = `${artifactName}-${soGetResponse.attributes.sha256}`; context.cache.set(cacheKey, soGetResponse.attributes.body); - // eslint-disable-next-line require-atomic-updates state[artifactName] = soGetResponse.attributes.sha256; updated = true; @@ -133,7 +134,6 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { `Change to artifact[${artifactName}] detected hash[${soResponse.attributes.sha256}]` ); - // eslint-disable-next-line require-atomic-updates state[artifactName] = soResponse.attributes.sha256; updated = true; @@ -154,6 +154,8 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { context.logger.debug('One or more changes detected. Updating manifest...'); // TODO: update manifest } + + return state; }; const getTaskRunner = (runnerContext: PackagerTaskRunnerContext): PackagerTaskRunner => { @@ -185,8 +187,8 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { createTaskRunner: ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => { return { run: async () => { - const state = Object.assign({}, ...taskInstance.state); - await run(taskInstance.id, state); + // const state = Object.assign({}, ...taskInstance.state); + const state = await run(taskInstance.id, taskInstance.state); const nextRun = new Date(); nextRun.setSeconds(nextRun.getSeconds() + context.config.packagerTaskInterval); diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 16cb77a6f21cb..2d1783fc9bc51 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -241,6 +241,11 @@ export class Plugin implements IPlugin Date: Thu, 18 Jun 2020 15:31:00 -0400 Subject: [PATCH 047/106] manifest rewrite (partial) --- .../server/lib/exceptions/index.ts | 2 +- .../server/lib/exceptions/manifest.ts | 90 +++++++++++++++++++ .../server/lib/exceptions/manifest_entry.ts | 82 +++++++++++++++++ .../download_endpoint_exception_list.test.ts | 2 +- .../download_endpoint_exception_list.ts | 2 +- .../server/lib/exceptions/task.ts | 38 ++++++-- .../security_solution/server/plugin.ts | 9 +- 7 files changed, 210 insertions(+), 15 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/exceptions/manifest_entry.ts diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/index.ts b/x-pack/plugins/security_solution/server/lib/exceptions/index.ts index 6c6566528cbba..baf73663654ae 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/index.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/index.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +export * from './cache'; export * from './lists'; export * from './manifest'; export * from './task'; -export * from './cache'; diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/manifest.ts b/x-pack/plugins/security_solution/server/lib/exceptions/manifest.ts index 41bc2aa258807..8beab289d067c 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/manifest.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/manifest.ts @@ -3,3 +3,93 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ + +import { createHash } from 'crypto'; + +import { SavedObjectsClient, SavedObjectsPluginStart } from '../../../../../../src/core/server'; + +import { ListPluginSetup } from '../../../../lists/server'; + +// eslint-disable-next-line @kbn/eslint/no-restricted-paths +import { ExceptionListClient } from '../../../../lists/server/services/exception_lists/exception_list_client'; + +import { GetFullEndpointExceptionList, CompressExceptionList } from './lists'; + +import { ManifestEntry } from './manifest_entry'; + +export const ArtifactConstants = { + GLOBAL_ALLOWLIST_NAME: 'endpoint-allowlist', + SAVED_OBJECT_TYPE: 'securitySolution-exceptions-artifact', + SUPPORTED_OPERATING_SYSTEMS: ['linux', 'windows'], + SUPPORTED_SCHEMA_VERSIONS: ['1.0.0'], +}; + +export interface ManifestOptions { + lists: ListPluginSetup; + savedObjects: SavedObjectsPluginStart; +} + +export class ManifestService { + private static instance: ManifestService; + + private entries: ManifestEntry[]; + private soClient: SavedObjectsClient; + private exceptionListClient: ExceptionListClient; + + private constructor(context: ManifestOptions) { + const savedObjectsRepository = context.savedObjects.createInternalRepository(); + this.soClient = new SavedObjectsClient(savedObjectsRepository); + this.exceptionListClient = context.lists.getExceptionListClient(this.soClient, 'kibana'); + this.entries = []; + } + + public static getInstance(context: ManifestOptions) { + // TODO: lock? + if (!ManifestService.instance) { + ManifestService.instance = new ManifestService(context); + } + + return ManifestService.instance; + } + + public static getArtifactName(os: string, schemaVersion: string) { + return `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-${os}-${schemaVersion}`; + } + + public getState(): object { + // TODO: type + // console.log(this.sha256); + return this.entries + .map((entry) => { + // console.log(this.sha256); + return entry.getState(); + }) + .reduce((map, state) => { + // console.log(state.sha256); + // console.log(state.size); + map[state.identifier] = { + url: state.url, + sha256: state.sha256, + size: state.size, + }; + return map; + }); + } + + public async refresh() { + this.entries = []; + const entries: ManifestEntry = []; + + for (const os of ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS) { + for (const schemaVersion of ArtifactConstants.SUPPORTED_SCHEMA_VERSIONS) { + entries.push(new ManifestEntry(this.exceptionListClient, os, schemaVersion)); + } + } + + entries.map(async (entry) => { + await entry.refresh(); + }); + + this.entries = entries; + } +} diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/manifest_entry.ts b/x-pack/plugins/security_solution/server/lib/exceptions/manifest_entry.ts new file mode 100644 index 0000000000000..9510db7b878ae --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/exceptions/manifest_entry.ts @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { createHash } from 'crypto'; + +import { SavedObjectsClient, SavedObjectsPluginStart } from '../../../../../../src/core/server'; + +import { ListPluginSetup } from '../../../../lists/server'; + +// eslint-disable-next-line @kbn/eslint/no-restricted-paths +import { ExceptionListClient } from '../../../../lists/server/services/exception_lists/exception_list_client'; + +import { GetFullEndpointExceptionList, CompressExceptionList } from './lists'; + +import { ArtifactConstants, ManifestService } from './manifest'; + +export class ManifestEntry { + private exceptionListClient: ExceptionListClient; + + private os: string; + private schemaVersion: string; + + private sha256: string; + private size: int; + + constructor(exceptionListClient: ExceptionListClient, os: string, schemaVersion: string) { + this.exceptionListClient = exceptionListClient; + this.os = os; + this.schemaVersion = schemaVersion; + } + + public getIdentifier(): string { + return ManifestService.getArtifactName(this.os, this.schemaVersion); + } + + public getUrl(): string { + return `/api/endpoint/allowlist/download/${this.getIdentifier()}/${this.sha256}`; + } + + public getState(): object { + // TODO: type + // console.log(this); + // console.log(sha256); + return { + identifier: this.getIdentifier(), + url: this.getUrl(), + sha256: this.sha256, + size: this.size, + }; + } + + public async refresh() { + const exceptions = await GetFullEndpointExceptionList( + this.exceptionListClient, + this.os, + this.schemaVersion + ); + const compressedExceptions: Buffer = await CompressExceptionList(exceptions); + + const sha256Hash = createHash('sha256') + .update(compressedExceptions.toString('utf8'), 'utf8') + .digest('hex'); + + this.sha256 = sha256Hash; + this.size = Buffer.from(JSON.stringify(exceptions)).byteLength; + + /* + this.savedObject = { + name: this.getIdentifier(), + schemaVersion: this.schemaVersion, + sha256: sha256Hash, + encoding: 'xz', + created: Date.now(), + body: compressedExceptions.toString('binary'), + size: Buffer.from(JSON.stringify(exceptions)).byteLength, + }; + */ + } +} diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.test.ts b/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.test.ts index 1dcd1c738e834..96e1b738fce64 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.test.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.test.ts @@ -22,7 +22,7 @@ import { } from 'src/core/server/mocks'; import { ExceptionsCache } from '../cache'; import { CompressExceptionList } from '../lists'; -import { ArtifactConstants } from '../task'; +import { ArtifactConstants } from '../manifest'; import { downloadEndpointExceptionListRoute } from './download_endpoint_exception_list'; const mockArtifactName = `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-windows-1.0.0`; diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.ts b/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.ts index 8fb9e4d55dd24..325ddafc228f1 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.ts @@ -8,7 +8,7 @@ import { IRouter, KibanaResponse } from 'src/core/server'; import { validate } from '../../../../common/validate'; import { buildRouteValidation } from '../../../utils/build_validation/route_validation'; import { ExceptionsCache } from '../cache'; -import { ArtifactConstants } from '../task'; +import { ArtifactConstants } from '../manifest'; import { ArtifactDownloadSchema, DownloadArtifactRequestParamsSchema, diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts index ad1515cb1b004..aa2fd987d4da6 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts @@ -5,7 +5,12 @@ */ import { createHash } from 'crypto'; -import { CoreSetup, Logger, SavedObjectsClient } from '../../../../../../src/core/server'; +import { + CoreSetup, + Logger, + SavedObjectsClient, + SavedObjectsPluginStart, +} from '../../../../../../src/core/server'; import { ConcreteTaskInstance, TaskManagerSetupContract, @@ -15,6 +20,7 @@ import { ListPluginSetup } from '../../../../lists/server'; import { ConfigType } from '../../config'; import { ExceptionsCache } from './cache'; import { GetFullEndpointExceptionList, CompressExceptionList } from './lists'; +import { ManifestService } from './manifest'; import { ArtifactSoSchema } from './schemas'; const PackagerTaskConstants = { @@ -23,13 +29,6 @@ const PackagerTaskConstants = { VERSION: '1.0.0', }; -export const ArtifactConstants = { - GLOBAL_ALLOWLIST_NAME: 'endpoint-allowlist', - SAVED_OBJECT_TYPE: 'securitySolution-exceptions-artifact', - SUPPORTED_OPERATING_SYSTEMS: ['linux', 'windows'], - SUPPORTED_SCHEMA_VERSIONS: ['1.0.0'], -}; - export interface PackagerTask { getTaskRunner: (context: PackagerTaskRunnerContext) => PackagerTaskRunner; } @@ -66,6 +65,18 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { return state; } + // Update manifest and return state + const [{ savedObjects }] = await context.core.getStartServices(); + const manifest = ManifestService.getInstance({ + lists: context.lists, + savedObjects, + }); + + await manifest.refresh(); + return manifest.getState(); + }; + + /* // Get clients const [{ savedObjects }] = await context.core.getStartServices(); const savedObjectsRepository = savedObjects.createInternalRepository(); @@ -157,6 +168,11 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { return state; }; + */ + + const getContext = (): PackagerTaskContext => { + return context; + }; const getTaskRunner = (runnerContext: PackagerTaskRunnerContext): PackagerTaskRunner => { return { @@ -187,8 +203,11 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { createTaskRunner: ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => { return { run: async () => { - // const state = Object.assign({}, ...taskInstance.state); const state = await run(taskInstance.id, taskInstance.state); + // console.log(state); + + // TODO: compare state to taskInstance.state + // If different, update manifest const nextRun = new Date(); nextRun.setSeconds(nextRun.getSeconds() + context.config.packagerTaskInterval); @@ -205,6 +224,7 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { }); return { + getContext, getTaskRunner, }; } diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 2d1783fc9bc51..ec8be339267ff 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -241,14 +241,17 @@ export class Plugin implements IPlugin Date: Fri, 19 Jun 2020 12:41:35 -0400 Subject: [PATCH 048/106] finish scaffolding new manifest logic --- .../server/lib/exceptions/manifest.ts | 18 +++---- .../server/lib/exceptions/manifest_entry.ts | 2 - .../server/lib/exceptions/task.ts | 52 ++++++++++++++++--- 3 files changed, 54 insertions(+), 18 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/manifest.ts b/x-pack/plugins/security_solution/server/lib/exceptions/manifest.ts index 8beab289d067c..f4192f19b0716 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/manifest.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/manifest.ts @@ -58,15 +58,11 @@ export class ManifestService { public getState(): object { // TODO: type - // console.log(this.sha256); return this.entries .map((entry) => { - // console.log(this.sha256); return entry.getState(); }) .reduce((map, state) => { - // console.log(state.sha256); - // console.log(state.size); map[state.identifier] = { url: state.url, sha256: state.sha256, @@ -82,14 +78,18 @@ export class ManifestService { for (const os of ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS) { for (const schemaVersion of ArtifactConstants.SUPPORTED_SCHEMA_VERSIONS) { - entries.push(new ManifestEntry(this.exceptionListClient, os, schemaVersion)); + const entry = new ManifestEntry(this.exceptionListClient, os, schemaVersion); + + try { + await entry.refresh(); + entries.push(entry); + } catch (err) { + // TODO: context logger? + // console.log(err); + } } } - entries.map(async (entry) => { - await entry.refresh(); - }); - this.entries = entries; } } diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/manifest_entry.ts b/x-pack/plugins/security_solution/server/lib/exceptions/manifest_entry.ts index 9510db7b878ae..492c90a3a579d 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/manifest_entry.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/manifest_entry.ts @@ -42,8 +42,6 @@ export class ManifestEntry { public getState(): object { // TODO: type - // console.log(this); - // console.log(sha256); return { identifier: this.getIdentifier(), url: this.getUrl(), diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts index aa2fd987d4da6..d33c9a41a7a95 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts @@ -20,7 +20,7 @@ import { ListPluginSetup } from '../../../../lists/server'; import { ConfigType } from '../../config'; import { ExceptionsCache } from './cache'; import { GetFullEndpointExceptionList, CompressExceptionList } from './lists'; -import { ManifestService } from './manifest'; +import { ArtifactConstants, ManifestService } from './manifest'; import { ArtifactSoSchema } from './schemas'; const PackagerTaskConstants = { @@ -55,8 +55,9 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { return `${PackagerTaskConstants.TYPE}:${PackagerTaskConstants.VERSION}`; }; - const run = async (taskId: int, taskState: Record): Record => { - const state = Object.assign({}, ...taskState); + const run = async (taskId: int, state: Record): Record => { + // console.log(taskState); + // const state = Object.assign({}, ...taskState); // Check that this task is current if (taskId !== getTaskId()) { @@ -72,7 +73,12 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { savedObjects, }); - await manifest.refresh(); + try { + await manifest.refresh(); + } catch (err) { + context.logger.error(err); + } + return manifest.getState(); }; @@ -204,10 +210,42 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { return { run: async () => { const state = await run(taskInstance.id, taskInstance.state); - // console.log(state); - // TODO: compare state to taskInstance.state - // If different, update manifest + // Get clients + const [{ savedObjects }] = await context.core.getStartServices(); + const savedObjectsRepository = savedObjects.createInternalRepository(); + const soClient = new SavedObjectsClient(savedObjectsRepository); + + // Create/update the artifacts + let updated = false; + state.forEach((id, entry) => { + const soResponse = await soClient.create( + ArtifactConstants.SAVED_OBJECT_TYPE, + entry.getSavedObject(), + { id, overwrite: true } + ); + + // If new, update state + if (state[id].sha256 !== soResponse.attributes.sha256) { + context.logger.info( + `Change to artifact[${id}] detected hash[${soResponse.attributes.sha256}]` + ); + + state[id] = entry; + updated = true; + } + + // TODO: Update the cache + /* + const cacheKey = `${id}-${soResponse.attributes.sha256}`; + // TODO: does this reset the ttl? + context.cache.set(cacheKey, compressedExceptions.toString('binary')); + */ + }); + + if (updated) { + // TODO: update manifest in config + } const nextRun = new Date(); nextRun.setSeconds(nextRun.getSeconds() + context.config.packagerTaskInterval); From a83dfcd6096dd22d3a57dc462adc5e3d78728acc Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Fri, 19 Jun 2020 13:04:06 -0400 Subject: [PATCH 049/106] syntax errors --- .../server/lib/exceptions/manifest_entry.ts | 8 +++- .../server/lib/exceptions/task.ts | 43 ++++++++++--------- 2 files changed, 28 insertions(+), 23 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/manifest_entry.ts b/x-pack/plugins/security_solution/server/lib/exceptions/manifest_entry.ts index 492c90a3a579d..df9a3b98a4043 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/manifest_entry.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/manifest_entry.ts @@ -26,6 +26,8 @@ export class ManifestEntry { private sha256: string; private size: int; + private savedObject: object; // TODO: type + constructor(exceptionListClient: ExceptionListClient, os: string, schemaVersion: string) { this.exceptionListClient = exceptionListClient; this.os = os; @@ -40,6 +42,10 @@ export class ManifestEntry { return `/api/endpoint/allowlist/download/${this.getIdentifier()}/${this.sha256}`; } + public getSavedObject(): object { + return this.savedObject; + } + public getState(): object { // TODO: type return { @@ -65,7 +71,6 @@ export class ManifestEntry { this.sha256 = sha256Hash; this.size = Buffer.from(JSON.stringify(exceptions)).byteLength; - /* this.savedObject = { name: this.getIdentifier(), schemaVersion: this.schemaVersion, @@ -75,6 +80,5 @@ export class ManifestEntry { body: compressedExceptions.toString('binary'), size: Buffer.from(JSON.stringify(exceptions)).byteLength, }; - */ } } diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts index d33c9a41a7a95..2ead9d2dfc930 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts +++ b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts @@ -218,30 +218,31 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { // Create/update the artifacts let updated = false; - state.forEach((id, entry) => { - const soResponse = await soClient.create( - ArtifactConstants.SAVED_OBJECT_TYPE, - entry.getSavedObject(), - { id, overwrite: true } - ); - - // If new, update state - if (state[id].sha256 !== soResponse.attributes.sha256) { - context.logger.info( - `Change to artifact[${id}] detected hash[${soResponse.attributes.sha256}]` + for (const id in state) { + if (Object.prototype.hasOwnProperty.call(state, id)) { + const soResponse = await soClient.create( + ArtifactConstants.SAVED_OBJECT_TYPE, + state[id].getSavedObject(), + { id, overwrite: true } ); - state[id] = entry; - updated = true; + // If new, update state + if (state[id].sha256 !== soResponse.attributes.sha256) { + context.logger.info( + `Change to artifact[${id}] detected hash[${soResponse.attributes.sha256}]` + ); + + updated = true; + } + + // TODO: Update the cache + /* + const cacheKey = `${id}-${soResponse.attributes.sha256}`; + // TODO: does this reset the ttl? + context.cache.set(cacheKey, compressedExceptions.toString('binary')); + */ } - - // TODO: Update the cache - /* - const cacheKey = `${id}-${soResponse.attributes.sha256}`; - // TODO: does this reset the ttl? - context.cache.set(cacheKey, compressedExceptions.toString('binary')); - */ - }); + } if (updated) { // TODO: update manifest in config From 3e9e557d38e12528d0a1d67c5d9727c8cd5a813c Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Sat, 20 Jun 2020 00:58:06 -0400 Subject: [PATCH 050/106] more refactoring --- x-pack/plugins/lists/server/index.ts | 1 + .../server/endpoint/artifact_services.ts | 32 +++ .../endpoint/artifacts/artifact_service.ts | 64 +++++ .../artifacts}/cache.test.ts | 0 .../artifacts}/cache.ts | 0 .../server/endpoint/artifacts/common.ts | 17 ++ .../artifacts}/index.ts | 2 + .../artifacts}/lists.test.ts | 0 .../artifacts}/lists.ts | 3 +- .../artifacts}/manifest.test.ts | 0 .../server/endpoint/artifacts/manifest.ts | 50 ++++ .../artifacts}/manifest_entry.ts | 11 +- .../endpoint/artifacts/manifest_service.ts | 46 +++ .../artifacts}/saved_object_mappings.ts | 36 ++- .../artifacts}/schemas/common.ts | 2 +- .../artifacts}/schemas/index.ts | 2 +- .../artifacts}/schemas/manifest_schema.ts | 0 .../request/download_artifact_schema.ts | 0 .../artifacts}/schemas/request/index.ts | 0 .../response/download_artifact_schema.ts | 0 .../artifacts}/schemas/response/index.ts | 0 .../artifacts/schemas/saved_objects.ts | 31 ++ .../server/endpoint/artifacts/task.ts | 166 +++++++++++ .../endpoint/endpoint_app_context_services.ts | 19 ++ .../download_endpoint_exception_list.test.ts | 6 +- .../download_endpoint_exception_list.ts | 10 +- .../routes/artifacts}/index.ts | 0 .../endpoint/services/artifacts/index.ts | 5 + .../server/lib/exceptions/manifest.ts | 95 ------- .../lib/exceptions/schemas/artifact_schema.ts | 22 -- .../server/lib/exceptions/task.ts | 269 ------------------ .../security_solution/server/plugin.ts | 35 ++- .../security_solution/server/saved_objects.ts | 3 +- 33 files changed, 506 insertions(+), 421 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/endpoint/artifact_services.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/artifacts/artifact_service.ts rename x-pack/plugins/security_solution/server/{lib/exceptions => endpoint/artifacts}/cache.test.ts (100%) rename x-pack/plugins/security_solution/server/{lib/exceptions => endpoint/artifacts}/cache.ts (100%) create mode 100644 x-pack/plugins/security_solution/server/endpoint/artifacts/common.ts rename x-pack/plugins/security_solution/server/{lib/exceptions => endpoint/artifacts}/index.ts (82%) rename x-pack/plugins/security_solution/server/{lib/exceptions => endpoint/artifacts}/lists.test.ts (100%) rename x-pack/plugins/security_solution/server/{lib/exceptions => endpoint/artifacts}/lists.ts (94%) rename x-pack/plugins/security_solution/server/{lib/exceptions => endpoint/artifacts}/manifest.test.ts (100%) create mode 100644 x-pack/plugins/security_solution/server/endpoint/artifacts/manifest.ts rename x-pack/plugins/security_solution/server/{lib/exceptions => endpoint/artifacts}/manifest_entry.ts (86%) create mode 100644 x-pack/plugins/security_solution/server/endpoint/artifacts/manifest_service.ts rename x-pack/plugins/security_solution/server/{lib/exceptions => endpoint/artifacts}/saved_object_mappings.ts (57%) rename x-pack/plugins/security_solution/server/{lib/exceptions => endpoint/artifacts}/schemas/common.ts (94%) rename x-pack/plugins/security_solution/server/{lib/exceptions => endpoint/artifacts}/schemas/index.ts (90%) rename x-pack/plugins/security_solution/server/{lib/exceptions => endpoint/artifacts}/schemas/manifest_schema.ts (100%) rename x-pack/plugins/security_solution/server/{lib/exceptions => endpoint/artifacts}/schemas/request/download_artifact_schema.ts (100%) rename x-pack/plugins/security_solution/server/{lib/exceptions => endpoint/artifacts}/schemas/request/index.ts (100%) rename x-pack/plugins/security_solution/server/{lib/exceptions => endpoint/artifacts}/schemas/response/download_artifact_schema.ts (100%) rename x-pack/plugins/security_solution/server/{lib/exceptions => endpoint/artifacts}/schemas/response/index.ts (100%) create mode 100644 x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/saved_objects.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/artifacts/task.ts rename x-pack/plugins/security_solution/server/{lib/exceptions/routes => endpoint/routes/artifacts}/download_endpoint_exception_list.test.ts (97%) rename x-pack/plugins/security_solution/server/{lib/exceptions/routes => endpoint/routes/artifacts}/download_endpoint_exception_list.ts (91%) rename x-pack/plugins/security_solution/server/{lib/exceptions/routes => endpoint/routes/artifacts}/index.ts (100%) create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/index.ts delete mode 100644 x-pack/plugins/security_solution/server/lib/exceptions/manifest.ts delete mode 100644 x-pack/plugins/security_solution/server/lib/exceptions/schemas/artifact_schema.ts delete mode 100644 x-pack/plugins/security_solution/server/lib/exceptions/task.ts diff --git a/x-pack/plugins/lists/server/index.ts b/x-pack/plugins/lists/server/index.ts index 33f58ba65d3c3..8279095c6ffa8 100644 --- a/x-pack/plugins/lists/server/index.ts +++ b/x-pack/plugins/lists/server/index.ts @@ -10,6 +10,7 @@ import { ConfigSchema } from './config'; import { ListPlugin } from './plugin'; // exporting these since its required at top level in siem plugin +export { ExceptionListClient } from './services/exception_lists/exception_list_client'; export { ListClient } from './services/lists/list_client'; export { ListPluginSetup } from './types'; diff --git a/x-pack/plugins/security_solution/server/endpoint/artifact_services.ts b/x-pack/plugins/security_solution/server/endpoint/artifact_services.ts new file mode 100644 index 0000000000000..9ce59f772d850 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/artifact_services.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { SavedObjectsClient } from '../../../../../src/core/server'; +import { ExceptionListClient } from '../../../lists'; +import { EndpointAppContextService } from './endpoint_app_context_services'; +import { Manifest } from './artifacts/manifest'; + +/** + * A singleton that holds shared services for working with artifacts. + */ +export class ArtifactService { + private endpointAppContextService: EndpointAppContextService | undefined; + private exceptionListClient: ExceptionListClient | undefined; + private savedObjectsClient: SavedObjectsClient | undefined; + + private manifest: Manifest | undefined; + + public start(deps: { + endpointAppContextService: EndpointAppContextService, + exceptionListClient; ExceptionListClient, + savedObjectsClient: SavedObjectsClient, + }) { + this.endpointAppContextService = deps.endpointAppContextService; + this.exceptionListClient = deps.exceptionListClient; + this.savedObjectsClient = deps.savedObjectsClient; + } + + public stop() {} +} diff --git a/x-pack/plugins/security_solution/server/endpoint/artifacts/artifact_service.ts b/x-pack/plugins/security_solution/server/endpoint/artifacts/artifact_service.ts new file mode 100644 index 0000000000000..d64b14ba10dc8 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/artifacts/artifact_service.ts @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { createHash } from 'crypto'; + +import { SavedObjectsClient } from '../../../../../../src/core/server'; +import { ExceptionListClient } from '../../../../lists/server'; + +import { ArtifactConstants } from './common'; +import { CompressExceptionList, GetFullEndpointExceptionList } from './lists'; +import { InternalArtifactSchema } from './schemas'; + +export interface ArtifactServiceOptions { + savedObjectsClient: SavedObjectsClient; +} + +export class ArtifactService { + + private soClient: SavedObjectsClient; + private exceptionListClient: ExceptionListClient; + + constructor(context: ManifestOptions) { + this.soClient = context.savedObjectsClient; + this.exceptionListClient = context.exceptionListClient; + } + + public async buildExceptionListArtifacts(): Promise> { + const artifacts: InternalArtifactSchema = []; + + for (const os of ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS) { + for (const schemaVersion of ArtifactConstants.SUPPORTED_SCHEMA_VERSIONS) { + const exceptions = await GetFullEndpointExceptionList( + this.exceptionListClient, + os, + schemaVersion, + ); + + const compressedExceptions: Buffer = await CompressExceptionList(exceptions); + + const sha256 = createHash('sha256') + .update(compressedExceptions.toString('utf8'), 'utf8') + .digest('hex'); + + artifacts.push({ + identifier: this.getIdentifier(), + sha256, + encoding: 'xz', + created: Date.now(), + body: compressedExceptions.toString('binary'), + size: Buffer.from(JSON.stringify(exceptions)).byteLength, + }); + } + } + + return artifacts; + } + + public async upsertArtifact(artifact: InternalArtifactSchema) { + // TODO + } +} diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/cache.test.ts b/x-pack/plugins/security_solution/server/endpoint/artifacts/cache.test.ts similarity index 100% rename from x-pack/plugins/security_solution/server/lib/exceptions/cache.test.ts rename to x-pack/plugins/security_solution/server/endpoint/artifacts/cache.test.ts diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/cache.ts b/x-pack/plugins/security_solution/server/endpoint/artifacts/cache.ts similarity index 100% rename from x-pack/plugins/security_solution/server/lib/exceptions/cache.ts rename to x-pack/plugins/security_solution/server/endpoint/artifacts/cache.ts diff --git a/x-pack/plugins/security_solution/server/endpoint/artifacts/common.ts b/x-pack/plugins/security_solution/server/endpoint/artifacts/common.ts new file mode 100644 index 0000000000000..92ff54e73fcf6 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/artifacts/common.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export const ArtifactConstants = { + GLOBAL_ALLOWLIST_NAME: 'endpoint-allowlist', + SAVED_OBJECT_TYPE: 'securitySolution:endpoint:exceptions-artifact', + SUPPORTED_OPERATING_SYSTEMS: ['linux', 'windows'], + SUPPORTED_SCHEMA_VERSIONS: ['1.0.0'], +}; + +export const ManifestConstants = { + SAVED_OBJECT_TYPE: 'securitySolution:endpoint:manifest-artifact', + SCHEMA_VERSION = '1.0.0'; +}; diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/index.ts b/x-pack/plugins/security_solution/server/endpoint/artifacts/index.ts similarity index 82% rename from x-pack/plugins/security_solution/server/lib/exceptions/index.ts rename to x-pack/plugins/security_solution/server/endpoint/artifacts/index.ts index baf73663654ae..4a2522baa3a1c 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/index.ts +++ b/x-pack/plugins/security_solution/server/endpoint/artifacts/index.ts @@ -4,7 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ +export * from './artifact_service'; export * from './cache'; export * from './lists'; export * from './manifest'; +export * from './manifest_service'; export * from './task'; diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/lists.test.ts b/x-pack/plugins/security_solution/server/endpoint/artifacts/lists.test.ts similarity index 100% rename from x-pack/plugins/security_solution/server/lib/exceptions/lists.test.ts rename to x-pack/plugins/security_solution/server/endpoint/artifacts/lists.test.ts diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/lists.ts b/x-pack/plugins/security_solution/server/endpoint/artifacts/lists.ts similarity index 94% rename from x-pack/plugins/security_solution/server/lib/exceptions/lists.ts rename to x-pack/plugins/security_solution/server/endpoint/artifacts/lists.ts index 7fce105582450..56a8eeaa61c71 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/lists.ts +++ b/x-pack/plugins/security_solution/server/endpoint/artifacts/lists.ts @@ -6,8 +6,7 @@ import lzma from 'lzma-native'; import { FoundExceptionListItemSchema } from '../../../../lists/common/schemas/response/found_exception_list_item_schema'; -// eslint-disable-next-line @kbn/eslint/no-restricted-paths -import { ExceptionListClient } from '../../../../lists/server/services/exception_lists/exception_list_client'; +import { ExceptionListClient } from '../../../../lists/server'; export interface EndpointExceptionList { exceptions_list: ExceptionsList[]; diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/manifest.test.ts b/x-pack/plugins/security_solution/server/endpoint/artifacts/manifest.test.ts similarity index 100% rename from x-pack/plugins/security_solution/server/lib/exceptions/manifest.test.ts rename to x-pack/plugins/security_solution/server/endpoint/artifacts/manifest.test.ts diff --git a/x-pack/plugins/security_solution/server/endpoint/artifacts/manifest.ts b/x-pack/plugins/security_solution/server/endpoint/artifacts/manifest.ts new file mode 100644 index 0000000000000..49a705354c8e3 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/artifacts/manifest.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { createHash } from 'crypto'; + +import { SavedObjectsClient, SavedObjectsPluginStart } from '../../../../../../src/core/server'; + +import { ListPluginSetup } from '../../../../lists/server'; + +import { ExceptionListClient } from '../../../../lists/server'; + +import { GetFullEndpointExceptionList, CompressExceptionList } from './lists'; + +import { ManifestEntry } from './manifest_entry'; + +import { ArtifactConstants } from './common'; + +export class Manifest { + private entries: ManifestEntry[]; + + constructor(context: ManifestOptions) { + this.entries = []; + } + + public getState(): object { + // TODO: type + return this.entries + .map((entry) => { + return entry.getState(); + }) + .reduce((map, state) => { + map[state.identifier] = { + url: state.url, + sha256: state.sha256, + size: state.size, + }; + return map; + }); + } + + public contains(artifact: InternalArtifactSchema): boolean { + // TODO + // calculate id (identifier + sha256) + // look up id + // return if found + } +} diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/manifest_entry.ts b/x-pack/plugins/security_solution/server/endpoint/artifacts/manifest_entry.ts similarity index 86% rename from x-pack/plugins/security_solution/server/lib/exceptions/manifest_entry.ts rename to x-pack/plugins/security_solution/server/endpoint/artifacts/manifest_entry.ts index df9a3b98a4043..4e96af9005287 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/manifest_entry.ts +++ b/x-pack/plugins/security_solution/server/endpoint/artifacts/manifest_entry.ts @@ -10,12 +10,11 @@ import { SavedObjectsClient, SavedObjectsPluginStart } from '../../../../../../s import { ListPluginSetup } from '../../../../lists/server'; -// eslint-disable-next-line @kbn/eslint/no-restricted-paths -import { ExceptionListClient } from '../../../../lists/server/services/exception_lists/exception_list_client'; +import { ExceptionListClient } from '../../../../lists/server'; import { GetFullEndpointExceptionList, CompressExceptionList } from './lists'; -import { ArtifactConstants, ManifestService } from './manifest'; +import { ArtifactConstants } from './common'; export class ManifestEntry { private exceptionListClient: ExceptionListClient; @@ -34,8 +33,12 @@ export class ManifestEntry { this.schemaVersion = schemaVersion; } + public static getArtifactName(os: string, schemaVersion: string) { + return `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-${os}-${schemaVersion}`; + } + public getIdentifier(): string { - return ManifestService.getArtifactName(this.os, this.schemaVersion); + return ManifestEntry.getArtifactName(this.os, this.schemaVersion); } public getUrl(): string { diff --git a/x-pack/plugins/security_solution/server/endpoint/artifacts/manifest_service.ts b/x-pack/plugins/security_solution/server/endpoint/artifacts/manifest_service.ts new file mode 100644 index 0000000000000..923724e32d12e --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/artifacts/manifest_service.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { createHash } from 'crypto'; + +import { SavedObjectsClient } from '../../../../../../src/core/server'; + +import { ManifestConstants } from './common'; +import { InternalManifestSchema } from './schemas'; + +export interface ManifestServiceOptions { + savedObjectsClient: SavedObjectsClient; +} + +export class ManifestService { + + private soClient: SavedObjectsClient; + + constructor(context: ManifestOptions) { + this.soClient = context.savedObjectsClient; + } + + public getManifestId(): string { + return `endpoint-manifest-${ManifestConstants.SCHEMA_VERSION}`; + } + + public async getManifest(): Promise { + return this.soClient.get( + ManifestConstants.SAVED_OBJECT_TYPE, + this.getManifestId(), + ); + } + + public async buildNewManifest(artifacts: InternalArtifactSchema[]): Promise { + // TODO: build and return a manifest from artifacts + } + + public async dispatchAndUpdate(manifest: Manifest) { + // TODO + // 1. Dispatch the manifest + // 2. Update the manifest in ES (ONLY if successful) + } +} diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/saved_object_mappings.ts b/x-pack/plugins/security_solution/server/endpoint/artifacts/saved_object_mappings.ts similarity index 57% rename from x-pack/plugins/security_solution/server/lib/exceptions/saved_object_mappings.ts rename to x-pack/plugins/security_solution/server/endpoint/artifacts/saved_object_mappings.ts index 492c475979023..0c920c4e48856 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/saved_object_mappings.ts +++ b/x-pack/plugins/security_solution/server/endpoint/artifacts/saved_object_mappings.ts @@ -6,15 +6,13 @@ import { SavedObjectsType } from '../../../../../../src/core/server'; -export const exceptionsArtifactSavedObjectType = 'securitySolution-exceptions-artifact'; +export const exceptionsArtifactSavedObjectType = 'securitySolution:endpoint:exceptions-artifact'; +export const manifestSavedObjectType = 'securitySolution:endpoint:artifact-manifest'; export const exceptionsArtifactSavedObjectMappings: SavedObjectsType['mappings'] = { properties: { - // e.g. 'global-whitelist-windows' - name: { - type: 'keyword', - }, - schemaVersion: { + // e.g. 'global-whitelist-windows-1.0.0' + identifier: { type: 'keyword', }, sha256: { @@ -35,9 +33,33 @@ export const exceptionsArtifactSavedObjectMappings: SavedObjectsType['mappings'] }, }; -export const type: SavedObjectsType = { +export const manifestSavedObjectMappings: SavedObjectType['mappings'] = { + properties: { + // manifest sequence number + manifestVersion: { + type: 'long', + }, + // manifest schema version + schemaVersion: { + type: 'keyword', + }, + // array of doc ids + ids: { + type: 'keyword', + }, + }, +} + +export const exceptionsArtifactType: SavedObjectsType = { name: exceptionsArtifactSavedObjectType, hidden: false, // TODO: should these be hidden? namespaceType: 'agnostic', mappings: exceptionsArtifactSavedObjectMappings, }; + +export const manifestType: SavedObjectsType = { + name: manifestSavedObjectType, + hidden: false, // TODO: should these be hidden? + namespaceType: 'agnostic', + mappings: manifestSavedObjectMappings, +}; diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/schemas/common.ts b/x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/common.ts similarity index 94% rename from x-pack/plugins/security_solution/server/lib/exceptions/schemas/common.ts rename to x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/common.ts index cb55c7a61ee57..3090e6e6da48a 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/schemas/common.ts +++ b/x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/common.ts @@ -6,7 +6,7 @@ import * as t from 'io-ts'; -export const artifactName = t.string; +export const identifier = t.string; export const body = t.string; diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/schemas/index.ts b/x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/index.ts similarity index 90% rename from x-pack/plugins/security_solution/server/lib/exceptions/schemas/index.ts rename to x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/index.ts index 23d65a29b52ab..15c2bac89720b 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/schemas/index.ts +++ b/x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/index.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -export * from './artifact_schema'; export * from './common'; export * from './request'; export * from './response'; +export * from './saved_objects'; diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/schemas/manifest_schema.ts b/x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/manifest_schema.ts similarity index 100% rename from x-pack/plugins/security_solution/server/lib/exceptions/schemas/manifest_schema.ts rename to x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/manifest_schema.ts diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/schemas/request/download_artifact_schema.ts b/x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/request/download_artifact_schema.ts similarity index 100% rename from x-pack/plugins/security_solution/server/lib/exceptions/schemas/request/download_artifact_schema.ts rename to x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/request/download_artifact_schema.ts diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/schemas/request/index.ts b/x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/request/index.ts similarity index 100% rename from x-pack/plugins/security_solution/server/lib/exceptions/schemas/request/index.ts rename to x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/request/index.ts diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/schemas/response/download_artifact_schema.ts b/x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/response/download_artifact_schema.ts similarity index 100% rename from x-pack/plugins/security_solution/server/lib/exceptions/schemas/response/download_artifact_schema.ts rename to x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/response/download_artifact_schema.ts diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/schemas/response/index.ts b/x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/response/index.ts similarity index 100% rename from x-pack/plugins/security_solution/server/lib/exceptions/schemas/response/index.ts rename to x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/response/index.ts diff --git a/x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/saved_objects.ts b/x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/saved_objects.ts new file mode 100644 index 0000000000000..0f5a7d45d5453 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/saved_objects.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; +import { body, created, encoding, identifier, sha256, size, } from './common'; + +export const internalArtifactSchema = t.exact( + t.type({ + identifier, + sha256, + encoding, + created, + body, + size, + }) +); + +export type InternalArtifactSchema = t.TypeOf; + +export const internalManifestSchema = t.exact( + t.type({ + manifestVersion, + schemaVersion: manifestSchemaVersion, + artifacts: t.record(artifactName, manifestEntrySchema), + }) +); + +export type InternalManifestSchema = t.TypeOf; diff --git a/x-pack/plugins/security_solution/server/endpoint/artifacts/task.ts b/x-pack/plugins/security_solution/server/endpoint/artifacts/task.ts new file mode 100644 index 0000000000000..09d7e54f5cb55 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/artifacts/task.ts @@ -0,0 +1,166 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { createHash } from 'crypto'; +import { + CoreSetup, + Logger, + SavedObjectsClient, + SavedObjectsPluginStart, +} from '../../../../../../src/core/server'; +import { + ConcreteTaskInstance, + TaskManagerSetupContract, + TaskManagerStartContract, +} from '../../../../../plugins/task_manager/server'; +import { ListPluginSetup } from '../../../../lists/server'; +import { ConfigType } from '../../config'; +import { EndpointAppContext } from '../../types'; +import { ExceptionsCache } from './cache'; +import { GetFullEndpointExceptionList, CompressExceptionList } from './lists'; +import { ArtifactConstants, ManifestService } from './manifest'; +import { ArtifactSoSchema } from './schemas'; + +const PackagerTaskConstants = { + TIMEOUT: '1m', + TYPE: 'securitySolution:endpoint:exceptions-packager', + VERSION: '1.0.0', +}; + +export interface PackagerTask { + getTaskRunner: (context: PackagerTaskRunnerContext) => PackagerTaskRunner; +} + +interface PackagerTaskRunner { + run: () => void; +} + +interface PackagerTaskContext { + endpointAppContext: EndpointAppContext; + taskManager: TaskManagerSetupContract; + lists: ListPluginSetup; + cache: ExceptionsCache; +} + +interface PackagerTaskRunnerContext { + taskManager: TaskManagerStartContract; +} + +export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { + const getTaskId = (): string => { + return `${PackagerTaskConstants.TYPE}:${PackagerTaskConstants.VERSION}`; + }; + + const logger = context.endpointAppContext.logFactory.get(getTaskId()); + + const run = async (taskId: int, state: Record): Record => { + + const artifactService = context.endpointAppContext.service.getArtifactService(); + const manifestService = context.endpointAppContext.service.getManifestService(); + + // Check that this task is current + if (taskId !== getTaskId()) { + // old task, return + logger.debug(`Outdated task running: ${taskId}`); + return; + } + + // TODO + // 1. Pull manifest, note version + // 2. No manifest, do nothing (should have been created on policy create) + // 3. If manifest, pull all associated artifacts (if not already in memory) + // 4. Construct new artifacts from current exception list items + // 5. Any differences? Update manifest, perform conflict check. + // 6. Has this manifest been dispatched? If not, dispatch. + let oldManifest: Manifest; + + try { + oldManifest = await manifestService.getManifest(); + catch (err) { + logger.debug('Manifest not created yet, nothing to do.'); + return; + } + + let artifacts = await artifactService.buildExceptionListArtifacts(); + for (const artifact in artifacts) { + if (!oldManifest.contains(artifact)) { + try { + await artifactService.upsertArtifact( + ArtifactConstants.SAVED_OBJECT_TYPE, + artifact, + { id: artifact.id, overwrite: true }, + ); + } catch (err) { + // Error updating... try again later + logger.error(err); + return; + } + } + } + + artifacts = await artifactService.buildExceptionListArtifacts(); + + const newManifest = buildNewManifest(artifacts); + if (oldManifest.diff(newManifest)) { + try { + await manifestService.dispatchAndUpdate(newManifest); + } catch(err) { + logger.error(err); + return; + } + } + }; + + + const getTaskRunner = (runnerContext: PackagerTaskRunnerContext): PackagerTaskRunner => { + return { + run: async () => { + const taskId = getTaskId(); + try { + await runnerContext.taskManager.ensureScheduled({ + id: taskId, + taskType: PackagerTaskConstants.TYPE, + scope: ['securitySolution'], + state: {}, + params: { version: PackagerTaskConstants.VERSION }, + }); + } catch (e) { + logger.debug(`Error scheduling task, received ${e.message}`); + } + + await runnerContext.taskManager.runNow(taskId); + }, + }; + }; + + context.taskManager.registerTaskDefinitions({ + [PackagerTaskConstants.TYPE]: { + title: 'Security Solution Endpoint Exceptions Handler', + type: PackagerTaskConstants.TYPE, + timeout: PackagerTaskConstants.TIMEOUT, + createTaskRunner: ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => { + return { + run: async () => { + await run(taskInstance.id, taskInstance.state); + + const nextRun = new Date(); + nextRun.setSeconds(nextRun.getSeconds() + context.endpointAppContext.config.packagerTaskInterval); + + return { + state, + runAt: nextRun, + }; + }, + cancel: async () => {}, + }; + }, + }, + }); + + return { + getTaskRunner, + }; +} diff --git a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts index cb8c913a73b8e..ec2c8935d0e91 100644 --- a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts +++ b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ import { AgentService } from '../../../ingest_manager/server'; +import { ArtifactService, ManifestService } from './artifacts'; /** * A singleton that holds shared services that are initialized during the start up phase @@ -11,9 +12,13 @@ import { AgentService } from '../../../ingest_manager/server'; */ export class EndpointAppContextService { private agentService: AgentService | undefined; + private artifactService: ArtifactService | undefined; + private manifestService: ManifestService | undefined; public start(dependencies: { agentService: AgentService }) { this.agentService = dependencies.agentService; + this.artifactService = dependencies.artifactService; + this.manifestService = dependencies.manifestService; } public stop() {} @@ -24,4 +29,18 @@ export class EndpointAppContextService { } return this.agentService; } + + public getArtifactService(): ArtifactService { + if (!this.artifactService) { + throw new Error(`must call start on ${EndpointAppContextService.name} to call getter`); + } + return this.artifactService; + } + + public getManifestService(): ManifestService { + if (!this.manifestService) { + throw new Error(`must call start on ${EndpointAppContextService.name} to call getter`); + } + return this.manifestService; + } } diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_endpoint_exception_list.test.ts similarity index 97% rename from x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.test.ts rename to x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_endpoint_exception_list.test.ts index 96e1b738fce64..2824cfc273e95 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_endpoint_exception_list.test.ts @@ -20,9 +20,9 @@ import { httpServiceMock, httpServerMock, } from 'src/core/server/mocks'; -import { ExceptionsCache } from '../cache'; -import { CompressExceptionList } from '../lists'; -import { ArtifactConstants } from '../manifest'; +import { ExceptionsCache } from '../../artifacts/cache'; +import { CompressExceptionList } from '../../artifacts/lists'; +import { ArtifactConstants } from '../../artifacts/manifest'; import { downloadEndpointExceptionListRoute } from './download_endpoint_exception_list'; const mockArtifactName = `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-windows-1.0.0`; diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_endpoint_exception_list.ts similarity index 91% rename from x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.ts rename to x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_endpoint_exception_list.ts index 325ddafc228f1..7253c19f75902 100644 --- a/x-pack/plugins/security_solution/server/lib/exceptions/routes/download_endpoint_exception_list.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_endpoint_exception_list.ts @@ -5,16 +5,16 @@ */ import { IRouter, KibanaResponse } from 'src/core/server'; -import { validate } from '../../../../common/validate'; -import { buildRouteValidation } from '../../../utils/build_validation/route_validation'; -import { ExceptionsCache } from '../cache'; -import { ArtifactConstants } from '../manifest'; +import { validate } from '../../../../../common/validate'; +import { buildRouteValidation } from '../../../../utils/build_validation/route_validation'; +import { ExceptionsCache } from '../../artifacts/cache'; +import { ArtifactConstants } from '../../artifacts/manifest'; import { ArtifactDownloadSchema, DownloadArtifactRequestParamsSchema, downloadArtifactRequestParamsSchema, downloadArtifactResponseSchema, -} from '../schemas'; +} from '../../artifacts/schemas'; const allowlistBaseRoute: string = '/api/endpoint/allowlist'; diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/routes/index.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/index.ts similarity index 100% rename from x-pack/plugins/security_solution/server/lib/exceptions/routes/index.ts rename to x-pack/plugins/security_solution/server/endpoint/routes/artifacts/index.ts diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/index.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/index.ts new file mode 100644 index 0000000000000..41bc2aa258807 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/index.ts @@ -0,0 +1,5 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/manifest.ts b/x-pack/plugins/security_solution/server/lib/exceptions/manifest.ts deleted file mode 100644 index f4192f19b0716..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/exceptions/manifest.ts +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { createHash } from 'crypto'; - -import { SavedObjectsClient, SavedObjectsPluginStart } from '../../../../../../src/core/server'; - -import { ListPluginSetup } from '../../../../lists/server'; - -// eslint-disable-next-line @kbn/eslint/no-restricted-paths -import { ExceptionListClient } from '../../../../lists/server/services/exception_lists/exception_list_client'; - -import { GetFullEndpointExceptionList, CompressExceptionList } from './lists'; - -import { ManifestEntry } from './manifest_entry'; - -export const ArtifactConstants = { - GLOBAL_ALLOWLIST_NAME: 'endpoint-allowlist', - SAVED_OBJECT_TYPE: 'securitySolution-exceptions-artifact', - SUPPORTED_OPERATING_SYSTEMS: ['linux', 'windows'], - SUPPORTED_SCHEMA_VERSIONS: ['1.0.0'], -}; - -export interface ManifestOptions { - lists: ListPluginSetup; - savedObjects: SavedObjectsPluginStart; -} - -export class ManifestService { - private static instance: ManifestService; - - private entries: ManifestEntry[]; - private soClient: SavedObjectsClient; - private exceptionListClient: ExceptionListClient; - - private constructor(context: ManifestOptions) { - const savedObjectsRepository = context.savedObjects.createInternalRepository(); - this.soClient = new SavedObjectsClient(savedObjectsRepository); - this.exceptionListClient = context.lists.getExceptionListClient(this.soClient, 'kibana'); - this.entries = []; - } - - public static getInstance(context: ManifestOptions) { - // TODO: lock? - if (!ManifestService.instance) { - ManifestService.instance = new ManifestService(context); - } - - return ManifestService.instance; - } - - public static getArtifactName(os: string, schemaVersion: string) { - return `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-${os}-${schemaVersion}`; - } - - public getState(): object { - // TODO: type - return this.entries - .map((entry) => { - return entry.getState(); - }) - .reduce((map, state) => { - map[state.identifier] = { - url: state.url, - sha256: state.sha256, - size: state.size, - }; - return map; - }); - } - - public async refresh() { - this.entries = []; - const entries: ManifestEntry = []; - - for (const os of ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS) { - for (const schemaVersion of ArtifactConstants.SUPPORTED_SCHEMA_VERSIONS) { - const entry = new ManifestEntry(this.exceptionListClient, os, schemaVersion); - - try { - await entry.refresh(); - entries.push(entry); - } catch (err) { - // TODO: context logger? - // console.log(err); - } - } - } - - this.entries = entries; - } -} diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/schemas/artifact_schema.ts b/x-pack/plugins/security_solution/server/lib/exceptions/schemas/artifact_schema.ts deleted file mode 100644 index 5f3d126d84087..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/exceptions/schemas/artifact_schema.ts +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import * as t from 'io-ts'; -import { artifactName, body, created, encoding, schemaVersion, sha256, size } from './common'; - -export const artifactSoSchema = t.exact( - t.type({ - artifactName, - schemaVersion, - sha256, - encoding, - created, - body, - size, - }) -); - -export type ArtifactSoSchema = t.TypeOf; diff --git a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts b/x-pack/plugins/security_solution/server/lib/exceptions/task.ts deleted file mode 100644 index 2ead9d2dfc930..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/exceptions/task.ts +++ /dev/null @@ -1,269 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { createHash } from 'crypto'; -import { - CoreSetup, - Logger, - SavedObjectsClient, - SavedObjectsPluginStart, -} from '../../../../../../src/core/server'; -import { - ConcreteTaskInstance, - TaskManagerSetupContract, - TaskManagerStartContract, -} from '../../../../../plugins/task_manager/server'; -import { ListPluginSetup } from '../../../../lists/server'; -import { ConfigType } from '../../config'; -import { ExceptionsCache } from './cache'; -import { GetFullEndpointExceptionList, CompressExceptionList } from './lists'; -import { ArtifactConstants, ManifestService } from './manifest'; -import { ArtifactSoSchema } from './schemas'; - -const PackagerTaskConstants = { - TIMEOUT: '1m', - TYPE: 'securitySolution:endpoint:exceptions-packager', - VERSION: '1.0.0', -}; - -export interface PackagerTask { - getTaskRunner: (context: PackagerTaskRunnerContext) => PackagerTaskRunner; -} - -interface PackagerTaskRunner { - run: () => void; -} - -interface PackagerTaskContext { - core: CoreSetup; - config: ConfigType; - logger: Logger; - taskManager: TaskManagerSetupContract; - lists: ListPluginSetup; - cache: ExceptionsCache; -} - -interface PackagerTaskRunnerContext { - taskManager: TaskManagerStartContract; -} - -export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { - const getTaskId = (): string => { - return `${PackagerTaskConstants.TYPE}:${PackagerTaskConstants.VERSION}`; - }; - - const run = async (taskId: int, state: Record): Record => { - // console.log(taskState); - // const state = Object.assign({}, ...taskState); - - // Check that this task is current - if (taskId !== getTaskId()) { - // old task, return - context.logger.debug(`Outdated task running: ${taskId}`); - return state; - } - - // Update manifest and return state - const [{ savedObjects }] = await context.core.getStartServices(); - const manifest = ManifestService.getInstance({ - lists: context.lists, - savedObjects, - }); - - try { - await manifest.refresh(); - } catch (err) { - context.logger.error(err); - } - - return manifest.getState(); - }; - - /* - // Get clients - const [{ savedObjects }] = await context.core.getStartServices(); - const savedObjectsRepository = savedObjects.createInternalRepository(); - const soClient = new SavedObjectsClient(savedObjectsRepository); - const exceptionListClient = context.lists.getExceptionListClient(soClient, 'kibana'); - - // Main loop - let updated = false; - - for (const os of ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS) { - for (const schemaVersion of ArtifactConstants.SUPPORTED_SCHEMA_VERSIONS) { - const artifactName = `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-${os}-${schemaVersion}`; - - // Initialize state and prime cache - if (state[artifactName] === undefined) { - try { - const soGetResponse = await soClient.get( - ArtifactConstants.SAVED_OBJECT_TYPE, - artifactName - ); - - const cacheKey = `${artifactName}-${soGetResponse.attributes.sha256}`; - context.cache.set(cacheKey, soGetResponse.attributes.body); - - state[artifactName] = soGetResponse.attributes.sha256; - - updated = true; - } catch (err) { - context.logger.debug(`No artifact found ${artifactName} -- cache not primed`); - } - } - - try { - // Retrieve exceptions, compute hash - const exceptions = await GetFullEndpointExceptionList( - exceptionListClient, - os, - schemaVersion - ); - const compressedExceptions: Buffer = await CompressExceptionList(exceptions); - - const sha256Hash = createHash('sha256') - .update(compressedExceptions.toString('utf8'), 'utf8') - .digest('hex'); - - const exceptionSO = { - name: artifactName, - schemaVersion, - sha256: sha256Hash, - encoding: 'xz', - created: Date.now(), - body: compressedExceptions.toString('binary'), - size: Buffer.from(JSON.stringify(exceptions)).byteLength, - }; - - // Create/update the artifact - const soResponse = await soClient.create( - ArtifactConstants.SAVED_OBJECT_TYPE, - exceptionSO, - { id: `${artifactName}`, overwrite: true } - ); - - // If new, update state - if (state[artifactName] !== soResponse.attributes.sha256) { - context.logger.info( - `Change to artifact[${artifactName}] detected hash[${soResponse.attributes.sha256}]` - ); - - state[artifactName] = soResponse.attributes.sha256; - - updated = true; - } - - // Update the cache - const cacheKey = `${artifactName}-${sha256Hash}`; - // TODO: does this reset the ttl? - context.cache.set(cacheKey, compressedExceptions.toString('binary')); - } catch (error) { - context.logger.error(error); - } - } - } - - // Update manifest if there are changes - if (updated) { - context.logger.debug('One or more changes detected. Updating manifest...'); - // TODO: update manifest - } - - return state; - }; - */ - - const getContext = (): PackagerTaskContext => { - return context; - }; - - const getTaskRunner = (runnerContext: PackagerTaskRunnerContext): PackagerTaskRunner => { - return { - run: async () => { - const taskId = getTaskId(); - try { - await runnerContext.taskManager.ensureScheduled({ - id: taskId, - taskType: PackagerTaskConstants.TYPE, - scope: ['securitySolution'], - state: {}, - params: { version: PackagerTaskConstants.VERSION }, - }); - } catch (e) { - context.logger.debug(`Error scheduling task, received ${e.message}`); - } - - await runnerContext.taskManager.runNow(taskId); - }, - }; - }; - - context.taskManager.registerTaskDefinitions({ - [PackagerTaskConstants.TYPE]: { - title: 'Security Solution Endpoint Exceptions Handler', - type: PackagerTaskConstants.TYPE, - timeout: PackagerTaskConstants.TIMEOUT, - createTaskRunner: ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => { - return { - run: async () => { - const state = await run(taskInstance.id, taskInstance.state); - - // Get clients - const [{ savedObjects }] = await context.core.getStartServices(); - const savedObjectsRepository = savedObjects.createInternalRepository(); - const soClient = new SavedObjectsClient(savedObjectsRepository); - - // Create/update the artifacts - let updated = false; - for (const id in state) { - if (Object.prototype.hasOwnProperty.call(state, id)) { - const soResponse = await soClient.create( - ArtifactConstants.SAVED_OBJECT_TYPE, - state[id].getSavedObject(), - { id, overwrite: true } - ); - - // If new, update state - if (state[id].sha256 !== soResponse.attributes.sha256) { - context.logger.info( - `Change to artifact[${id}] detected hash[${soResponse.attributes.sha256}]` - ); - - updated = true; - } - - // TODO: Update the cache - /* - const cacheKey = `${id}-${soResponse.attributes.sha256}`; - // TODO: does this reset the ttl? - context.cache.set(cacheKey, compressedExceptions.toString('binary')); - */ - } - } - - if (updated) { - // TODO: update manifest in config - } - - const nextRun = new Date(); - nextRun.setSeconds(nextRun.getSeconds() + context.config.packagerTaskInterval); - - return { - state, - runAt: nextRun, - }; - }, - cancel: async () => {}, - }; - }, - }, - }); - - return { - getContext, - getTaskRunner, - }; -} diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index ec8be339267ff..e9250cd4eac38 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -11,9 +11,10 @@ import { i18n } from '@kbn/i18n'; import { CoreSetup, CoreStart, + Logger, Plugin as IPlugin, PluginInitializerContext, - Logger, + SavedObjectsClient, } from '../../../../src/core/server'; import { PluginSetupContract as AlertingSetup } from '../../alerts/server'; import { SecurityPluginSetup as SecuritySetup } from '../../security/server'; @@ -33,7 +34,7 @@ import { signalRulesAlertType } from './lib/detection_engine/signals/signal_rule import { rulesNotificationAlertType } from './lib/detection_engine/notifications/rules_notification_alert_type'; import { isNotificationAlertExecutor } from './lib/detection_engine/notifications/types'; import { hasListsFeature, listsEnvFeatureFlagName } from './lib/detection_engine/feature_flags'; -import { PackagerTask, setupPackagerTask, ExceptionsCache } from './lib/exceptions'; +import { PackagerTask, setupPackagerTask, ExceptionsCache } from './endpoint/artifacts'; import { initSavedObjects, savedObjectTypes } from './saved_objects'; import { AppClientFactory } from './client'; import { createConfig$, ConfigType } from './config'; @@ -43,9 +44,10 @@ import { registerEndpointRoutes } from './endpoint/routes/metadata'; import { registerResolverRoutes } from './endpoint/routes/resolver'; import { registerAlertRoutes } from './endpoint/alerts/routes'; import { registerPolicyRoutes } from './endpoint/routes/policy'; +import { ArtifactService } from './endpoint/artifact_services'; import { EndpointAppContextService } from './endpoint/endpoint_app_context_services'; import { EndpointAppContext } from './endpoint/types'; -import { downloadEndpointExceptionListRoute } from './lib/exceptions/routes/download_endpoint_exception_list'; +import { downloadEndpointExceptionListRoute } from './endpoint/routes/artifacts/download_endpoint_exception_list'; export interface SetupPlugins { alerts: AlertingSetup; @@ -75,6 +77,9 @@ export class Plugin implements IPlugin type.name); From 3d325e2c7843527eef6ec75518f8bd8d647d74ed Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Sat, 20 Jun 2020 22:06:06 -0400 Subject: [PATCH 051/106] Move to endpoint directory --- .../endpoint/alerts/handlers/alerts.test.ts | 10 +- .../server/endpoint/artifact_services.ts | 32 ----- .../endpoint/artifacts/artifact_service.ts | 64 ---------- .../server/endpoint/artifacts/manifest.ts | 50 -------- .../endpoint/artifacts/manifest_entry.ts | 87 ------------- .../endpoint/artifacts/manifest_service.ts | 46 ------- .../server/endpoint/config.ts | 5 + .../endpoint_app_context_services.test.ts | 10 +- .../endpoint/endpoint_app_context_services.ts | 16 ++- .../{ => lib}/artifacts/cache.test.ts | 0 .../endpoint/{ => lib}/artifacts/cache.ts | 0 .../endpoint/{ => lib}/artifacts/common.ts | 6 +- .../endpoint/{ => lib}/artifacts/index.ts | 4 +- .../{ => lib}/artifacts/lists.test.ts | 8 +- .../endpoint/{ => lib}/artifacts/lists.ts | 4 +- .../{ => lib}/artifacts/manifest.test.ts | 0 .../server/endpoint/lib/artifacts/manifest.ts | 96 +++++++++++++++ .../endpoint/lib/artifacts/manifest_entry.ts | 47 ++++++++ .../server/endpoint/lib/artifacts/refresh.ts | 114 ++++++++++++++++++ .../artifacts/saved_object_mappings.ts | 16 ++- .../endpoint/{ => lib}/artifacts/task.ts | 75 ++---------- .../server/endpoint/lib/index.ts | 5 + .../server/endpoint/mocks.ts | 32 +++++ ...est.ts => download_exception_list.test.ts} | 10 +- ...ion_list.ts => download_exception_list.ts} | 34 +++--- .../server/endpoint/routes/artifacts/index.ts | 2 +- .../schemas => schemas/artifacts}/common.ts | 4 +- .../schemas => schemas/artifacts}/index.ts | 1 + .../schemas/artifacts/manifest_schema.mock.ts | 5 + .../artifacts}/manifest_schema.ts | 18 +-- .../request/download_artifact_schema.ts | 4 +- .../artifacts}/request/index.ts | 0 .../response/download_artifact_schema.ts | 0 .../artifacts}/response/index.ts | 0 .../schemas/artifacts/saved_objects.mock.ts | 5 + .../artifacts}/saved_objects.ts | 7 +- .../server/endpoint/schemas/index.ts | 7 ++ .../services/artifacts/artifact_service.ts | 34 ++++++ .../endpoint/services/artifacts/index.ts | 3 + .../services/artifacts/manifest_service.ts | 78 ++++++++++++ .../server/endpoint/services/index.ts | 7 ++ .../routes/__mocks__/index.ts | 1 + .../security_solution/server/plugin.ts | 32 ++--- .../security_solution/server/saved_objects.ts | 5 +- 44 files changed, 561 insertions(+), 423 deletions(-) delete mode 100644 x-pack/plugins/security_solution/server/endpoint/artifact_services.ts delete mode 100644 x-pack/plugins/security_solution/server/endpoint/artifacts/artifact_service.ts delete mode 100644 x-pack/plugins/security_solution/server/endpoint/artifacts/manifest.ts delete mode 100644 x-pack/plugins/security_solution/server/endpoint/artifacts/manifest_entry.ts delete mode 100644 x-pack/plugins/security_solution/server/endpoint/artifacts/manifest_service.ts rename x-pack/plugins/security_solution/server/endpoint/{ => lib}/artifacts/cache.test.ts (100%) rename x-pack/plugins/security_solution/server/endpoint/{ => lib}/artifacts/cache.ts (100%) rename x-pack/plugins/security_solution/server/endpoint/{ => lib}/artifacts/common.ts (78%) rename x-pack/plugins/security_solution/server/endpoint/{ => lib}/artifacts/index.ts (82%) rename x-pack/plugins/security_solution/server/endpoint/{ => lib}/artifacts/lists.test.ts (86%) rename x-pack/plugins/security_solution/server/endpoint/{ => lib}/artifacts/lists.ts (93%) rename x-pack/plugins/security_solution/server/endpoint/{ => lib}/artifacts/manifest.test.ts (100%) create mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/artifacts/refresh.ts rename x-pack/plugins/security_solution/server/endpoint/{ => lib}/artifacts/saved_object_mappings.ts (75%) rename x-pack/plugins/security_solution/server/endpoint/{ => lib}/artifacts/task.ts (53%) create mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/index.ts rename x-pack/plugins/security_solution/server/endpoint/routes/artifacts/{download_endpoint_exception_list.test.ts => download_exception_list.test.ts} (94%) rename x-pack/plugins/security_solution/server/endpoint/routes/artifacts/{download_endpoint_exception_list.ts => download_exception_list.ts} (74%) rename x-pack/plugins/security_solution/server/endpoint/{artifacts/schemas => schemas/artifacts}/common.ts (82%) rename x-pack/plugins/security_solution/server/endpoint/{artifacts/schemas => schemas/artifacts}/index.ts (91%) create mode 100644 x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/manifest_schema.mock.ts rename x-pack/plugins/security_solution/server/endpoint/{artifacts/schemas => schemas/artifacts}/manifest_schema.ts (74%) rename x-pack/plugins/security_solution/server/endpoint/{artifacts/schemas => schemas/artifacts}/request/download_artifact_schema.ts (87%) rename x-pack/plugins/security_solution/server/endpoint/{artifacts/schemas => schemas/artifacts}/request/index.ts (100%) rename x-pack/plugins/security_solution/server/endpoint/{artifacts/schemas => schemas/artifacts}/response/download_artifact_schema.ts (100%) rename x-pack/plugins/security_solution/server/endpoint/{artifacts/schemas => schemas/artifacts}/response/index.ts (100%) create mode 100644 x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts rename x-pack/plugins/security_solution/server/endpoint/{artifacts/schemas => schemas/artifacts}/saved_objects.ts (72%) create mode 100644 x-pack/plugins/security_solution/server/endpoint/schemas/index.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_service.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_service.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/index.ts diff --git a/x-pack/plugins/security_solution/server/endpoint/alerts/handlers/alerts.test.ts b/x-pack/plugins/security_solution/server/endpoint/alerts/handlers/alerts.test.ts index fd785bca4aa24..05a27653ca56d 100644 --- a/x-pack/plugins/security_solution/server/endpoint/alerts/handlers/alerts.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/alerts/handlers/alerts.test.ts @@ -11,7 +11,12 @@ import { } from '../../../../../../../src/core/server/mocks'; import { registerAlertRoutes } from '../routes'; import { alertingIndexGetQuerySchema } from '../../../../common/endpoint_alerts/schema/alert_index'; -import { createMockAgentService } from '../../mocks'; +import { + createMockAgentService, + createMockArtifactService, + createMockManifestService, + createMockExceptionListClient, +} from '../../mocks'; import { EndpointAppContextService } from '../../endpoint_app_context_services'; import { createMockConfig } from '../../../lib/detection_engine/routes/__mocks__'; @@ -30,6 +35,9 @@ describe('test alerts route', () => { endpointAppContextService = new EndpointAppContextService(); endpointAppContextService.start({ agentService: createMockAgentService(), + artifactService: createMockArtifactService(), + manifestService: createMockManifestService(), + exceptionListClient: createMockExceptionListClient(), }); registerAlertRoutes(routerMock, { diff --git a/x-pack/plugins/security_solution/server/endpoint/artifact_services.ts b/x-pack/plugins/security_solution/server/endpoint/artifact_services.ts deleted file mode 100644 index 9ce59f772d850..0000000000000 --- a/x-pack/plugins/security_solution/server/endpoint/artifact_services.ts +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -import { SavedObjectsClient } from '../../../../../src/core/server'; -import { ExceptionListClient } from '../../../lists'; -import { EndpointAppContextService } from './endpoint_app_context_services'; -import { Manifest } from './artifacts/manifest'; - -/** - * A singleton that holds shared services for working with artifacts. - */ -export class ArtifactService { - private endpointAppContextService: EndpointAppContextService | undefined; - private exceptionListClient: ExceptionListClient | undefined; - private savedObjectsClient: SavedObjectsClient | undefined; - - private manifest: Manifest | undefined; - - public start(deps: { - endpointAppContextService: EndpointAppContextService, - exceptionListClient; ExceptionListClient, - savedObjectsClient: SavedObjectsClient, - }) { - this.endpointAppContextService = deps.endpointAppContextService; - this.exceptionListClient = deps.exceptionListClient; - this.savedObjectsClient = deps.savedObjectsClient; - } - - public stop() {} -} diff --git a/x-pack/plugins/security_solution/server/endpoint/artifacts/artifact_service.ts b/x-pack/plugins/security_solution/server/endpoint/artifacts/artifact_service.ts deleted file mode 100644 index d64b14ba10dc8..0000000000000 --- a/x-pack/plugins/security_solution/server/endpoint/artifacts/artifact_service.ts +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { createHash } from 'crypto'; - -import { SavedObjectsClient } from '../../../../../../src/core/server'; -import { ExceptionListClient } from '../../../../lists/server'; - -import { ArtifactConstants } from './common'; -import { CompressExceptionList, GetFullEndpointExceptionList } from './lists'; -import { InternalArtifactSchema } from './schemas'; - -export interface ArtifactServiceOptions { - savedObjectsClient: SavedObjectsClient; -} - -export class ArtifactService { - - private soClient: SavedObjectsClient; - private exceptionListClient: ExceptionListClient; - - constructor(context: ManifestOptions) { - this.soClient = context.savedObjectsClient; - this.exceptionListClient = context.exceptionListClient; - } - - public async buildExceptionListArtifacts(): Promise> { - const artifacts: InternalArtifactSchema = []; - - for (const os of ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS) { - for (const schemaVersion of ArtifactConstants.SUPPORTED_SCHEMA_VERSIONS) { - const exceptions = await GetFullEndpointExceptionList( - this.exceptionListClient, - os, - schemaVersion, - ); - - const compressedExceptions: Buffer = await CompressExceptionList(exceptions); - - const sha256 = createHash('sha256') - .update(compressedExceptions.toString('utf8'), 'utf8') - .digest('hex'); - - artifacts.push({ - identifier: this.getIdentifier(), - sha256, - encoding: 'xz', - created: Date.now(), - body: compressedExceptions.toString('binary'), - size: Buffer.from(JSON.stringify(exceptions)).byteLength, - }); - } - } - - return artifacts; - } - - public async upsertArtifact(artifact: InternalArtifactSchema) { - // TODO - } -} diff --git a/x-pack/plugins/security_solution/server/endpoint/artifacts/manifest.ts b/x-pack/plugins/security_solution/server/endpoint/artifacts/manifest.ts deleted file mode 100644 index 49a705354c8e3..0000000000000 --- a/x-pack/plugins/security_solution/server/endpoint/artifacts/manifest.ts +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { createHash } from 'crypto'; - -import { SavedObjectsClient, SavedObjectsPluginStart } from '../../../../../../src/core/server'; - -import { ListPluginSetup } from '../../../../lists/server'; - -import { ExceptionListClient } from '../../../../lists/server'; - -import { GetFullEndpointExceptionList, CompressExceptionList } from './lists'; - -import { ManifestEntry } from './manifest_entry'; - -import { ArtifactConstants } from './common'; - -export class Manifest { - private entries: ManifestEntry[]; - - constructor(context: ManifestOptions) { - this.entries = []; - } - - public getState(): object { - // TODO: type - return this.entries - .map((entry) => { - return entry.getState(); - }) - .reduce((map, state) => { - map[state.identifier] = { - url: state.url, - sha256: state.sha256, - size: state.size, - }; - return map; - }); - } - - public contains(artifact: InternalArtifactSchema): boolean { - // TODO - // calculate id (identifier + sha256) - // look up id - // return if found - } -} diff --git a/x-pack/plugins/security_solution/server/endpoint/artifacts/manifest_entry.ts b/x-pack/plugins/security_solution/server/endpoint/artifacts/manifest_entry.ts deleted file mode 100644 index 4e96af9005287..0000000000000 --- a/x-pack/plugins/security_solution/server/endpoint/artifacts/manifest_entry.ts +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { createHash } from 'crypto'; - -import { SavedObjectsClient, SavedObjectsPluginStart } from '../../../../../../src/core/server'; - -import { ListPluginSetup } from '../../../../lists/server'; - -import { ExceptionListClient } from '../../../../lists/server'; - -import { GetFullEndpointExceptionList, CompressExceptionList } from './lists'; - -import { ArtifactConstants } from './common'; - -export class ManifestEntry { - private exceptionListClient: ExceptionListClient; - - private os: string; - private schemaVersion: string; - - private sha256: string; - private size: int; - - private savedObject: object; // TODO: type - - constructor(exceptionListClient: ExceptionListClient, os: string, schemaVersion: string) { - this.exceptionListClient = exceptionListClient; - this.os = os; - this.schemaVersion = schemaVersion; - } - - public static getArtifactName(os: string, schemaVersion: string) { - return `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-${os}-${schemaVersion}`; - } - - public getIdentifier(): string { - return ManifestEntry.getArtifactName(this.os, this.schemaVersion); - } - - public getUrl(): string { - return `/api/endpoint/allowlist/download/${this.getIdentifier()}/${this.sha256}`; - } - - public getSavedObject(): object { - return this.savedObject; - } - - public getState(): object { - // TODO: type - return { - identifier: this.getIdentifier(), - url: this.getUrl(), - sha256: this.sha256, - size: this.size, - }; - } - - public async refresh() { - const exceptions = await GetFullEndpointExceptionList( - this.exceptionListClient, - this.os, - this.schemaVersion - ); - const compressedExceptions: Buffer = await CompressExceptionList(exceptions); - - const sha256Hash = createHash('sha256') - .update(compressedExceptions.toString('utf8'), 'utf8') - .digest('hex'); - - this.sha256 = sha256Hash; - this.size = Buffer.from(JSON.stringify(exceptions)).byteLength; - - this.savedObject = { - name: this.getIdentifier(), - schemaVersion: this.schemaVersion, - sha256: sha256Hash, - encoding: 'xz', - created: Date.now(), - body: compressedExceptions.toString('binary'), - size: Buffer.from(JSON.stringify(exceptions)).byteLength, - }; - } -} diff --git a/x-pack/plugins/security_solution/server/endpoint/artifacts/manifest_service.ts b/x-pack/plugins/security_solution/server/endpoint/artifacts/manifest_service.ts deleted file mode 100644 index 923724e32d12e..0000000000000 --- a/x-pack/plugins/security_solution/server/endpoint/artifacts/manifest_service.ts +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { createHash } from 'crypto'; - -import { SavedObjectsClient } from '../../../../../../src/core/server'; - -import { ManifestConstants } from './common'; -import { InternalManifestSchema } from './schemas'; - -export interface ManifestServiceOptions { - savedObjectsClient: SavedObjectsClient; -} - -export class ManifestService { - - private soClient: SavedObjectsClient; - - constructor(context: ManifestOptions) { - this.soClient = context.savedObjectsClient; - } - - public getManifestId(): string { - return `endpoint-manifest-${ManifestConstants.SCHEMA_VERSION}`; - } - - public async getManifest(): Promise { - return this.soClient.get( - ManifestConstants.SAVED_OBJECT_TYPE, - this.getManifestId(), - ); - } - - public async buildNewManifest(artifacts: InternalArtifactSchema[]): Promise { - // TODO: build and return a manifest from artifacts - } - - public async dispatchAndUpdate(manifest: Manifest) { - // TODO - // 1. Dispatch the manifest - // 2. Update the manifest in ES (ONLY if successful) - } -} diff --git a/x-pack/plugins/security_solution/server/endpoint/config.ts b/x-pack/plugins/security_solution/server/endpoint/config.ts index 908e14468c5c7..d514f08326ad0 100644 --- a/x-pack/plugins/security_solution/server/endpoint/config.ts +++ b/x-pack/plugins/security_solution/server/endpoint/config.ts @@ -27,6 +27,11 @@ export const EndpointConfigSchema = schema.object({ from: schema.string({ defaultValue: 'now-15m' }), to: schema.string({ defaultValue: 'now' }), }), + + /** + * Artifact Configuration + */ + packagerTaskInterval: schema.number({ defaultValue: 60 }), }); export function createConfig$(context: PluginInitializerContext) { diff --git a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts index 8cf2ada9907d3..abc9a06497817 100644 --- a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts @@ -6,8 +6,16 @@ import { EndpointAppContextService } from './endpoint_app_context_services'; describe('test endpoint app context services', () => { - it('should throw error if start is not called', async () => { + it('should throw error on getAgentService if start is not called', async () => { const endpointAppContextService = new EndpointAppContextService(); expect(() => endpointAppContextService.getAgentService()).toThrow(Error); }); + it('should throw error on getArtifactService if start is not called', async () => { + const endpointAppContextService = new EndpointAppContextService(); + expect(() => endpointAppContextService.getArtifactService()).toThrow(Error); + }); + it('should throw error on getManifestService if start is not called', async () => { + const endpointAppContextService = new EndpointAppContextService(); + expect(() => endpointAppContextService.getManifestService()).toThrow(Error); + }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts index ec2c8935d0e91..a8df2dd5207dd 100644 --- a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts +++ b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts @@ -4,7 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ import { AgentService } from '../../../ingest_manager/server'; -import { ArtifactService, ManifestService } from './artifacts'; +import { ExceptionListClient } from '../../../lists/server'; +import { ArtifactService, ManifestService } from './services'; /** * A singleton that holds shared services that are initialized during the start up phase @@ -14,11 +15,18 @@ export class EndpointAppContextService { private agentService: AgentService | undefined; private artifactService: ArtifactService | undefined; private manifestService: ManifestService | undefined; + private exceptionListClient: ExceptionListClient | undefined; - public start(dependencies: { agentService: AgentService }) { + public start(dependencies: { + agentService: AgentService; + artifactService: ArtifactService; + manifestService: ManifestService; + exceptionListClient: ExceptionListClient | undefined; + }) { this.agentService = dependencies.agentService; this.artifactService = dependencies.artifactService; this.manifestService = dependencies.manifestService; + this.exceptionListClient = dependencies.exceptionListClient; } public stop() {} @@ -43,4 +51,8 @@ export class EndpointAppContextService { } return this.manifestService; } + + public getExceptionListClient(): ExceptionListClient | undefined { + return this.exceptionListClient; + } } diff --git a/x-pack/plugins/security_solution/server/endpoint/artifacts/cache.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts similarity index 100% rename from x-pack/plugins/security_solution/server/endpoint/artifacts/cache.test.ts rename to x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts diff --git a/x-pack/plugins/security_solution/server/endpoint/artifacts/cache.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts similarity index 100% rename from x-pack/plugins/security_solution/server/endpoint/artifacts/cache.ts rename to x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts diff --git a/x-pack/plugins/security_solution/server/endpoint/artifacts/common.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts similarity index 78% rename from x-pack/plugins/security_solution/server/endpoint/artifacts/common.ts rename to x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts index 92ff54e73fcf6..a4d18cb0cff7d 100644 --- a/x-pack/plugins/security_solution/server/endpoint/artifacts/common.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts @@ -8,10 +8,10 @@ export const ArtifactConstants = { GLOBAL_ALLOWLIST_NAME: 'endpoint-allowlist', SAVED_OBJECT_TYPE: 'securitySolution:endpoint:exceptions-artifact', SUPPORTED_OPERATING_SYSTEMS: ['linux', 'windows'], - SUPPORTED_SCHEMA_VERSIONS: ['1.0.0'], + SCHEMA_VERSION: '1.0.0', }; export const ManifestConstants = { - SAVED_OBJECT_TYPE: 'securitySolution:endpoint:manifest-artifact', - SCHEMA_VERSION = '1.0.0'; + SAVED_OBJECT_TYPE: 'securitySolution:endpoint:exceptions-manifest', + SCHEMA_VERSION: '1.0.0', }; diff --git a/x-pack/plugins/security_solution/server/endpoint/artifacts/index.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/index.ts similarity index 82% rename from x-pack/plugins/security_solution/server/endpoint/artifacts/index.ts rename to x-pack/plugins/security_solution/server/endpoint/lib/artifacts/index.ts index 4a2522baa3a1c..ee7d44459aa38 100644 --- a/x-pack/plugins/security_solution/server/endpoint/artifacts/index.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/index.ts @@ -4,9 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ -export * from './artifact_service'; export * from './cache'; +export * from './common'; export * from './lists'; export * from './manifest'; -export * from './manifest_service'; +export * from './manifest_entry'; export * from './task'; diff --git a/x-pack/plugins/security_solution/server/endpoint/artifacts/lists.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts similarity index 86% rename from x-pack/plugins/security_solution/server/endpoint/artifacts/lists.test.ts rename to x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts index 285cdcbf5942a..9add8ec93c3c7 100644 --- a/x-pack/plugins/security_solution/server/endpoint/artifacts/lists.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts @@ -4,10 +4,10 @@ * you may not use this file except in compliance with the Elastic License. */ -import { ExceptionListClient } from '../../../../lists/server/services/exception_lists/exception_list_client'; -import { listMock } from '../../../../lists/server/mocks'; -import { getFoundExceptionListItemSchemaMock } from '../../../../lists/common/schemas/response/found_exception_list_item_schema.mock'; -import { getExceptionListItemSchemaMock } from '../../../../lists/common/schemas/response/exception_list_item_schema.mock'; +import { ExceptionListClient } from '../../../../../lists/server'; +import { listMock } from '../../../../../lists/server/mocks'; +import { getFoundExceptionListItemSchemaMock } from '../../../../../lists/common/schemas/response/found_exception_list_item_schema.mock'; +import { getExceptionListItemSchemaMock } from '../../../../../lists/common/schemas/response/exception_list_item_schema.mock'; import { GetFullEndpointExceptionList } from './lists'; describe('buildEventTypeSignal', () => { diff --git a/x-pack/plugins/security_solution/server/endpoint/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts similarity index 93% rename from x-pack/plugins/security_solution/server/endpoint/artifacts/lists.ts rename to x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts index 56a8eeaa61c71..50816d6676ebf 100644 --- a/x-pack/plugins/security_solution/server/endpoint/artifacts/lists.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts @@ -5,8 +5,8 @@ */ import lzma from 'lzma-native'; -import { FoundExceptionListItemSchema } from '../../../../lists/common/schemas/response/found_exception_list_item_schema'; -import { ExceptionListClient } from '../../../../lists/server'; +import { FoundExceptionListItemSchema } from '../../../../../lists/common/schemas/response/found_exception_list_item_schema'; +import { ExceptionListClient } from '../../../../../lists/server'; export interface EndpointExceptionList { exceptions_list: ExceptionsList[]; diff --git a/x-pack/plugins/security_solution/server/endpoint/artifacts/manifest.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts similarity index 100% rename from x-pack/plugins/security_solution/server/endpoint/artifacts/manifest.test.ts rename to x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts new file mode 100644 index 0000000000000..b98ac371fb813 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ManifestEntry } from './manifest_entry'; +import { + InternalArtifactSchema, + InternalManifestSchema, + ManifestSchema, + ManifestSchemaVersion, +} from '../../schemas/artifacts'; + +export class Manifest { + private entries: ManifestEntry[]; + private schemaVersion: string; + + // For concurrency control + private version: string | undefined; + + constructor(schemaVersion: string) { + this.entries = []; + this.schemaVersion = schemaVersion; + } + + public static fromArtifacts( + artifacts: InternalArtifactSchema[], + schemaVersion: string + ): Manifest { + const manifest = new Manifest(schemaVersion); + artifacts.forEach((artifact) => { + manifest.addEntry(artifact); + }); + return manifest; + } + + public getSchemaVersion(): string { + return this.schemaVersion; + } + + public getVersion(): string | undefined { + return this.version; + } + + public setVersion(version: string | undefined) { + this.version = version; + } + + public addEntry(artifact: InternalArtifactSchema) { + this.entries.push(new ManifestEntry(artifact)); + } + + public contains(artifact: InternalArtifactSchema): boolean { + for (const entry of this.entries) { + if (artifact.identifier === entry.getIdentifier() && artifact.sha256 === entry.getSha256()) { + return true; + } + } + return false; + } + + // Returns true if same + public equals(manifest: Manifest): boolean { + if (manifest.entries.length !== this.entries.length) { + return false; + } + + for (const entry of this.entries) { + if (!manifest.contains(entry.getArtifact())) { + return false; + } + } + + return true; + } + + public toEndpointFormat(): ManifestSchema { + const manifestObj: object = { + manifestVersion: 'todo', + schemaVersion: 'todo', + artifacts: {}, + }; + this.entries.forEach((entry) => { + manifestObj.artifacts[entry.getIdentifier()] = entry.getRecord(); + }); + return manifestObj as ManifestSchema; + } + + public toSavedObject(): InternalManifestSchema { + return { + schemaVersion: this.schemaVersion as ManifestSchemaVersion, + ids: this.entries.map((entry) => entry.getDocId()), + }; + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts new file mode 100644 index 0000000000000..d90ae54d89f83 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { InternalArtifactSchema, ManifestEntrySchema } from '../../schemas/artifacts'; + +export class ManifestEntry { + private artifact: InternalArtifactSchema; + + constructor(artifact: InternalArtifactSchema) { + this.artifact = artifact; + } + + public getDocId(): string { + return `${this.getIdentifier()}-${this.getSha256()}`; + } + + public getIdentifier(): string { + return this.artifact.identifier; + } + + public getSha256(): string { + return this.artifact.sha256; + } + + public getSize(): number { + return this.artifact.size; + } + + public getUrl(): string { + return `/api/endpoint/allowlist/download/${this.getIdentifier()}/${this.getSha256()}`; + } + + public getArtifact(): InternalArtifactSchema { + return this.artifact; + } + + public getRecord(): ManifestEntrySchema { + return { + url: this.getUrl(), + sha256: this.getSha256(), + size: this.getSize(), + }; + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/refresh.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/refresh.ts new file mode 100644 index 0000000000000..c53ccff82e8ff --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/refresh.ts @@ -0,0 +1,114 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +/* +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { createHash } from 'crypto'; + +import { ExceptionListClient } from '../../../../../lists/server'; + +import { InternalArtifactSchema } from '../../schemas/artifacts'; +import { EndpointAppContext } from '../../types'; + +import { ArtifactConstants, ManifestConstants } from './common'; +import { GetFullEndpointExceptionList, CompressExceptionList } from './lists'; +import { Manifest } from './manifest'; + +const buildExceptionListArtifacts = async ( + schemaVersion: string, + exceptionListClient: ExceptionListClient +): Promise => { + const artifacts: InternalArtifactSchema[] = []; + + for (const os of ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS) { + const exceptions = await GetFullEndpointExceptionList(exceptionListClient, os, schemaVersion); + + const compressedExceptions: Buffer = await CompressExceptionList(exceptions); + + const sha256 = createHash('sha256') + .update(compressedExceptions.toString('utf8'), 'utf8') + .digest('hex'); + + artifacts.push({ + identifier: `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-${os}-${schemaVersion}`, + sha256, + encoding: 'xz', + created: Date.now(), + body: compressedExceptions.toString('binary'), + size: Buffer.from(JSON.stringify(exceptions)).byteLength, + }); + } + + return artifacts; +}; + +export const refreshManifest = async (context: EndpointAppContext, createInitial: boolean) => { + const exceptionListClient = context.service.getExceptionListClient(); + const logger = context.logFactory.get('endpoint-artifact-refresh'); + + if (exceptionListClient === undefined) { + logger.debug('Lists plugin not available. Unable to refresh manifest.'); + return; + } + + const artifactService = context.service.getArtifactService(); + const manifestService = context.service.getManifestService(); + + let oldManifest: Manifest; + + try { + oldManifest = await manifestService.getManifest(ManifestConstants.SCHEMA_VERSION); + } catch (err) { + // TODO: does this need to be more specific? + // 1. could fail pulling artifacts + // 2. could fail if manifest does not exist yet + if (createInitial) { + // TODO: implement this when ready to integrate with Paul's code + oldManifest = new Manifest(ManifestConstants.SCHEMA_VERSION); // create empty manifest + } else { + throw err; + } + } + + const artifacts = await buildExceptionListArtifacts( + ArtifactConstants.SCHEMA_VERSION, + exceptionListClient + ); + artifacts.forEach(async (artifact: InternalArtifactSchema) => { + if (!oldManifest.contains(artifact)) { + try { + await artifactService.createArtifact(artifact); + } catch (err) { + if (err.status === 409) { + // This artifact already existed... + logger.debug( + `Tried to create artifact ${artifact.identifier}-${artifact.sha256}, but it already exists.` + ); + } else { + throw err; + } + } + } + }); + + const newManifest = Manifest.fromArtifacts(artifacts, ManifestConstants.SCHEMA_VERSION); + newManifest.setVersion(oldManifest.getVersion()); + + if (!oldManifest.equals(newManifest)) { + try { + await manifestService.dispatchAndUpdate(newManifest); + } catch (err) { + logger.error(err); + } + } + + // TODO: clean up old artifacts +}; diff --git a/x-pack/plugins/security_solution/server/endpoint/artifacts/saved_object_mappings.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts similarity index 75% rename from x-pack/plugins/security_solution/server/endpoint/artifacts/saved_object_mappings.ts rename to x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts index 0c920c4e48856..8953207deae92 100644 --- a/x-pack/plugins/security_solution/server/endpoint/artifacts/saved_object_mappings.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts @@ -4,10 +4,12 @@ * you may not use this file except in compliance with the Elastic License. */ -import { SavedObjectsType } from '../../../../../../src/core/server'; +import { SavedObjectsType } from '../../../../../../../src/core/server'; -export const exceptionsArtifactSavedObjectType = 'securitySolution:endpoint:exceptions-artifact'; -export const manifestSavedObjectType = 'securitySolution:endpoint:artifact-manifest'; +import { ArtifactConstants, ManifestConstants } from './common'; + +export const exceptionsArtifactSavedObjectType = ArtifactConstants.SAVED_OBJECT_TYPE; +export const manifestSavedObjectType = ManifestConstants.SAVED_OBJECT_TYPE; export const exceptionsArtifactSavedObjectMappings: SavedObjectsType['mappings'] = { properties: { @@ -33,12 +35,8 @@ export const exceptionsArtifactSavedObjectMappings: SavedObjectsType['mappings'] }, }; -export const manifestSavedObjectMappings: SavedObjectType['mappings'] = { +export const manifestSavedObjectMappings: SavedObjectsType['mappings'] = { properties: { - // manifest sequence number - manifestVersion: { - type: 'long', - }, // manifest schema version schemaVersion: { type: 'keyword', @@ -48,7 +46,7 @@ export const manifestSavedObjectMappings: SavedObjectType['mappings'] = { type: 'keyword', }, }, -} +}; export const exceptionsArtifactType: SavedObjectsType = { name: exceptionsArtifactSavedObjectType, diff --git a/x-pack/plugins/security_solution/server/endpoint/artifacts/task.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts similarity index 53% rename from x-pack/plugins/security_solution/server/endpoint/artifacts/task.ts rename to x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts index 09d7e54f5cb55..5ba86769f7f12 100644 --- a/x-pack/plugins/security_solution/server/endpoint/artifacts/task.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts @@ -4,25 +4,14 @@ * you may not use this file except in compliance with the Elastic License. */ -import { createHash } from 'crypto'; -import { - CoreSetup, - Logger, - SavedObjectsClient, - SavedObjectsPluginStart, -} from '../../../../../../src/core/server'; import { ConcreteTaskInstance, TaskManagerSetupContract, TaskManagerStartContract, -} from '../../../../../plugins/task_manager/server'; -import { ListPluginSetup } from '../../../../lists/server'; -import { ConfigType } from '../../config'; +} from '../../../../../../plugins/task_manager/server'; import { EndpointAppContext } from '../../types'; import { ExceptionsCache } from './cache'; -import { GetFullEndpointExceptionList, CompressExceptionList } from './lists'; -import { ArtifactConstants, ManifestService } from './manifest'; -import { ArtifactSoSchema } from './schemas'; +import { refreshManifest } from './refresh'; const PackagerTaskConstants = { TIMEOUT: '1m', @@ -41,7 +30,6 @@ interface PackagerTaskRunner { interface PackagerTaskContext { endpointAppContext: EndpointAppContext; taskManager: TaskManagerSetupContract; - lists: ListPluginSetup; cache: ExceptionsCache; } @@ -56,11 +44,7 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { const logger = context.endpointAppContext.logFactory.get(getTaskId()); - const run = async (taskId: int, state: Record): Record => { - - const artifactService = context.endpointAppContext.service.getArtifactService(); - const manifestService = context.endpointAppContext.service.getManifestService(); - + const run = async (taskId: string) => { // Check that this task is current if (taskId !== getTaskId()) { // old task, return @@ -68,53 +52,15 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { return; } - // TODO - // 1. Pull manifest, note version - // 2. No manifest, do nothing (should have been created on policy create) - // 3. If manifest, pull all associated artifacts (if not already in memory) - // 4. Construct new artifacts from current exception list items - // 5. Any differences? Update manifest, perform conflict check. - // 6. Has this manifest been dispatched? If not, dispatch. - let oldManifest: Manifest; - try { - oldManifest = await manifestService.getManifest(); - catch (err) { + // await refreshManifest(context.endpointAppContext, false); + // TODO: change this to 'false' when we hook up the ingestManager callback + await refreshManifest(context.endpointAppContext, true); + } catch (err) { logger.debug('Manifest not created yet, nothing to do.'); - return; - } - - let artifacts = await artifactService.buildExceptionListArtifacts(); - for (const artifact in artifacts) { - if (!oldManifest.contains(artifact)) { - try { - await artifactService.upsertArtifact( - ArtifactConstants.SAVED_OBJECT_TYPE, - artifact, - { id: artifact.id, overwrite: true }, - ); - } catch (err) { - // Error updating... try again later - logger.error(err); - return; - } - } - } - - artifacts = await artifactService.buildExceptionListArtifacts(); - - const newManifest = buildNewManifest(artifacts); - if (oldManifest.diff(newManifest)) { - try { - await manifestService.dispatchAndUpdate(newManifest); - } catch(err) { - logger.error(err); - return; - } } }; - const getTaskRunner = (runnerContext: PackagerTaskRunnerContext): PackagerTaskRunner => { return { run: async () => { @@ -144,13 +90,14 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { createTaskRunner: ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => { return { run: async () => { - await run(taskInstance.id, taskInstance.state); + await run(taskInstance.id); const nextRun = new Date(); - nextRun.setSeconds(nextRun.getSeconds() + context.endpointAppContext.config.packagerTaskInterval); + const config = await context.endpointAppContext.config(); + nextRun.setSeconds(nextRun.getSeconds() + config.packagerTaskInterval); return { - state, + state: {}, runAt: nextRun, }; }, diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/index.ts b/x-pack/plugins/security_solution/server/endpoint/lib/index.ts new file mode 100644 index 0000000000000..41bc2aa258807 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/lib/index.ts @@ -0,0 +1,5 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ diff --git a/x-pack/plugins/security_solution/server/endpoint/mocks.ts b/x-pack/plugins/security_solution/server/endpoint/mocks.ts index b10e9e4dc90e7..fad18613fab1d 100644 --- a/x-pack/plugins/security_solution/server/endpoint/mocks.ts +++ b/x-pack/plugins/security_solution/server/endpoint/mocks.ts @@ -5,9 +5,14 @@ */ import { IScopedClusterClient, SavedObjectsClientContract } from 'kibana/server'; + import { xpackMocks } from '../../../../mocks'; import { AgentService, IngestManagerStartContract } from '../../../ingest_manager/server'; +import { ExceptionListClient } from '../../../lists/server'; + +import { ArtifactService, ManifestService } from './services'; + /** * Creates a mock AgentService */ @@ -17,6 +22,33 @@ export const createMockAgentService = (): jest.Mocked => { }; }; +/** + * Creates a mock ArtifactService + */ +export const createMockArtifactService = (): jest.Mocked => { + return { + // TODO + }; +}; + +/** + * Creates a mock ManifestService + */ +export const createMockManifestService = (): jest.Mocked => { + return { + // TODO + }; +}; + +/** + * Creates a mock ExceptionListClient + */ +export const createMockExceptionListClient = (): jest.Mocked => { + return { + // TODO + }; +}; + /** * Creates a mock IndexPatternService for use in tests that need to interact with the Ingest Manager's * ESIndexPatternService. diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_endpoint_exception_list.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts similarity index 94% rename from x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_endpoint_exception_list.test.ts rename to x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts index 2824cfc273e95..8909540bb04a2 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_endpoint_exception_list.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts @@ -20,10 +20,10 @@ import { httpServiceMock, httpServerMock, } from 'src/core/server/mocks'; -import { ExceptionsCache } from '../../artifacts/cache'; -import { CompressExceptionList } from '../../artifacts/lists'; -import { ArtifactConstants } from '../../artifacts/manifest'; -import { downloadEndpointExceptionListRoute } from './download_endpoint_exception_list'; +import { ExceptionsCache } from '../../lib/artifacts/cache'; +import { CompressExceptionList } from '../../lib/artifacts/lists'; +import { ArtifactConstants } from '../../lib/artifacts'; +import { registerDownloadExceptionListRoute } from './download_exception_list'; const mockArtifactName = `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-windows-1.0.0`; const expectedEndpointExceptions = { @@ -65,7 +65,7 @@ describe('test alerts route', () => { routerMock = httpServiceMock.createRouter(); cache = new ExceptionsCache(10000); // TODO - downloadEndpointExceptionListRoute(routerMock, cache); + registerDownloadExceptionListRoute(routerMock, cache); }); it('should serve the compressed artifact to download', async () => { diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_endpoint_exception_list.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts similarity index 74% rename from x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_endpoint_exception_list.ts rename to x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts index 7253c19f75902..a30b239796479 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_endpoint_exception_list.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts @@ -4,27 +4,26 @@ * you may not use this file except in compliance with the Elastic License. */ -import { IRouter, KibanaResponse } from 'src/core/server'; -import { validate } from '../../../../../common/validate'; -import { buildRouteValidation } from '../../../../utils/build_validation/route_validation'; -import { ExceptionsCache } from '../../artifacts/cache'; -import { ArtifactConstants } from '../../artifacts/manifest'; +import { IRouter } from 'src/core/server'; +import { validate } from '../../../../common/validate'; +import { buildRouteValidation } from '../../../utils/build_validation/route_validation'; +import { ArtifactConstants, ExceptionsCache } from '../../lib/artifacts'; import { - ArtifactDownloadSchema, DownloadArtifactRequestParamsSchema, downloadArtifactRequestParamsSchema, downloadArtifactResponseSchema, -} from '../../artifacts/schemas'; + InternalArtifactSchema, +} from '../../schemas/artifacts'; const allowlistBaseRoute: string = '/api/endpoint/allowlist'; /** * Registers the exception list route to enable sensors to download a compressed allowlist */ -export function downloadEndpointExceptionListRoute(router: IRouter, cache: ExceptionsCache) { +export function registerDownloadExceptionListRoute(router: IRouter, cache: ExceptionsCache) { router.get( { - path: `${allowlistBaseRoute}/download/{artifactName}/{sha256}`, + path: `${allowlistBaseRoute}/download/{identifier}/{sha256}`, validate: { params: buildRouteValidation< typeof downloadArtifactRequestParamsSchema, @@ -39,16 +38,16 @@ export function downloadEndpointExceptionListRoute(router: IRouter, cache: Excep // TODO: authenticate api key // https://github.com/elastic/kibana/issues/69329 - const validateResponse = (resp: object): KibanaResponse => { + const validateResponse = (resp: object): object => { const [validated, errors] = validate(resp, downloadArtifactResponseSchema); if (errors != null) { return res.internalError({ body: errors }); } else { - return res.ok(validated); + return res.ok({ body: validated ?? {} }); } }; - const cacheKey = `${req.params.artifactName}-${req.params.sha256}`; + const cacheKey = `${req.params.identifier}-${req.params.sha256}`; const cacheResp = cache.get(cacheKey); if (cacheResp) { @@ -57,23 +56,24 @@ export function downloadEndpointExceptionListRoute(router: IRouter, cache: Excep body: Buffer.from(cacheResp, 'binary'), headers: { 'content-encoding': 'xz', - 'content-disposition': `attachment; filename=${req.params.artifactName}.xz`, + 'content-disposition': `attachment; filename=${req.params.identifier}.xz`, }, }; return validateResponse(downloadResponse); } else { // CACHE MISS return soClient - .get( + .get( ArtifactConstants.SAVED_OBJECT_TYPE, - `${req.params.artifactName}` + `${req.params.identifier}` ) .then((artifact) => { const outBuffer = Buffer.from(artifact.attributes.body, 'binary'); + // TODO: probably don't need this anymore if (artifact.attributes.sha256 !== req.params.sha256) { return res.notFound({ - body: `No artifact matching sha256: ${req.params.sha256} for type ${req.params.artifactName}`, + body: `No artifact matching sha256: ${req.params.sha256} for type ${req.params.identifier}`, }); } @@ -84,7 +84,7 @@ export function downloadEndpointExceptionListRoute(router: IRouter, cache: Excep body: outBuffer, headers: { 'content-encoding': 'xz', - 'content-disposition': `attachment; filename=${artifact.attributes.name}.xz`, + 'content-disposition': `attachment; filename=${artifact.attributes.identifier}.xz`, }, }; return validateResponse(downloadResponse); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/index.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/index.ts index 741702f37d047..945646c73c46c 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/index.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/index.ts @@ -4,4 +4,4 @@ * you may not use this file except in compliance with the Elastic License. */ -export * from './download_endpoint_exception_list'; +export * from './download_exception_list'; diff --git a/x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/common.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/common.ts similarity index 82% rename from x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/common.ts rename to x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/common.ts index 3090e6e6da48a..3b437a5ee0de8 100644 --- a/x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/common.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/common.ts @@ -10,7 +10,7 @@ export const identifier = t.string; export const body = t.string; -export const created = t.string; // TODO: Make this into an ISO Date string check +export const created = t.number; // TODO: Make this into an ISO Date string check export const encoding = t.keyof({ xz: null, @@ -31,3 +31,5 @@ export const sha256 = t.string; export const size = t.number; export const url = t.string; + +export type ManifestSchemaVersion = t.TypeOf; diff --git a/x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/index.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/index.ts similarity index 91% rename from x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/index.ts rename to x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/index.ts index 15c2bac89720b..cac1055c36e85 100644 --- a/x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/index.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/index.ts @@ -5,6 +5,7 @@ */ export * from './common'; +export * from './manifest_schema'; export * from './request'; export * from './response'; export * from './saved_objects'; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/manifest_schema.mock.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/manifest_schema.mock.ts new file mode 100644 index 0000000000000..41bc2aa258807 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/manifest_schema.mock.ts @@ -0,0 +1,5 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ diff --git a/x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/manifest_schema.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/manifest_schema.ts similarity index 74% rename from x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/manifest_schema.ts rename to x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/manifest_schema.ts index 0333c79121710..e1c5a989734b6 100644 --- a/x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/manifest_schema.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/manifest_schema.ts @@ -5,15 +5,7 @@ */ import * as t from 'io-ts'; -import { artifactName, manifestSchemaVersion, manifestVersion, sha256, size, url } from './common'; - -export const manifestSchema = t.exact( - t.type({ - manifestVersion, - manifestSchemaVersion, - artifacts: t.record(artifactName, manifestEntrySchema), - }) -); +import { identifier, manifestSchemaVersion, manifestVersion, sha256, size, url } from './common'; export const manifestEntrySchema = t.exact( t.type({ @@ -23,5 +15,13 @@ export const manifestEntrySchema = t.exact( }) ); +export const manifestSchema = t.exact( + t.type({ + manifestVersion, + schemaVersion: manifestSchemaVersion, + artifacts: t.record(identifier, manifestEntrySchema), + }) +); + export type ManifestEntrySchema = t.TypeOf; export type ManifestSchema = t.TypeOf; diff --git a/x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/request/download_artifact_schema.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/request/download_artifact_schema.ts similarity index 87% rename from x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/request/download_artifact_schema.ts rename to x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/request/download_artifact_schema.ts index 5d311afcb93e8..6a6d15aae9759 100644 --- a/x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/request/download_artifact_schema.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/request/download_artifact_schema.ts @@ -5,11 +5,11 @@ */ import * as t from 'io-ts'; -import { artifactName, sha256 } from '../common'; +import { identifier, sha256 } from '../common'; export const downloadArtifactRequestParamsSchema = t.exact( t.type({ - artifactName, + identifier, sha256, }) ); diff --git a/x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/request/index.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/request/index.ts similarity index 100% rename from x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/request/index.ts rename to x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/request/index.ts diff --git a/x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/response/download_artifact_schema.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/response/download_artifact_schema.ts similarity index 100% rename from x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/response/download_artifact_schema.ts rename to x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/response/download_artifact_schema.ts diff --git a/x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/response/index.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/response/index.ts similarity index 100% rename from x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/response/index.ts rename to x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/response/index.ts diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts new file mode 100644 index 0000000000000..41bc2aa258807 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts @@ -0,0 +1,5 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ diff --git a/x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/saved_objects.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts similarity index 72% rename from x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/saved_objects.ts rename to x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts index 0f5a7d45d5453..027b5ce77fcb1 100644 --- a/x-pack/plugins/security_solution/server/endpoint/artifacts/schemas/saved_objects.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts @@ -5,7 +5,7 @@ */ import * as t from 'io-ts'; -import { body, created, encoding, identifier, sha256, size, } from './common'; +import { body, created, encoding, identifier, manifestSchemaVersion, sha256, size } from './common'; export const internalArtifactSchema = t.exact( t.type({ @@ -22,10 +22,9 @@ export type InternalArtifactSchema = t.TypeOf; export const internalManifestSchema = t.exact( t.type({ - manifestVersion, schemaVersion: manifestSchemaVersion, - artifacts: t.record(artifactName, manifestEntrySchema), + ids: t.array(identifier), }) ); -export type InternalManifestSchema = t.TypeOf; +export type InternalManifestSchema = t.TypeOf; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/index.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/index.ts new file mode 100644 index 0000000000000..a3b6e68e4ada2 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './artifacts'; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_service.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_service.ts new file mode 100644 index 0000000000000..adddf9a222717 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_service.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SavedObjectsClient } from '../../../../../../../src/core/server'; +import { ArtifactConstants } from '../../lib/artifacts'; +import { InternalArtifactSchema } from '../../schemas/artifacts'; + +export interface ArtifactServiceOptions { + savedObjectsClient: SavedObjectsClient; +} + +export class ArtifactService { + private soClient: SavedObjectsClient; + + constructor(context: ArtifactServiceOptions) { + this.soClient = context.savedObjectsClient; + } + + public async getArtifact(id: string) { + // TODO: add sha256 to id? + return this.soClient.get(ArtifactConstants.SAVED_OBJECT_TYPE, id); + } + + public async createArtifact(artifact: InternalArtifactSchema) { + return this.soClient.create( + ArtifactConstants.SAVED_OBJECT_TYPE, + artifact, + { id: `${artifact.identifier}-${artifact.sha256}` } + ); + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/index.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/index.ts index 41bc2aa258807..a229a38c82456 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/index.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/index.ts @@ -3,3 +3,6 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ + +export * from './artifact_service'; +export * from './manifest_service'; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_service.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_service.ts new file mode 100644 index 0000000000000..11ca0c17c4921 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_service.ts @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SavedObjectsClient } from '../../../../../../../src/core/server'; +import { Manifest, ManifestConstants } from '../../lib/artifacts'; +import { InternalManifestSchema } from '../../schemas/artifacts'; +import { ArtifactService } from './artifact_service'; + +export interface ManifestServiceOptions { + artifactService: ArtifactService; + savedObjectsClient: SavedObjectsClient; +} + +export class ManifestService { + private artifactService: ArtifactService; + private soClient: SavedObjectsClient; + + constructor(context: ManifestServiceOptions) { + this.artifactService = context.artifactService; + this.soClient = context.savedObjectsClient; + } + + private getManifestId(schemaVersion: string): string { + return `endpoint-manifest-${schemaVersion}`; + } + + // public async getManifest(schemaVersion: string): Promise { + public async getManifest(schemaVersion: string): Manifest { + const manifestSo = await this.soClient.get( + ManifestConstants.SAVED_OBJECT_TYPE, + this.getManifestId(schemaVersion) + ); + + const manifest = new Manifest(manifestSo.attributes.schemaVersion); + manifest.setVersion(manifestSo.version); + + for (const id of manifestSo.attributes.ids) { + const artifactSo = await this.artifactService.getArtifact(id); + manifest.addEntry(artifactSo.attributes); + } + + return manifest; + } + + public async dispatchAndUpdate(manifest: Manifest) { + // TODO + // 1. Dispatch the manifest + // 2. Update the manifest in ES (ONLY if successful) + // 3. Use version to resolve conflicts + // 4. If update fails, it was likely already dispatched and updated + // And if not, we'll redispatch it and update next time. + if (manifest.getVersion() === undefined) { + await this.soClient.create( + ManifestConstants.SAVED_OBJECT_TYPE, + manifest.toSavedObject(), + { id: this.getManifestId(manifest.getSchemaVersion()) } + ); + } else { + try { + await this.soClient.update( + ManifestConstants.SAVED_OBJECT_TYPE, + this.getManifestId(manifest.getSchemaVersion()), + manifest.toSavedObject(), + { version: manifest.getVersion() } + ); + } catch (err) { + if (err.status === 409) { + // TODO: log and return + } else { + throw err; + } + } + } + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/services/index.ts b/x-pack/plugins/security_solution/server/endpoint/services/index.ts new file mode 100644 index 0000000000000..a3b6e68e4ada2 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './artifacts'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/index.ts index 0cec1832dab83..67efb35dbf372 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/index.ts @@ -25,4 +25,5 @@ export const createMockConfig = () => ({ from: 'now-15m', to: 'now', }, + packagerTaskInterval: 60, }); diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index e9250cd4eac38..30b7c0c84e4b1 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -20,7 +20,7 @@ import { PluginSetupContract as AlertingSetup } from '../../alerts/server'; import { SecurityPluginSetup as SecuritySetup } from '../../security/server'; import { PluginSetupContract as FeaturesSetup } from '../../features/server'; import { MlPluginSetup as MlSetup } from '../../ml/server'; -import { ListPluginSetup } from '../../lists/server'; +import { ExceptionListClient, ListPluginSetup } from '../../lists/server'; import { EncryptedSavedObjectsPluginSetup as EncryptedSavedObjectsSetup } from '../../encrypted_saved_objects/server'; import { SpacesPluginSetup as SpacesSetup } from '../../spaces/server'; import { LicensingPluginSetup } from '../../licensing/server'; @@ -34,7 +34,7 @@ import { signalRulesAlertType } from './lib/detection_engine/signals/signal_rule import { rulesNotificationAlertType } from './lib/detection_engine/notifications/rules_notification_alert_type'; import { isNotificationAlertExecutor } from './lib/detection_engine/notifications/types'; import { hasListsFeature, listsEnvFeatureFlagName } from './lib/detection_engine/feature_flags'; -import { PackagerTask, setupPackagerTask, ExceptionsCache } from './endpoint/artifacts'; +import { PackagerTask, setupPackagerTask, ExceptionsCache } from './endpoint/lib/artifacts'; import { initSavedObjects, savedObjectTypes } from './saved_objects'; import { AppClientFactory } from './client'; import { createConfig$, ConfigType } from './config'; @@ -44,10 +44,10 @@ import { registerEndpointRoutes } from './endpoint/routes/metadata'; import { registerResolverRoutes } from './endpoint/routes/resolver'; import { registerAlertRoutes } from './endpoint/alerts/routes'; import { registerPolicyRoutes } from './endpoint/routes/policy'; -import { ArtifactService } from './endpoint/artifact_services'; +import { ArtifactService, ManifestService } from './endpoint/services'; import { EndpointAppContextService } from './endpoint/endpoint_app_context_services'; import { EndpointAppContext } from './endpoint/types'; -import { downloadEndpointExceptionListRoute } from './endpoint/routes/artifacts/download_endpoint_exception_list'; +import { registerDownloadExceptionListRoute } from './endpoint/routes/artifacts'; export interface SetupPlugins { alerts: AlertingSetup; @@ -77,10 +77,9 @@ export class Plugin implements IPlugin Date: Sat, 20 Jun 2020 22:16:32 -0400 Subject: [PATCH 052/106] minor cleanup --- .../security_solution/server/endpoint/lib/artifacts/common.ts | 2 +- .../server/endpoint/services/artifacts/artifact_service.ts | 1 - .../server/endpoint/services/artifacts/manifest_service.ts | 3 +-- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts index a4d18cb0cff7d..f0c5ac2bac36c 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts @@ -7,7 +7,7 @@ export const ArtifactConstants = { GLOBAL_ALLOWLIST_NAME: 'endpoint-allowlist', SAVED_OBJECT_TYPE: 'securitySolution:endpoint:exceptions-artifact', - SUPPORTED_OPERATING_SYSTEMS: ['linux', 'windows'], + SUPPORTED_OPERATING_SYSTEMS: ['linux', 'macos', 'windows'], SCHEMA_VERSION: '1.0.0', }; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_service.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_service.ts index adddf9a222717..a62231e20f717 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_service.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_service.ts @@ -20,7 +20,6 @@ export class ArtifactService { } public async getArtifact(id: string) { - // TODO: add sha256 to id? return this.soClient.get(ArtifactConstants.SAVED_OBJECT_TYPE, id); } diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_service.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_service.ts index 11ca0c17c4921..ba203a3c21761 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_service.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_service.ts @@ -27,8 +27,7 @@ export class ManifestService { return `endpoint-manifest-${schemaVersion}`; } - // public async getManifest(schemaVersion: string): Promise { - public async getManifest(schemaVersion: string): Manifest { + public async getManifest(schemaVersion: string): Promise { const manifestSo = await this.soClient.get( ManifestConstants.SAVED_OBJECT_TYPE, this.getManifestId(schemaVersion) From 3f861abd4f388325e440477444bfee4557211356 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Sat, 20 Jun 2020 22:57:53 -0400 Subject: [PATCH 053/106] clean up old artifacts --- .../server/endpoint/lib/artifacts/manifest.ts | 27 ++++++++++++++----- .../server/endpoint/lib/artifacts/refresh.ts | 21 ++++++++++++--- .../services/artifacts/artifact_service.ts | 4 +++ 3 files changed, 43 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts index b98ac371fb813..91634ab5a33ac 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts @@ -12,6 +12,11 @@ import { ManifestSchemaVersion, } from '../../schemas/artifacts'; +export interface ManifestDiff { + type: string; + id: string; +} + export class Manifest { private entries: ManifestEntry[]; private schemaVersion: string; @@ -60,19 +65,27 @@ export class Manifest { return false; } - // Returns true if same - public equals(manifest: Manifest): boolean { - if (manifest.entries.length !== this.entries.length) { - return false; + public getEntries(): ManifestEntry[] { + return this.entries; + } + + // Returns artifacts that are superceded in this manifest. + public diff(manifest: Manifest): ManifestDiff[] { + const diffs: string[] = []; + + for (const entry of manifest.getEntries()) { + if (!this.contains(entry.getArtifact())) { + diffs.push({ type: 'delete', id: entry.getDocId() }); + } } for (const entry of this.entries) { if (!manifest.contains(entry.getArtifact())) { - return false; + diffs.push({ type: 'add', id: entry.getDocId() }); } } - return true; + return diffs; } public toEndpointFormat(): ManifestSchema { @@ -81,9 +94,11 @@ export class Manifest { schemaVersion: 'todo', artifacts: {}, }; + this.entries.forEach((entry) => { manifestObj.artifacts[entry.getIdentifier()] = entry.getRecord(); }); + return manifestObj as ManifestSchema; } diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/refresh.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/refresh.ts index c53ccff82e8ff..2ef7ff2eb9698 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/refresh.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/refresh.ts @@ -78,10 +78,13 @@ export const refreshManifest = async (context: EndpointAppContext, createInitial } } + // TODO: can we do diff here? + const artifacts = await buildExceptionListArtifacts( ArtifactConstants.SCHEMA_VERSION, exceptionListClient ); + artifacts.forEach(async (artifact: InternalArtifactSchema) => { if (!oldManifest.contains(artifact)) { try { @@ -102,13 +105,25 @@ export const refreshManifest = async (context: EndpointAppContext, createInitial const newManifest = Manifest.fromArtifacts(artifacts, ManifestConstants.SCHEMA_VERSION); newManifest.setVersion(oldManifest.getVersion()); - if (!oldManifest.equals(newManifest)) { + const diffs = newManifest.diff(oldManifest); + if (diffs && diffs.length > 0) { try { + logger.debug('Dispatching new manifest'); await manifestService.dispatchAndUpdate(newManifest); } catch (err) { logger.error(err); + return; } - } - // TODO: clean up old artifacts + logger.debug('Cleaning up outdated artifacts.'); + diffs.forEach(async (diff) => { + try { + if (diff.type === 'delete') { + await artifactService.delete(diff.id); + } + } catch (err) { + logger.error(err); + } + }); + } }; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_service.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_service.ts index a62231e20f717..573a41e23f3a0 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_service.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_service.ts @@ -30,4 +30,8 @@ export class ArtifactService { { id: `${artifact.identifier}-${artifact.sha256}` } ); } + + public async deleteArtifact(id: string) { + return this.soClient.delete(ArtifactConstants.SAVED_OBJECT_TYPE, id); + } } From f61de08397204d505209ce63a651f05cb147817c Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Sun, 21 Jun 2020 01:51:41 -0400 Subject: [PATCH 054/106] Use diff appropriately --- .../server/endpoint/lib/artifacts/manifest.ts | 8 ++++ .../server/endpoint/lib/artifacts/refresh.ts | 44 +++++++++++-------- 2 files changed, 33 insertions(+), 19 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts index 91634ab5a33ac..c052a533d388b 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts @@ -69,6 +69,14 @@ export class Manifest { return this.entries; } + public getArtifact(id: string): InternalArtifactSchema { + for (const entry of this.entries) { + if (entry.getDocId() === id) { + return entry.getArtifact(); + } + } + } + // Returns artifacts that are superceded in this manifest. public diff(manifest: Manifest): ManifestDiff[] { const diffs: string[] = []; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/refresh.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/refresh.ts index 2ef7ff2eb9698..7ed411d5e3ad0 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/refresh.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/refresh.ts @@ -64,6 +64,7 @@ export const refreshManifest = async (context: EndpointAppContext, createInitial let oldManifest: Manifest; + // Get the last-dispatched manifest try { oldManifest = await manifestService.getManifest(ManifestConstants.SCHEMA_VERSION); } catch (err) { @@ -78,15 +79,23 @@ export const refreshManifest = async (context: EndpointAppContext, createInitial } } - // TODO: can we do diff here? - + // Build new exception list artifacts const artifacts = await buildExceptionListArtifacts( ArtifactConstants.SCHEMA_VERSION, exceptionListClient ); - artifacts.forEach(async (artifact: InternalArtifactSchema) => { - if (!oldManifest.contains(artifact)) { + // Build new manifest + const newManifest = Manifest.fromArtifacts(artifacts, ManifestConstants.SCHEMA_VERSION); + newManifest.setVersion(oldManifest.getVersion()); + + // Get diffs + const diffs = newManifest.diff(oldManifest); + + // Create new artifacts + diffs.forEach(async (diff) => { + if (diff.type === 'add') { + const artifact = newManifest.getArtifact(diff.id); try { await artifactService.createArtifact(artifact); } catch (err) { @@ -102,11 +111,8 @@ export const refreshManifest = async (context: EndpointAppContext, createInitial } }); - const newManifest = Manifest.fromArtifacts(artifacts, ManifestConstants.SCHEMA_VERSION); - newManifest.setVersion(oldManifest.getVersion()); - - const diffs = newManifest.diff(oldManifest); - if (diffs && diffs.length > 0) { + // Dispatch manifest if new + if (diffs.length > 0) { try { logger.debug('Dispatching new manifest'); await manifestService.dispatchAndUpdate(newManifest); @@ -114,16 +120,16 @@ export const refreshManifest = async (context: EndpointAppContext, createInitial logger.error(err); return; } + } - logger.debug('Cleaning up outdated artifacts.'); - diffs.forEach(async (diff) => { - try { - if (diff.type === 'delete') { - await artifactService.delete(diff.id); - } - } catch (err) { - logger.error(err); + // Clean up old artifacts + diffs.forEach(async (diff) => { + try { + if (diff.type === 'delete') { + await artifactService.delete(diff.id); } - }); - } + } catch (err) { + logger.error(err); + } + }); }; From 2e1dc739a194e6d9e9fdd98661b56f51620bbadd Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Sun, 21 Jun 2020 23:42:14 -0400 Subject: [PATCH 055/106] Fix download --- .../artifacts/download_exception_list.ts | 32 ++++++++----------- .../security_solution/server/plugin.ts | 2 +- 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts index a30b239796479..548532edcdb2d 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts @@ -14,13 +14,18 @@ import { downloadArtifactResponseSchema, InternalArtifactSchema, } from '../../schemas/artifacts'; +import { EndpointAppContext } from '../../types'; const allowlistBaseRoute: string = '/api/endpoint/allowlist'; /** * Registers the exception list route to enable sensors to download a compressed allowlist */ -export function registerDownloadExceptionListRoute(router: IRouter, cache: ExceptionsCache) { +export function registerDownloadExceptionListRoute( + router: IRouter, + endpointContext: EndpointAppContext, + cache: ExceptionsCache +) { router.get( { path: `${allowlistBaseRoute}/download/{identifier}/{sha256}`, @@ -34,6 +39,7 @@ export function registerDownloadExceptionListRoute(router: IRouter, cache: Excep }, async (context, req, res) => { const soClient = context.core.savedObjects.client; + const logger = endpointContext.logFactory.get('download_exception_list'); // TODO: authenticate api key // https://github.com/elastic/kibana/issues/69329 @@ -43,15 +49,16 @@ export function registerDownloadExceptionListRoute(router: IRouter, cache: Excep if (errors != null) { return res.internalError({ body: errors }); } else { - return res.ok({ body: validated ?? {} }); + return res.ok(validated); } }; - const cacheKey = `${req.params.identifier}-${req.params.sha256}`; - const cacheResp = cache.get(cacheKey); + const id = `${req.params.identifier}-${req.params.sha256}`; + const cacheResp = cache.get(id); if (cacheResp) { // CACHE HIT + logger.debug(`Cache HIT artifact ${id}`); const downloadResponse = { body: Buffer.from(cacheResp, 'binary'), headers: { @@ -62,23 +69,12 @@ export function registerDownloadExceptionListRoute(router: IRouter, cache: Excep return validateResponse(downloadResponse); } else { // CACHE MISS + logger.debug(`Cache MISS artifact ${id}`); return soClient - .get( - ArtifactConstants.SAVED_OBJECT_TYPE, - `${req.params.identifier}` - ) + .get(ArtifactConstants.SAVED_OBJECT_TYPE, id) .then((artifact) => { const outBuffer = Buffer.from(artifact.attributes.body, 'binary'); - - // TODO: probably don't need this anymore - if (artifact.attributes.sha256 !== req.params.sha256) { - return res.notFound({ - body: `No artifact matching sha256: ${req.params.sha256} for type ${req.params.identifier}`, - }); - } - - // Hashes match... populate cache - cache.set(cacheKey, artifact.attributes.body); + cache.set(id, artifact.attributes.body); const downloadResponse = { body: outBuffer, diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 30b7c0c84e4b1..b0708daf491cf 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -136,7 +136,7 @@ export class Plugin implements IPlugin Date: Sun, 21 Jun 2020 23:57:21 -0400 Subject: [PATCH 056/106] schedule task on interval --- .../server/endpoint/lib/artifacts/task.ts | 14 +++----------- x-pack/plugins/security_solution/server/plugin.ts | 1 + 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts index 5ba86769f7f12..67007e884c32c 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts @@ -70,14 +70,15 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { id: taskId, taskType: PackagerTaskConstants.TYPE, scope: ['securitySolution'], + schedule: { + interval: context.endpointAppContext.config.packagerTaskInterval, + }, state: {}, params: { version: PackagerTaskConstants.VERSION }, }); } catch (e) { logger.debug(`Error scheduling task, received ${e.message}`); } - - await runnerContext.taskManager.runNow(taskId); }, }; }; @@ -91,15 +92,6 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { return { run: async () => { await run(taskInstance.id); - - const nextRun = new Date(); - const config = await context.endpointAppContext.config(); - nextRun.setSeconds(nextRun.getSeconds() + config.packagerTaskInterval); - - return { - state: {}, - runAt: nextRun, - }; }, cancel: async () => {}, }; diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index b0708daf491cf..b239367da4e1a 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -79,6 +79,7 @@ export class Plugin implements IPlugin Date: Mon, 22 Jun 2020 23:46:55 -0400 Subject: [PATCH 057/106] Split up into client/manager --- .../security_solution/server/config.ts | 5 - .../server/endpoint/config.ts | 5 - .../endpoint/endpoint_app_context_services.ts | 33 +-- .../server/endpoint/lib/artifacts/manifest.ts | 62 +++--- .../lib/artifacts/manifest_entry.test.ts | 5 + .../server/endpoint/lib/artifacts/refresh.ts | 135 ------------ .../lib/artifacts/saved_object_mappings.ts | 5 +- .../server/endpoint/lib/artifacts/task.ts | 19 +- .../schemas/artifacts/saved_objects.mock.ts | 16 ++ .../schemas/artifacts/saved_objects.ts | 4 +- .../artifacts/artifact_client.mock.ts | 14 ++ .../services/artifacts/artifact_client.ts | 38 ++++ .../services/artifacts/artifact_service.ts | 37 ---- .../endpoint/services/artifacts/index.ts | 4 +- .../artifacts/manifest_client.mock.ts | 15 ++ .../services/artifacts/manifest_client.ts | 67 ++++++ .../get_last_dispatched_manifest.test.ts | 5 + .../get_last_dispatched_manifest.ts | 5 + .../manifest_manager/get_manifest_id.test.ts | 5 + .../manifest_manager/get_manifest_id.ts | 5 + .../artifacts/manifest_manager/index.ts | 7 + .../manifest_manager/manifest_manager.test.ts | 5 + .../manifest_manager/manifest_manager.ts | 193 ++++++++++++++++++ .../services/artifacts/manifest_service.ts | 77 ------- .../routes/__mocks__/index.ts | 1 - .../security_solution/server/plugin.ts | 23 ++- 26 files changed, 449 insertions(+), 341 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts delete mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/artifacts/refresh.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.mock.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts delete mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_service.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.mock.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/get_last_dispatched_manifest.test.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/get_last_dispatched_manifest.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/get_manifest_id.test.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/get_manifest_id.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/index.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts delete mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_service.ts diff --git a/x-pack/plugins/security_solution/server/config.ts b/x-pack/plugins/security_solution/server/config.ts index 6868c7dde84c4..f7d961ae3ec5c 100644 --- a/x-pack/plugins/security_solution/server/config.ts +++ b/x-pack/plugins/security_solution/server/config.ts @@ -29,11 +29,6 @@ export const configSchema = schema.object({ from: schema.string({ defaultValue: 'now-15m' }), to: schema.string({ defaultValue: 'now' }), }), - - /** - * Artifact Configuration - */ - packagerTaskInterval: schema.number({ defaultValue: 60 }), }); export const createConfig$ = (context: PluginInitializerContext) => diff --git a/x-pack/plugins/security_solution/server/endpoint/config.ts b/x-pack/plugins/security_solution/server/endpoint/config.ts index d514f08326ad0..908e14468c5c7 100644 --- a/x-pack/plugins/security_solution/server/endpoint/config.ts +++ b/x-pack/plugins/security_solution/server/endpoint/config.ts @@ -27,11 +27,6 @@ export const EndpointConfigSchema = schema.object({ from: schema.string({ defaultValue: 'now-15m' }), to: schema.string({ defaultValue: 'now' }), }), - - /** - * Artifact Configuration - */ - packagerTaskInterval: schema.number({ defaultValue: 60 }), }); export function createConfig$(context: PluginInitializerContext) { diff --git a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts index a8df2dd5207dd..f6ece72c1572d 100644 --- a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts +++ b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts @@ -4,8 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ import { AgentService } from '../../../ingest_manager/server'; -import { ExceptionListClient } from '../../../lists/server'; -import { ArtifactService, ManifestService } from './services'; +import { ManifestManager } from './services'; /** * A singleton that holds shared services that are initialized during the start up phase @@ -13,20 +12,14 @@ import { ArtifactService, ManifestService } from './services'; */ export class EndpointAppContextService { private agentService: AgentService | undefined; - private artifactService: ArtifactService | undefined; - private manifestService: ManifestService | undefined; - private exceptionListClient: ExceptionListClient | undefined; + private manifestManager: ManifestManager | undefined; public start(dependencies: { agentService: AgentService; - artifactService: ArtifactService; - manifestService: ManifestService; - exceptionListClient: ExceptionListClient | undefined; + manifestManager: ManifestManager | undefined; }) { this.agentService = dependencies.agentService; - this.artifactService = dependencies.artifactService; - this.manifestService = dependencies.manifestService; - this.exceptionListClient = dependencies.exceptionListClient; + this.manifestManager = dependencies.manifestManager; } public stop() {} @@ -38,21 +31,7 @@ export class EndpointAppContextService { return this.agentService; } - public getArtifactService(): ArtifactService { - if (!this.artifactService) { - throw new Error(`must call start on ${EndpointAppContextService.name} to call getter`); - } - return this.artifactService; - } - - public getManifestService(): ManifestService { - if (!this.manifestService) { - throw new Error(`must call start on ${EndpointAppContextService.name} to call getter`); - } - return this.manifestService; - } - - public getExceptionListClient(): ExceptionListClient | undefined { - return this.exceptionListClient; + public getManifestManager(): ManifestManager | undefined { + return this.manifestManager; } } diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts index c052a533d388b..2a89c1965ede2 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts @@ -18,14 +18,16 @@ export interface ManifestDiff { } export class Manifest { - private entries: ManifestEntry[]; + private created: Date; + private entries: Record; private schemaVersion: string; // For concurrency control private version: string | undefined; - constructor(schemaVersion: string) { - this.entries = []; + constructor(created: Date, schemaVersion: string) { + this.created = created; + this.entries = {}; this.schemaVersion = schemaVersion; } @@ -33,7 +35,7 @@ export class Manifest { artifacts: InternalArtifactSchema[], schemaVersion: string ): Manifest { - const manifest = new Manifest(schemaVersion); + const manifest = new Manifest(Date.now(), schemaVersion); artifacts.forEach((artifact) => { manifest.addEntry(artifact); }); @@ -53,43 +55,41 @@ export class Manifest { } public addEntry(artifact: InternalArtifactSchema) { - this.entries.push(new ManifestEntry(artifact)); + const entry = new ManifestEntry(artifact); + this.entries[entry.getDocId()] = entry; } - public contains(artifact: InternalArtifactSchema): boolean { - for (const entry of this.entries) { - if (artifact.identifier === entry.getIdentifier() && artifact.sha256 === entry.getSha256()) { - return true; - } - } - return false; + public contains(artifactId: string): boolean { + return artifactId in this.entries; } - public getEntries(): ManifestEntry[] { + public getEntries(): Record { return this.entries; } - public getArtifact(id: string): InternalArtifactSchema { - for (const entry of this.entries) { - if (entry.getDocId() === id) { - return entry.getArtifact(); - } - } + public getArtifact(artifactId: string): InternalArtifactSchema { + return this.entries[artifactId].getArtifact(); + } + + public copy(): Manifest { + const manifest = new Manifest(this.created, this.schemaVersion); + manifest.entries = { ...this.entries }; + manifest.version = this.version; + return manifest; } - // Returns artifacts that are superceded in this manifest. public diff(manifest: Manifest): ManifestDiff[] { - const diffs: string[] = []; + const diffs: ManifestDiff[] = []; - for (const entry of manifest.getEntries()) { - if (!this.contains(entry.getArtifact())) { - diffs.push({ type: 'delete', id: entry.getDocId() }); + for (const id in manifest.getEntries()) { + if (!this.contains(id)) { + diffs.push({ type: 'delete', id }); } } - for (const entry of this.entries) { - if (!manifest.contains(entry.getArtifact())) { - diffs.push({ type: 'add', id: entry.getDocId() }); + for (const id in this.entries) { + if (!manifest.contains(id)) { + diffs.push({ type: 'add', id }); } } @@ -103,17 +103,17 @@ export class Manifest { artifacts: {}, }; - this.entries.forEach((entry) => { + for (const entry of Object.values(this.entries)) { manifestObj.artifacts[entry.getIdentifier()] = entry.getRecord(); - }); + } return manifestObj as ManifestSchema; } public toSavedObject(): InternalManifestSchema { return { - schemaVersion: this.schemaVersion as ManifestSchemaVersion, - ids: this.entries.map((entry) => entry.getDocId()), + created: this.created, + ids: Object.keys(this.entries), }; } } diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts new file mode 100644 index 0000000000000..41bc2aa258807 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts @@ -0,0 +1,5 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/refresh.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/refresh.ts deleted file mode 100644 index 7ed411d5e3ad0..0000000000000 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/refresh.ts +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -/* -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { createHash } from 'crypto'; - -import { ExceptionListClient } from '../../../../../lists/server'; - -import { InternalArtifactSchema } from '../../schemas/artifacts'; -import { EndpointAppContext } from '../../types'; - -import { ArtifactConstants, ManifestConstants } from './common'; -import { GetFullEndpointExceptionList, CompressExceptionList } from './lists'; -import { Manifest } from './manifest'; - -const buildExceptionListArtifacts = async ( - schemaVersion: string, - exceptionListClient: ExceptionListClient -): Promise => { - const artifacts: InternalArtifactSchema[] = []; - - for (const os of ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS) { - const exceptions = await GetFullEndpointExceptionList(exceptionListClient, os, schemaVersion); - - const compressedExceptions: Buffer = await CompressExceptionList(exceptions); - - const sha256 = createHash('sha256') - .update(compressedExceptions.toString('utf8'), 'utf8') - .digest('hex'); - - artifacts.push({ - identifier: `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-${os}-${schemaVersion}`, - sha256, - encoding: 'xz', - created: Date.now(), - body: compressedExceptions.toString('binary'), - size: Buffer.from(JSON.stringify(exceptions)).byteLength, - }); - } - - return artifacts; -}; - -export const refreshManifest = async (context: EndpointAppContext, createInitial: boolean) => { - const exceptionListClient = context.service.getExceptionListClient(); - const logger = context.logFactory.get('endpoint-artifact-refresh'); - - if (exceptionListClient === undefined) { - logger.debug('Lists plugin not available. Unable to refresh manifest.'); - return; - } - - const artifactService = context.service.getArtifactService(); - const manifestService = context.service.getManifestService(); - - let oldManifest: Manifest; - - // Get the last-dispatched manifest - try { - oldManifest = await manifestService.getManifest(ManifestConstants.SCHEMA_VERSION); - } catch (err) { - // TODO: does this need to be more specific? - // 1. could fail pulling artifacts - // 2. could fail if manifest does not exist yet - if (createInitial) { - // TODO: implement this when ready to integrate with Paul's code - oldManifest = new Manifest(ManifestConstants.SCHEMA_VERSION); // create empty manifest - } else { - throw err; - } - } - - // Build new exception list artifacts - const artifacts = await buildExceptionListArtifacts( - ArtifactConstants.SCHEMA_VERSION, - exceptionListClient - ); - - // Build new manifest - const newManifest = Manifest.fromArtifacts(artifacts, ManifestConstants.SCHEMA_VERSION); - newManifest.setVersion(oldManifest.getVersion()); - - // Get diffs - const diffs = newManifest.diff(oldManifest); - - // Create new artifacts - diffs.forEach(async (diff) => { - if (diff.type === 'add') { - const artifact = newManifest.getArtifact(diff.id); - try { - await artifactService.createArtifact(artifact); - } catch (err) { - if (err.status === 409) { - // This artifact already existed... - logger.debug( - `Tried to create artifact ${artifact.identifier}-${artifact.sha256}, but it already exists.` - ); - } else { - throw err; - } - } - } - }); - - // Dispatch manifest if new - if (diffs.length > 0) { - try { - logger.debug('Dispatching new manifest'); - await manifestService.dispatchAndUpdate(newManifest); - } catch (err) { - logger.error(err); - return; - } - } - - // Clean up old artifacts - diffs.forEach(async (diff) => { - try { - if (diff.type === 'delete') { - await artifactService.delete(diff.id); - } - } catch (err) { - logger.error(err); - } - }); -}; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts index 8953207deae92..f805c117e34e1 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts @@ -37,9 +37,8 @@ export const exceptionsArtifactSavedObjectMappings: SavedObjectsType['mappings'] export const manifestSavedObjectMappings: SavedObjectsType['mappings'] = { properties: { - // manifest schema version - schemaVersion: { - type: 'keyword', + created: { + type: 'date', }, // array of doc ids ids: { diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts index 67007e884c32c..0f74f0116662b 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts @@ -11,7 +11,6 @@ import { } from '../../../../../../plugins/task_manager/server'; import { EndpointAppContext } from '../../types'; import { ExceptionsCache } from './cache'; -import { refreshManifest } from './refresh'; const PackagerTaskConstants = { TIMEOUT: '1m', @@ -42,7 +41,9 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { return `${PackagerTaskConstants.TYPE}:${PackagerTaskConstants.VERSION}`; }; - const logger = context.endpointAppContext.logFactory.get(getTaskId()); + const logger = context.endpointAppContext.logFactory.get( + `endpoint_manifest_refresh_${getTaskId()}` + ); const run = async (taskId: string) => { // Check that this task is current @@ -52,11 +53,18 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { return; } + const manifestManager = context.endpointAppContext.service.getManifestManager(); + + if (manifestManager === undefined) { + logger.debug('Manifest Manager not available.'); + return; + } + try { - // await refreshManifest(context.endpointAppContext, false); // TODO: change this to 'false' when we hook up the ingestManager callback - await refreshManifest(context.endpointAppContext, true); + await manifestManager.refresh(true); } catch (err) { + logger.error(err); logger.debug('Manifest not created yet, nothing to do.'); } }; @@ -71,7 +79,8 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { taskType: PackagerTaskConstants.TYPE, scope: ['securitySolution'], schedule: { - interval: context.endpointAppContext.config.packagerTaskInterval, + // TODO: change this to '60s' before merging + interval: '5s', }, state: {}, params: { version: PackagerTaskConstants.VERSION }, diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts index 41bc2aa258807..45bbc55bad037 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts @@ -3,3 +3,19 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ + +import { InternalArtifactSchema, InternalManifestSchema } from './saved_objects'; + +export const getInternalArtifactSchemaMock = (): InternalArtifactSchema => ({ + identifier: '', + sha256: '', + encoding: '', + created: '', + body: '', + size: '', +}); + +export const getInternalManifestSchemaMock = (): InternalManifestSchema => ({ + created: Date.now(), + ids: [], +}); diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts index 027b5ce77fcb1..3473dfc952e45 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts @@ -5,7 +5,7 @@ */ import * as t from 'io-ts'; -import { body, created, encoding, identifier, manifestSchemaVersion, sha256, size } from './common'; +import { body, created, encoding, identifier, sha256, size } from './common'; export const internalArtifactSchema = t.exact( t.type({ @@ -22,7 +22,7 @@ export type InternalArtifactSchema = t.TypeOf; export const internalManifestSchema = t.exact( t.type({ - schemaVersion: manifestSchemaVersion, + created, ids: t.array(identifier), }) ); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.mock.ts new file mode 100644 index 0000000000000..01a09c76d17b1 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.mock.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ArtifactService } from './artifact_service'; +import { getInternalArtifactSchemaMock } from '../../schemas'; + +export class ArtifactServiceMock extends ArtifactService { + public getArtifact = jest.fn().mockResolvedValue(getInternalArtifactSchemaMock()); + public createArtifact = jest.fn().mockResolvedValue(getInternalArtifactSchemaMock()); + public deleteArtifact = jest.fn().mockResolvedValue(getInternalArtifactSchemaMock()); +} diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts new file mode 100644 index 0000000000000..1313adb916d86 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SavedObject, SavedObjectsClient } from '../../../../../../../src/core/server'; +import { ArtifactConstants } from '../../lib/artifacts'; +import { InternalArtifactSchema } from '../../schemas/artifacts'; + +export class ArtifactClient { + private savedObjectsClient: SavedObjectsClient; + + constructor(savedObjectsClient: SavedObjectsClient) { + this.savedObjectsClient = savedObjectsClient; + } + + public async getArtifact(id: string): Promise> { + return this.savedObjectsClient.get( + ArtifactConstants.SAVED_OBJECT_TYPE, + id + ); + } + + public async createArtifact( + artifact: InternalArtifactSchema + ): Promise> { + return this.savedObjectsClient.create( + ArtifactConstants.SAVED_OBJECT_TYPE, + artifact, + { id: `${artifact.identifier}-${artifact.sha256}` } + ); + } + + public async deleteArtifact(id: string) { + return this.savedObjectsClient.delete(ArtifactConstants.SAVED_OBJECT_TYPE, id); + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_service.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_service.ts deleted file mode 100644 index 573a41e23f3a0..0000000000000 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_service.ts +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { SavedObjectsClient } from '../../../../../../../src/core/server'; -import { ArtifactConstants } from '../../lib/artifacts'; -import { InternalArtifactSchema } from '../../schemas/artifacts'; - -export interface ArtifactServiceOptions { - savedObjectsClient: SavedObjectsClient; -} - -export class ArtifactService { - private soClient: SavedObjectsClient; - - constructor(context: ArtifactServiceOptions) { - this.soClient = context.savedObjectsClient; - } - - public async getArtifact(id: string) { - return this.soClient.get(ArtifactConstants.SAVED_OBJECT_TYPE, id); - } - - public async createArtifact(artifact: InternalArtifactSchema) { - return this.soClient.create( - ArtifactConstants.SAVED_OBJECT_TYPE, - artifact, - { id: `${artifact.identifier}-${artifact.sha256}` } - ); - } - - public async deleteArtifact(id: string) { - return this.soClient.delete(ArtifactConstants.SAVED_OBJECT_TYPE, id); - } -} diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/index.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/index.ts index a229a38c82456..44a4d7e77dbcb 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/index.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/index.ts @@ -4,5 +4,5 @@ * you may not use this file except in compliance with the Elastic License. */ -export * from './artifact_service'; -export * from './manifest_service'; +export * from './artifact_client'; +export * from './manifest_manager'; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.mock.ts new file mode 100644 index 0000000000000..5bf37ffea5556 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.mock.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ManifestClient } from './manifest_service'; +import { getInternalManifestSchemaMock } from '../../schemas'; + +export class ManifestClientMock extends ManifestClient { + public createManifest = jest.fn().mockResolvedValue(getInternalManifestSchemaMock()); + public getManifest = jest.fn().mockResolvedValue(getInternalManifestSchemaMock()); + public updateManifest = jest.fn().mockResolvedValue(getInternalManifestSchemaMock()); + public deleteManifest = jest.fn().mockResolvedValue(getInternalManifestSchemaMock()); +} diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.ts new file mode 100644 index 0000000000000..8641963caebbf --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.ts @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + SavedObject, + SavedObjectsClient, + SavedObjectsUpdateResponse, +} from '../../../../../../../src/core/server'; +import { ManifestConstants } from '../../lib/artifacts'; +import { InternalManifestSchema } from '../../schemas/artifacts'; + +interface UpdateManifestOpts { + version: string; +} + +export class ManifestClient { + private schemaVersion: string; + private savedObjectsClient: SavedObjectsClient; + + constructor(savedObjectsClient: SavedObjectsClient, schemaVersion: string) { + this.savedObjectsClient = savedObjectsClient; + this.schemaVersion = schemaVersion; + } + + private getManifestId(): string { + return `endpoint-manifest-${this.schemaVersion}`; + } + + public async getManifest(): Promise> { + return this.savedObjectsClient.get( + ManifestConstants.SAVED_OBJECT_TYPE, + this.getManifestId() + ); + } + + public async createManifest( + manifest: InternalManifestSchema + ): Promise> { + return this.savedObjectsClient.create( + ManifestConstants.SAVED_OBJECT_TYPE, + manifest, + { id: this.getManifestId() } + ); + } + + public async updateManifest( + manifest: InternalManifestSchema, + opts?: UpdateManifestOpts + ): Promise> { + return this.savedObjectsClient.update( + ManifestConstants.SAVED_OBJECT_TYPE, + this.getManifestId(), + manifest, + opts + ); + } + + public async deleteManifest() { + return this.savedObjectsClient.delete( + ManifestConstants.SAVED_OBJECT_TYPE, + this.getManifestId() + ); + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/get_last_dispatched_manifest.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/get_last_dispatched_manifest.test.ts new file mode 100644 index 0000000000000..41bc2aa258807 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/get_last_dispatched_manifest.test.ts @@ -0,0 +1,5 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/get_last_dispatched_manifest.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/get_last_dispatched_manifest.ts new file mode 100644 index 0000000000000..41bc2aa258807 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/get_last_dispatched_manifest.ts @@ -0,0 +1,5 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/get_manifest_id.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/get_manifest_id.test.ts new file mode 100644 index 0000000000000..41bc2aa258807 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/get_manifest_id.test.ts @@ -0,0 +1,5 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/get_manifest_id.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/get_manifest_id.ts new file mode 100644 index 0000000000000..41bc2aa258807 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/get_manifest_id.ts @@ -0,0 +1,5 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/index.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/index.ts new file mode 100644 index 0000000000000..03d5d27b3ff78 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './manifest_manager'; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts new file mode 100644 index 0000000000000..41bc2aa258807 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts @@ -0,0 +1,5 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts new file mode 100644 index 0000000000000..ddf2d57e98256 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts @@ -0,0 +1,193 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { createHash } from 'crypto'; + +import { Logger, SavedObjectsClient } from '../../../../../../../../src/core/server'; +import { ExceptionListClient } from '../../../../../../lists/server'; +import { + ArtifactConstants, + ManifestConstants, + Manifest, + GetFullEndpointExceptionList, + CompressExceptionList, +} from '../../../lib/artifacts'; +import { InternalArtifactSchema, InternalManifestSchema } from '../../../schemas/artifacts'; +import { ArtifactClient } from '../artifact_client'; +import { ManifestClient } from '../manifest_client'; + +export interface ManifestManagerContext { + savedObjectsClient: SavedObjectsClient; + artifactClient: ArtifactClient; + exceptionListClient: ExceptionListClient; + logger: Logger; +} + +export class ManifestManager { + private artifactClient: ArtifactClient; + private exceptionListClient: ExceptionListClient; + private savedObjectsClient: SavedObjectsClient; + private logger: Logger; + + constructor(context: ManifestManagerContext) { + this.artifactClient = context.artifactClient; + this.exceptionListClient = context.exceptionListClient; + this.savedObjectsClient = context.savedObjectsClient; + this.logger = context.logger; + } + + private async dispatchAndUpdate(manifest: Manifest) { + const manifestClient = new ManifestClient(this.savedObjectsClient, manifest.getSchemaVersion()); + + // TODO: dispatch and only update if successful + + if (manifest.getVersion() === undefined) { + await manifestClient.createManifest(manifest.toSavedObject()); + } else { + try { + await manifestClient.updateManifest(manifest.toSavedObject(), { + version: manifest.getVersion(), + }); + } catch (err) { + if (err.status === 409) { + // TODO: log and return + } else { + throw err; + } + } + } + } + + private async buildExceptionListArtifacts( + schemaVersion: string + ): Promise { + const artifacts: InternalArtifactSchema[] = []; + + for (const os of ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS) { + const exceptions = await GetFullEndpointExceptionList( + this.exceptionListClient, + os, + schemaVersion + ); + const compressedExceptions: Buffer = await CompressExceptionList(exceptions); + + const sha256 = createHash('sha256') + .update(compressedExceptions.toString('utf8'), 'utf8') + .digest('hex'); + + artifacts.push({ + identifier: `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-${os}-${schemaVersion}`, + sha256, + encoding: 'xz', + created: Date.now(), + body: compressedExceptions.toString('binary'), + size: Buffer.from(JSON.stringify(exceptions)).byteLength, + }); + } + + return artifacts; + } + + public async getLastDispatchedManifest(schemaVersion: string): Promise { + const manifestClient = new ManifestClient(this.savedObjectsClient, schemaVersion); + + let manifestSo: InternalManifestSchema; + try { + manifestSo = await manifestClient.getManifest(); + } catch (err) { + if (err.output.statusCode !== 404) { + throw err; + } + } + + if (manifestSo !== undefined) { + const manifest = new Manifest(manifestSo.attributes.created, schemaVersion); + manifest.setVersion(manifestSo.version); + + for (const id of manifestSo.attributes.ids) { + const artifactSo = await this.artifactClient.getArtifact(id); + manifest.addEntry(artifactSo.attributes); + } + + return manifest; + } else { + return null; + } + } + + public async refresh(createInitial: boolean) { + let oldManifest: Manifest; + + // Get the last-dispatched manifest + try { + oldManifest = await this.getLastDispatchedManifest(ManifestConstants.SCHEMA_VERSION); + } catch (err) { + this.logger.error(err); + return; + } + + if (oldManifest === null) { + if (createInitial) { + // TODO: implement this when ready to integrate with Paul's code + oldManifest = new Manifest(Date.now(), ManifestConstants.SCHEMA_VERSION); // create empty manifest + } else { + this.logger.debug('Manifest does not exist yet. Waiting...'); + return; + } + } + + this.logger.debug(oldManifest); + + // Build new exception list artifacts + const artifacts = await this.buildExceptionListArtifacts(ArtifactConstants.SCHEMA_VERSION); + + // Build new manifest + const newManifest = Manifest.fromArtifacts(artifacts, ManifestConstants.SCHEMA_VERSION); + newManifest.setVersion(oldManifest.getVersion()); + + // Get diffs + const diffs = newManifest.diff(oldManifest); + + // Create new artifacts + diffs.forEach(async (diff) => { + if (diff.type === 'add') { + const artifact = newManifest.getArtifact(diff.id); + try { + await this.artifactClient.createArtifact(artifact); + } catch (err) { + if (err.status === 409) { + // This artifact already existed... + this.logger.debug(`Tried to create artifact ${diff.id}, but it already exists.`); + } else { + throw err; + } + } + } + }, this); + + // Dispatch manifest if new + if (diffs.length > 0) { + try { + this.logger.debug('Dispatching new manifest'); + await this.dispatchAndUpdate(newManifest); + } catch (err) { + this.logger.error(err); + return; + } + } + + // Clean up old artifacts + diffs.forEach(async (diff) => { + try { + if (diff.type === 'delete') { + await this.artifactClient.deleteArtifact(diff.id); + } + } catch (err) { + this.logger.error(err); + } + }, this); + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_service.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_service.ts deleted file mode 100644 index ba203a3c21761..0000000000000 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_service.ts +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { SavedObjectsClient } from '../../../../../../../src/core/server'; -import { Manifest, ManifestConstants } from '../../lib/artifacts'; -import { InternalManifestSchema } from '../../schemas/artifacts'; -import { ArtifactService } from './artifact_service'; - -export interface ManifestServiceOptions { - artifactService: ArtifactService; - savedObjectsClient: SavedObjectsClient; -} - -export class ManifestService { - private artifactService: ArtifactService; - private soClient: SavedObjectsClient; - - constructor(context: ManifestServiceOptions) { - this.artifactService = context.artifactService; - this.soClient = context.savedObjectsClient; - } - - private getManifestId(schemaVersion: string): string { - return `endpoint-manifest-${schemaVersion}`; - } - - public async getManifest(schemaVersion: string): Promise { - const manifestSo = await this.soClient.get( - ManifestConstants.SAVED_OBJECT_TYPE, - this.getManifestId(schemaVersion) - ); - - const manifest = new Manifest(manifestSo.attributes.schemaVersion); - manifest.setVersion(manifestSo.version); - - for (const id of manifestSo.attributes.ids) { - const artifactSo = await this.artifactService.getArtifact(id); - manifest.addEntry(artifactSo.attributes); - } - - return manifest; - } - - public async dispatchAndUpdate(manifest: Manifest) { - // TODO - // 1. Dispatch the manifest - // 2. Update the manifest in ES (ONLY if successful) - // 3. Use version to resolve conflicts - // 4. If update fails, it was likely already dispatched and updated - // And if not, we'll redispatch it and update next time. - if (manifest.getVersion() === undefined) { - await this.soClient.create( - ManifestConstants.SAVED_OBJECT_TYPE, - manifest.toSavedObject(), - { id: this.getManifestId(manifest.getSchemaVersion()) } - ); - } else { - try { - await this.soClient.update( - ManifestConstants.SAVED_OBJECT_TYPE, - this.getManifestId(manifest.getSchemaVersion()), - manifest.toSavedObject(), - { version: manifest.getVersion() } - ); - } catch (err) { - if (err.status === 409) { - // TODO: log and return - } else { - throw err; - } - } - } - } -} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/index.ts index 67efb35dbf372..0cec1832dab83 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/index.ts @@ -25,5 +25,4 @@ export const createMockConfig = () => ({ from: 'now-15m', to: 'now', }, - packagerTaskInterval: 60, }); diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index b239367da4e1a..e4cc35e7f96e9 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -20,7 +20,7 @@ import { PluginSetupContract as AlertingSetup } from '../../alerts/server'; import { SecurityPluginSetup as SecuritySetup } from '../../security/server'; import { PluginSetupContract as FeaturesSetup } from '../../features/server'; import { MlPluginSetup as MlSetup } from '../../ml/server'; -import { ExceptionListClient, ListPluginSetup } from '../../lists/server'; +import { ListPluginSetup } from '../../lists/server'; import { EncryptedSavedObjectsPluginSetup as EncryptedSavedObjectsSetup } from '../../encrypted_saved_objects/server'; import { SpacesPluginSetup as SpacesSetup } from '../../spaces/server'; import { LicensingPluginSetup } from '../../licensing/server'; @@ -44,7 +44,7 @@ import { registerEndpointRoutes } from './endpoint/routes/metadata'; import { registerResolverRoutes } from './endpoint/routes/resolver'; import { registerAlertRoutes } from './endpoint/alerts/routes'; import { registerPolicyRoutes } from './endpoint/routes/policy'; -import { ArtifactService, ManifestService } from './endpoint/services'; +import { ArtifactClient, ManifestManager } from './endpoint/services'; import { EndpointAppContextService } from './endpoint/endpoint_app_context_services'; import { EndpointAppContext } from './endpoint/types'; import { registerDownloadExceptionListRoute } from './endpoint/routes/artifacts'; @@ -243,21 +243,22 @@ export class Plugin implements IPlugin Date: Tue, 23 Jun 2020 00:08:44 -0400 Subject: [PATCH 058/106] more mocks --- .../endpoint/alerts/handlers/alerts.test.ts | 11 ++--------- .../endpoint_app_context_services.test.ts | 8 ++------ .../artifacts/task.test.ts} | 0 .../security_solution/server/endpoint/mocks.ts | 18 ++++++++++-------- .../get_last_dispatched_manifest.ts | 5 ----- .../manifest_manager/get_manifest_id.test.ts | 5 ----- .../manifest_manager/get_manifest_id.ts | 5 ----- .../manifest_manager/manifest_manager.ts | 2 +- 8 files changed, 15 insertions(+), 39 deletions(-) rename x-pack/plugins/security_solution/server/endpoint/{services/artifacts/manifest_manager/get_last_dispatched_manifest.test.ts => lib/artifacts/task.test.ts} (100%) delete mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/get_last_dispatched_manifest.ts delete mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/get_manifest_id.test.ts delete mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/get_manifest_id.ts diff --git a/x-pack/plugins/security_solution/server/endpoint/alerts/handlers/alerts.test.ts b/x-pack/plugins/security_solution/server/endpoint/alerts/handlers/alerts.test.ts index 05a27653ca56d..50a529fe31a00 100644 --- a/x-pack/plugins/security_solution/server/endpoint/alerts/handlers/alerts.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/alerts/handlers/alerts.test.ts @@ -11,12 +11,7 @@ import { } from '../../../../../../../src/core/server/mocks'; import { registerAlertRoutes } from '../routes'; import { alertingIndexGetQuerySchema } from '../../../../common/endpoint_alerts/schema/alert_index'; -import { - createMockAgentService, - createMockArtifactService, - createMockManifestService, - createMockExceptionListClient, -} from '../../mocks'; +import { createMockAgentService, createMockManifestManager } from '../../mocks'; import { EndpointAppContextService } from '../../endpoint_app_context_services'; import { createMockConfig } from '../../../lib/detection_engine/routes/__mocks__'; @@ -35,9 +30,7 @@ describe('test alerts route', () => { endpointAppContextService = new EndpointAppContextService(); endpointAppContextService.start({ agentService: createMockAgentService(), - artifactService: createMockArtifactService(), - manifestService: createMockManifestService(), - exceptionListClient: createMockExceptionListClient(), + manifestManager: createMockManifestManager(), }); registerAlertRoutes(routerMock, { diff --git a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts index abc9a06497817..ff09d67a972bc 100644 --- a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts @@ -10,12 +10,8 @@ describe('test endpoint app context services', () => { const endpointAppContextService = new EndpointAppContextService(); expect(() => endpointAppContextService.getAgentService()).toThrow(Error); }); - it('should throw error on getArtifactService if start is not called', async () => { + it('should throw error on getManifestManager if start is not called', async () => { const endpointAppContextService = new EndpointAppContextService(); - expect(() => endpointAppContextService.getArtifactService()).toThrow(Error); - }); - it('should throw error on getManifestService if start is not called', async () => { - const endpointAppContextService = new EndpointAppContextService(); - expect(() => endpointAppContextService.getManifestService()).toThrow(Error); + expect(() => endpointAppContextService.getManifestManager()).toThrow(Error); }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/get_last_dispatched_manifest.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.test.ts similarity index 100% rename from x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/get_last_dispatched_manifest.test.ts rename to x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.test.ts diff --git a/x-pack/plugins/security_solution/server/endpoint/mocks.ts b/x-pack/plugins/security_solution/server/endpoint/mocks.ts index fad18613fab1d..e2e957fe71e04 100644 --- a/x-pack/plugins/security_solution/server/endpoint/mocks.ts +++ b/x-pack/plugins/security_solution/server/endpoint/mocks.ts @@ -11,7 +11,7 @@ import { AgentService, IngestManagerStartContract } from '../../../ingest_manage import { ExceptionListClient } from '../../../lists/server'; -import { ArtifactService, ManifestService } from './services'; +import { ArtifactClient, ManifestManager } from './services'; /** * Creates a mock AgentService @@ -23,20 +23,22 @@ export const createMockAgentService = (): jest.Mocked => { }; /** - * Creates a mock ArtifactService + * Creates a mock ArtifactClient */ -export const createMockArtifactService = (): jest.Mocked => { +export const createMockArtifactClient = (): jest.Mocked => { return { - // TODO + getArtifact: 'TODO', + createArtifact: 'TODO', + deleteArtifact: 'TODO', }; }; /** - * Creates a mock ManifestService + * Creates a mock ManifestManager */ -export const createMockManifestService = (): jest.Mocked => { +export const createMockManifestManager = (): jest.Mocked => { return { - // TODO + refresh: 'TODO', }; }; @@ -45,7 +47,7 @@ export const createMockManifestService = (): jest.Mocked => { */ export const createMockExceptionListClient = (): jest.Mocked => { return { - // TODO + findExceptionListItem: 'TODO', }; }; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/get_last_dispatched_manifest.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/get_last_dispatched_manifest.ts deleted file mode 100644 index 41bc2aa258807..0000000000000 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/get_last_dispatched_manifest.ts +++ /dev/null @@ -1,5 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/get_manifest_id.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/get_manifest_id.test.ts deleted file mode 100644 index 41bc2aa258807..0000000000000 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/get_manifest_id.test.ts +++ /dev/null @@ -1,5 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/get_manifest_id.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/get_manifest_id.ts deleted file mode 100644 index 41bc2aa258807..0000000000000 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/get_manifest_id.ts +++ /dev/null @@ -1,5 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts index ddf2d57e98256..394ba568fab46 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts @@ -91,7 +91,7 @@ export class ManifestManager { return artifacts; } - public async getLastDispatchedManifest(schemaVersion: string): Promise { + private async getLastDispatchedManifest(schemaVersion: string): Promise { const manifestClient = new ManifestClient(this.savedObjectsClient, schemaVersion); let manifestSo: InternalManifestSchema; From a94436ff902aed6621192876fe6974b147289831 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 23 Jun 2020 10:33:29 -0400 Subject: [PATCH 059/106] config interval --- x-pack/plugins/security_solution/server/config.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugins/security_solution/server/config.ts b/x-pack/plugins/security_solution/server/config.ts index f7d961ae3ec5c..4f51ecd53d193 100644 --- a/x-pack/plugins/security_solution/server/config.ts +++ b/x-pack/plugins/security_solution/server/config.ts @@ -29,6 +29,8 @@ export const configSchema = schema.object({ from: schema.string({ defaultValue: 'now-15m' }), to: schema.string({ defaultValue: 'now' }), }), + // TODO: why can't we remove this? :) + packagerTaskInterval: schema.string({ defaultValue: '60s' }), }); export const createConfig$ = (context: PluginInitializerContext) => From a4acdc056092e555fa9ea2715bfbd77b8d31eb6a Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Tue, 23 Jun 2020 14:25:20 -0400 Subject: [PATCH 060/106] Fixing download tests and adding cache tests --- .../endpoint/lib/artifacts/cache.test.ts | 36 +++++++++++++ .../server/endpoint/lib/artifacts/cache.ts | 9 ++-- .../artifacts/download_exception_list.test.ts | 52 +++++++++---------- .../artifacts/download_exception_list.ts | 42 +++++++-------- 4 files changed, 87 insertions(+), 52 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts index 41bc2aa258807..4149ff4a81623 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts @@ -3,3 +3,39 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ + +import { ExceptionsCache } from './cache'; + +describe('ExceptionsCache tests', () => { + let cache: ExceptionsCache; + + beforeEach(() => { + cache = new ExceptionsCache(100); + }); + + test('it should cache', async () => { + cache.set('test', 'body'); + const cacheResp = cache.get('test'); + expect(cacheResp).toEqual('body'); + }); + + test('it should handle cache miss', async () => { + cache.set('test', 'body'); + const cacheResp = cache.get('not test'); + expect(cacheResp).toEqual(undefined); + }); + + test('it should handle cache clean', async () => { + cache.set('test', 'body'); + const cacheResp = cache.get('test'); + expect(cacheResp).toEqual('body'); + + // Clean will remove all entries from the cache that have not been called by `get` since the last time it was cleaned + cache.clean(); + + // Need to call clean again to simulate a ttl period has gone by without `test` being requested + cache.clean(); + const cacheRespCleaned = cache.get('test'); + expect(cacheRespCleaned).toEqual(undefined); + }); +}); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts index 8f84fac37b0b6..fa0002b383a28 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts @@ -4,6 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ +const DEFAULT_TTL = 1000; + export class ExceptionsCache { private cache: Map; private requested: string[]; @@ -12,7 +14,7 @@ export class ExceptionsCache { constructor(ttl: number) { this.cache = new Map(); this.requested = []; - this.ttl = ttl; + this.ttl = ttl ? ttl : DEFAULT_TTL; this.startClean(); } @@ -20,12 +22,13 @@ export class ExceptionsCache { setInterval(() => this.clean, this.ttl); } - private clean() { - for (const v of this.cache.values()) { + clean() { + for (const v of this.cache.keys()) { if (!this.requested.includes(v)) { this.cache.delete(v); } } + this.requested = []; } set(id: string, body: string) { diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts index 8909540bb04a2..7cdb11c6982a1 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts @@ -19,11 +19,15 @@ import { savedObjectsClientMock, httpServiceMock, httpServerMock, + loggingServiceMock, } from 'src/core/server/mocks'; import { ExceptionsCache } from '../../lib/artifacts/cache'; import { CompressExceptionList } from '../../lib/artifacts/lists'; import { ArtifactConstants } from '../../lib/artifacts'; import { registerDownloadExceptionListRoute } from './download_exception_list'; +import { EndpointAppContextService } from '../../endpoint_app_context_services'; +import { createMockAgentService } from '../../mocks'; +import { createMockConfig } from '../../../lib/detection_engine/routes/__mocks__'; const mockArtifactName = `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-windows-1.0.0`; const expectedEndpointExceptions = { @@ -54,6 +58,7 @@ describe('test alerts route', () => { let mockResponse: jest.Mocked; let routeConfig: RouteConfig; let routeHandler: RequestHandler; + let endpointAppContextService: EndpointAppContextService; let cache: ExceptionsCache; beforeEach(() => { @@ -63,9 +68,23 @@ describe('test alerts route', () => { mockResponse = httpServerMock.createResponseFactory(); mockClusterClient.asScoped.mockReturnValue(mockScopedClient); routerMock = httpServiceMock.createRouter(); + endpointAppContextService = new EndpointAppContextService(); cache = new ExceptionsCache(10000); // TODO - registerDownloadExceptionListRoute(routerMock, cache); + endpointAppContextService.start({ + agentService: createMockAgentService(), + manifestManager: undefined, + }); + + registerDownloadExceptionListRoute( + routerMock, + { + logFactory: loggingServiceMock.create(), + service: endpointAppContextService, + config: () => Promise.resolve(createMockConfig()), + }, + cache + ); }); it('should serve the compressed artifact to download', async () => { @@ -82,7 +101,7 @@ describe('test alerts route', () => { type: 'test', references: [], attributes: { - name: mockArtifactName, + identifier: mockArtifactName, schemaVersion: '1.0.0', sha256: '123456', encoding: 'xz', @@ -125,35 +144,16 @@ describe('test alerts route', () => { expect(compressedArtifact).toEqual(mockCompressedArtifact); }); - it('should handle a sha256 mismatch', async () => { + it('should handle fetching a non-existent artifact', async () => { const mockRequest = httpServerMock.createKibanaRequest({ path: `/api/endpoint/allowlist/download/${mockArtifactName}/123456`, method: 'get', params: { sha256: '789' }, }); - const mockCompressedArtifact = await CompressExceptionList(expectedEndpointExceptions); - - const mockArtifact = { - id: '2468', - type: 'test', - references: [], - attributes: { - name: mockArtifactName, - schemaVersion: '1.0.0', - sha256: '123456', - encoding: 'xz', - created: Date.now(), - body: mockCompressedArtifact, - size: 100, - }, - }; - - const soFindResp: SavedObject = { - ...mockArtifact, - }; - - mockSavedObjectClient.get.mockImplementationOnce(() => Promise.resolve(soFindResp)); + mockSavedObjectClient.get.mockImplementationOnce(() => + Promise.reject({ output: { statusCode: 404 } }) + ); [routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) => path.startsWith('/api/endpoint/allowlist/download') @@ -178,7 +178,7 @@ describe('test alerts route', () => { const mockRequest = httpServerMock.createKibanaRequest({ path: `/api/endpoint/allowlist/download/${mockArtifactName}/${mockSha}`, method: 'get', - params: { sha256: mockSha, artifactName: mockArtifactName }, + params: { sha256: mockSha, identifier: mockArtifactName }, }); // Add to the download cache diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts index 548532edcdb2d..7a1bc8181932e 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts @@ -44,8 +44,15 @@ export function registerDownloadExceptionListRoute( // TODO: authenticate api key // https://github.com/elastic/kibana/issues/69329 - const validateResponse = (resp: object): object => { - const [validated, errors] = validate(resp, downloadArtifactResponseSchema); + const buildAndValidateResponse = (artName: string, body: string): object => { + const artifact = { + body: Buffer.from(body, 'binary'), + headers: { + 'content-encoding': 'xz', + 'content-disposition': `attachment; filename=${artName}.xz`, + }, + }; + const [validated, errors] = validate(artifact, downloadArtifactResponseSchema); if (errors != null) { return res.internalError({ body: errors }); } else { @@ -57,36 +64,25 @@ export function registerDownloadExceptionListRoute( const cacheResp = cache.get(id); if (cacheResp) { - // CACHE HIT logger.debug(`Cache HIT artifact ${id}`); - const downloadResponse = { - body: Buffer.from(cacheResp, 'binary'), - headers: { - 'content-encoding': 'xz', - 'content-disposition': `attachment; filename=${req.params.identifier}.xz`, - }, - }; - return validateResponse(downloadResponse); + return buildAndValidateResponse(req.params.identifier, cacheResp); } else { - // CACHE MISS logger.debug(`Cache MISS artifact ${id}`); return soClient .get(ArtifactConstants.SAVED_OBJECT_TYPE, id) .then((artifact) => { - const outBuffer = Buffer.from(artifact.attributes.body, 'binary'); cache.set(id, artifact.attributes.body); - - const downloadResponse = { - body: outBuffer, - headers: { - 'content-encoding': 'xz', - 'content-disposition': `attachment; filename=${artifact.attributes.identifier}.xz`, - }, - }; - return validateResponse(downloadResponse); + return buildAndValidateResponse( + artifact.attributes.identifier, + artifact.attributes.body + ); }) .catch((err) => { - return res.internalError({ body: err }); + if (err?.output?.statusCode === 404) { + return res.notFound({ body: `No artifact found for ${id}` }); + } else { + return res.internalError({ body: err }); + } }); } } From 9d8b7b73d9ea59e95dd63b529e5f1600dca2514e Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Tue, 23 Jun 2020 14:25:36 -0400 Subject: [PATCH 061/106] lint --- .../endpoint/routes/artifacts/download_exception_list.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts index 7cdb11c6982a1..4b5fbc809fa1a 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts @@ -152,6 +152,7 @@ describe('test alerts route', () => { }); mockSavedObjectClient.get.mockImplementationOnce(() => + // eslint-disable-next-line prefer-promise-reject-errors Promise.reject({ output: { statusCode: 404 } }) ); From 9bd4b4c70795f1352fef0fc90d592aa3251fadfa Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 23 Jun 2020 14:49:04 -0400 Subject: [PATCH 062/106] mo money, mo progress --- .../server/endpoint/lib/artifacts/lists.ts | 111 ++++++++++-------- .../artifacts/download_exception_list.ts | 1 + .../endpoint/schemas/artifacts/index.ts | 1 + .../endpoint/schemas/artifacts/lists.mock.ts | 5 + .../endpoint/schemas/artifacts/lists.ts | 59 ++++++++++ .../schemas/artifacts/saved_objects.mock.ts | 7 +- .../artifacts/artifact_client.mock.ts | 4 +- .../manifest_manager/manifest_manager.mock.ts | 34 ++++++ .../manifest_manager/manifest_manager.ts | 53 +++------ .../security_solution/server/plugin.ts | 2 +- 10 files changed, 188 insertions(+), 89 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.mock.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts index 50816d6676ebf..1a1704d986b41 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts @@ -4,31 +4,42 @@ * you may not use this file except in compliance with the Elastic License. */ +import { createHash } from 'crypto'; + import lzma from 'lzma-native'; import { FoundExceptionListItemSchema } from '../../../../../lists/common/schemas/response/found_exception_list_item_schema'; import { ExceptionListClient } from '../../../../../lists/server'; - -export interface EndpointExceptionList { - exceptions_list: ExceptionsList[]; -} - -export interface ExceptionsList { - type: string; - entries: EntryElement[]; -} - -export interface EntryElement { - field: string; - operator: string; - entry: EntryEntry; -} - -export interface EntryEntry { - exact_caseless?: string; - exact_caseless_any?: string[]; +import { + TranslatedExceptionList, + TranslatedEntryNested, + TranslatedEntryMatch, + TranslatedEntryMatchAny, +} from '../../schemas'; +import { ArtifactConstants } from './common'; + +export async function buildArtifact( + exceptionListClient: ExceptionListClient, + os: string, + schemaVersion: string +): InternalArtifactSchema { + const exceptions = await GetFullEndpointExceptionList(exceptionListClient, os, schemaVersion); + const compressedExceptions: Buffer = await CompressExceptionList(exceptions); + + const sha256 = createHash('sha256') + .update(compressedExceptions.toString('utf8'), 'utf8') + .digest('hex'); + + return { + identifier: `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-${os}-${schemaVersion}`, + sha256, + encoding: 'xz', + created: Date.now(), + body: compressedExceptions.toString('binary'), + size: Buffer.from(JSON.stringify(exceptions)).byteLength, + }; } -export async function GetFullEndpointExceptionList( +async function GetFullEndpointExceptionList( eClient: ExceptionListClient, os: string, // TODO: make type schemaVersion: string @@ -64,52 +75,60 @@ export async function GetFullEndpointExceptionList( return exceptions; } +function translateEntry(schemaVersion: string, entry: ExceptionListItemSchema) { + const type = entry.type; + const translatedEntry: TranslatedEntry = { + field: entry.field, + type: entry.type, + }; + if (entry.type === 'nested') { + translatedEntry.entries = []; + entry.entries.forEach((nestedEntry) => { + translatedEntry.entries.push(translateEntry(schemaVersion, nestedEntry)); + }); + } else if (entry.type === 'match') { + // TODO: sync with pedro, when to use cased/caseless + } else if (entry.type === 'match_any') { + // TODO: sync with pedro, when to use cased/caseless + } else { + // TODO: log and return + } +} + /** * Translates Exception list items to Exceptions the endpoint can understand * @param exc */ -export function translateToEndpointExceptions( +function translateToEndpointExceptions( exc: FoundExceptionListItemSchema, schemaVersion: string -): ExceptionsList[] { - const translated: ExceptionsList[] = []; +): TranslatedExceptionList { + const translatedList: TranslatedExceptionList = { + type: exc.type, + entries: [], + }; if (schemaVersion === '1.0.0') { // Transform to endpoint format - exc.data.forEach((item) => { - const endpointItem: ExceptionsList = { - type: item.type, - entries: [], - }; - item.entries.forEach((entry) => { + exc.data.forEach((list) => { + list.entries.forEach((entry) => { // TODO case sensitive? - const e: EntryEntry = {}; - if (entry.match) { - e.exact_caseless = entry.match; - } - - if (entry.match_any) { - e.exact_caseless_any = entry.match_any; - } - - endpointItem.entries.push({ - field: entry.field, - operator: entry.operator, - entry: e, - }); + translatedList.entries.push(translateEntry(entry)); }); - translated.push(endpointItem); }); } else { throw new Error('unsupported schemaVersion'); } - return translated; + + // TODO: validate + + return translatedList; } /** * Compresses the exception list */ -export function CompressExceptionList(exceptionList: EndpointExceptionList): Promise { +function CompressExceptionList(exceptionList: TranslatedExceptionList): Promise { return lzma.compress(JSON.stringify(exceptionList), (res: Buffer) => { return res; }); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts index 548532edcdb2d..2f9b2d97a3d8a 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts @@ -43,6 +43,7 @@ export function registerDownloadExceptionListRoute( // TODO: authenticate api key // https://github.com/elastic/kibana/issues/69329 + // PR: https://github.com/elastic/kibana/pull/69650 const validateResponse = (resp: object): object => { const [validated, errors] = validate(resp, downloadArtifactResponseSchema); diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/index.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/index.ts index cac1055c36e85..b538a9b3328cc 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/index.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/index.ts @@ -5,6 +5,7 @@ */ export * from './common'; +export * from './lists'; export * from './manifest_schema'; export * from './request'; export * from './response'; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.mock.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.mock.ts new file mode 100644 index 0000000000000..41bc2aa258807 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.mock.ts @@ -0,0 +1,5 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts new file mode 100644 index 0000000000000..0820aaaeddd9f --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; + +import { operator } from '../../../../../lists/common/schemas'; + +export const translatedEntryMatchAny = t.exact( + t.type({ + field: t.string, + operator, + type: t.keyof({ + exact_cased_any: null, + exact_caseless_any: null, + }), + value: t.array(t.string), + }) +); +export type TranslatedEntryMatchAny = t.TypeOf; + +export const translatedEntryMatch = t.exact( + t.type({ + field: t.string, + operator, + type: t.keyof({ + exact_cased: null, + exact_caseless: null, + }), + value: t.string, + }) +); +export type TranslatedEntryMatch = t.TypeOf; + +export const translatedEntryNested = t.exact( + t.type({ + field: t.string, + type: t.keyof({ nested: null }), + entries: t.array(t.union([translatedEntryMatch, translatedEntryMatchAny])), + }) +); +export type TranslatedEntryNested = t.TypeOf; + +export const translatedEntry = t.union([ + translatedEntryNested, + translatedEntryMatch, + translatedEntryMatchAny, +]); +export type TranslatedEntry = t.TypeOf; + +export const translatedExceptionList = t.exact( + t.type({ + type: t.string, + entries: t.array(translatedEntry), + }) +); +export type TranslatedExceptionList = t.TypeOf; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts index 45bbc55bad037..e23b627f76180 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts @@ -6,7 +6,8 @@ import { InternalArtifactSchema, InternalManifestSchema } from './saved_objects'; -export const getInternalArtifactSchemaMock = (): InternalArtifactSchema => ({ +// TODO: os type +export const getInternalArtifactMock = (os?: string): InternalArtifactSchema => ({ identifier: '', sha256: '', encoding: '', @@ -15,7 +16,9 @@ export const getInternalArtifactSchemaMock = (): InternalArtifactSchema => ({ size: '', }); -export const getInternalManifestSchemaMock = (): InternalManifestSchema => ({ +export const getInternalArtifactsMock = (): InternalArtifactSchema[] => [{}, {}]; + +export const getInternalManifestMock = (): InternalManifestSchema => ({ created: Date.now(), ids: [], }); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.mock.ts index 01a09c76d17b1..07d2731312591 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.mock.ts @@ -4,10 +4,10 @@ * you may not use this file except in compliance with the Elastic License. */ -import { ArtifactService } from './artifact_service'; +import { ArtifactClient } from './artifact_client'; import { getInternalArtifactSchemaMock } from '../../schemas'; -export class ArtifactServiceMock extends ArtifactService { +export class ArtifactClientMock extends ArtifactClient { public getArtifact = jest.fn().mockResolvedValue(getInternalArtifactSchemaMock()); public createArtifact = jest.fn().mockResolvedValue(getInternalArtifactSchemaMock()); public deleteArtifact = jest.fn().mockResolvedValue(getInternalArtifactSchemaMock()); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts new file mode 100644 index 0000000000000..d165cec605e28 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ManifestManager } from './artifact_client'; + +import { getInternalArtifactMock, getInternalArtifactsMock } from '../../../schemas'; + +function mockBuildExceptionListArtifacts() { + // mock buildArtifactFunction + // pass in OS, mock implementation of ExceptionListItemSchemaMock more than once + // getInternalArtifactsMock() +} + +/* +export class ManifestManagerMock extends ManifestManager { + private buildExceptionListArtifacts = jest + .fn() + .mockResolvedValue(mockBuildExceptionListArtifacts()); + private getLastDispatchedManifest = jest.fn().mockResolvedValue(getManifestMock()); + private getManifestClient = jest.fn().mockValue(getManifestClientMock()); +} + +export const getManifestManagerMock = (): ManifestManager => { + const mock = new ManifestManager({ + artifactClient: getArtifactClientMock(), + exceptionListClient: getExceptionListClientMock(), + savedObjectsClient: savedObjectsClientMock.create(), + logger: loggerMock.create(), + }); +}; +*/ diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts index 394ba568fab46..445d5944ef6e5 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts @@ -4,16 +4,13 @@ * you may not use this file except in compliance with the Elastic License. */ -import { createHash } from 'crypto'; - import { Logger, SavedObjectsClient } from '../../../../../../../../src/core/server'; import { ExceptionListClient } from '../../../../../../lists/server'; import { ArtifactConstants, ManifestConstants, Manifest, - GetFullEndpointExceptionList, - CompressExceptionList, + buildArtifact, } from '../../../lib/artifacts'; import { InternalArtifactSchema, InternalManifestSchema } from '../../../schemas/artifacts'; import { ArtifactClient } from '../artifact_client'; @@ -39,25 +36,21 @@ export class ManifestManager { this.logger = context.logger; } - private async dispatchAndUpdate(manifest: Manifest) { - const manifestClient = new ManifestClient(this.savedObjectsClient, manifest.getSchemaVersion()); + private async getManifestClient(schemaVersion: string): ManifestClient { + return new ManifestClient(this.savedObjectsClient, schemaVersion); + } + + private async dispatch(manifest: Manifest) { + const manifestClient = this.getManifestClient(manifest.getSchemaVersion()); // TODO: dispatch and only update if successful if (manifest.getVersion() === undefined) { await manifestClient.createManifest(manifest.toSavedObject()); } else { - try { - await manifestClient.updateManifest(manifest.toSavedObject(), { - version: manifest.getVersion(), - }); - } catch (err) { - if (err.status === 409) { - // TODO: log and return - } else { - throw err; - } - } + await manifestClient.updateManifest(manifest.toSavedObject(), { + version: manifest.getVersion(), + }); } } @@ -67,32 +60,16 @@ export class ManifestManager { const artifacts: InternalArtifactSchema[] = []; for (const os of ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS) { - const exceptions = await GetFullEndpointExceptionList( - this.exceptionListClient, - os, - schemaVersion - ); - const compressedExceptions: Buffer = await CompressExceptionList(exceptions); - - const sha256 = createHash('sha256') - .update(compressedExceptions.toString('utf8'), 'utf8') - .digest('hex'); - - artifacts.push({ - identifier: `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-${os}-${schemaVersion}`, - sha256, - encoding: 'xz', - created: Date.now(), - body: compressedExceptions.toString('binary'), - size: Buffer.from(JSON.stringify(exceptions)).byteLength, - }); + const artifact = buildArtifact(this.exceptionListClient, os, schemaVersion); + + artifacts.push(artifact); } return artifacts; } private async getLastDispatchedManifest(schemaVersion: string): Promise { - const manifestClient = new ManifestClient(this.savedObjectsClient, schemaVersion); + const manifestClient = this.getManifestClient(schemaVersion); let manifestSo: InternalManifestSchema; try { @@ -172,7 +149,7 @@ export class ManifestManager { if (diffs.length > 0) { try { this.logger.debug('Dispatching new manifest'); - await this.dispatchAndUpdate(newManifest); + await this.dispatch(newManifest); } catch (err) { this.logger.error(err); return; diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index e4cc35e7f96e9..cf52b79f73737 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -263,7 +263,7 @@ export class Plugin implements IPlugin Date: Tue, 23 Jun 2020 17:23:49 -0400 Subject: [PATCH 063/106] Converting to io-ts --- .../endpoint/lib/artifacts/lists.test.ts | 21 +-- .../server/endpoint/lib/artifacts/lists.ts | 125 ++++++++++-------- .../endpoint/schemas/artifacts/lists.ts | 7 + 3 files changed, 91 insertions(+), 62 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts index 9add8ec93c3c7..4b90be591556c 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts @@ -24,17 +24,20 @@ describe('buildEventTypeSignal', () => { { entries: [ { - entry: { exact_caseless: 'Elastic, N.V.' }, - field: 'actingProcess.file.signer', - operator: 'included', - }, - { - entry: { exact_caseless_any: ['process', 'malware'] }, - field: 'event.category', + field: 'some.not.nested.field', operator: 'included', + type: 'exact_cased', + value: 'some value', }, ], - type: 'simple', + field: 'some.field', + type: 'nested', + }, + { + field: 'some.not.nested.field', + operator: 'included', + type: 'exact_cased', + value: 'some value', }, ], }; @@ -62,6 +65,6 @@ describe('buildEventTypeSignal', () => { .mockReturnValueOnce(second) .mockReturnValueOnce(third); const resp = await GetFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); - expect(resp.exceptions_list.length).toEqual(3); + expect(resp.exceptions_list.length).toEqual(6); }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts index 1a1704d986b41..13f0db98928f8 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts @@ -7,44 +7,51 @@ import { createHash } from 'crypto'; import lzma from 'lzma-native'; +import { + Entry, + EntryNested, + EntryMatch, + EntryMatchAny, +} from '../../../../../lists/common/schemas/types/entries'; import { FoundExceptionListItemSchema } from '../../../../../lists/common/schemas/response/found_exception_list_item_schema'; import { ExceptionListClient } from '../../../../../lists/server'; import { TranslatedExceptionList, - TranslatedEntryNested, + TranslatedEntry, TranslatedEntryMatch, TranslatedEntryMatchAny, + TranslatedEntryNested, + FinalTranslatedExceptionList, } from '../../schemas'; -import { ArtifactConstants } from './common'; - -export async function buildArtifact( - exceptionListClient: ExceptionListClient, - os: string, - schemaVersion: string -): InternalArtifactSchema { - const exceptions = await GetFullEndpointExceptionList(exceptionListClient, os, schemaVersion); - const compressedExceptions: Buffer = await CompressExceptionList(exceptions); - - const sha256 = createHash('sha256') - .update(compressedExceptions.toString('utf8'), 'utf8') - .digest('hex'); - - return { - identifier: `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-${os}-${schemaVersion}`, - sha256, - encoding: 'xz', - created: Date.now(), - body: compressedExceptions.toString('binary'), - size: Buffer.from(JSON.stringify(exceptions)).byteLength, - }; -} -async function GetFullEndpointExceptionList( +// export async function buildArtifact( +// exceptionListClient: ExceptionListClient, +// os: string, +// schemaVersion: string +// ): InternalArtifactSchema { +// const exceptions = await GetFullEndpointExceptionList(exceptionListClient, os, schemaVersion); +// const compressedExceptions: Buffer = await CompressExceptionList(exceptions); + +// const sha256 = createHash('sha256') +// .update(compressedExceptions.toString('utf8'), 'utf8') +// .digest('hex'); + +// return { +// identifier: `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-${os}-${schemaVersion}`, +// sha256, +// encoding: 'xz', +// created: Date.now(), +// body: compressedExceptions.toString('binary'), +// size: Buffer.from(JSON.stringify(exceptions)).byteLength, +// }; +// } + +export async function GetFullEndpointExceptionList( eClient: ExceptionListClient, os: string, // TODO: make type schemaVersion: string -): Promise { - const exceptions: EndpointExceptionList = { exceptions_list: [] }; +): Promise { + const exceptions: FinalTranslatedExceptionList = { exceptions_list: [] }; // todo type let numResponses = 0; let page = 1; @@ -75,26 +82,6 @@ async function GetFullEndpointExceptionList( return exceptions; } -function translateEntry(schemaVersion: string, entry: ExceptionListItemSchema) { - const type = entry.type; - const translatedEntry: TranslatedEntry = { - field: entry.field, - type: entry.type, - }; - if (entry.type === 'nested') { - translatedEntry.entries = []; - entry.entries.forEach((nestedEntry) => { - translatedEntry.entries.push(translateEntry(schemaVersion, nestedEntry)); - }); - } else if (entry.type === 'match') { - // TODO: sync with pedro, when to use cased/caseless - } else if (entry.type === 'match_any') { - // TODO: sync with pedro, when to use cased/caseless - } else { - // TODO: log and return - } -} - /** * Translates Exception list items to Exceptions the endpoint can understand * @param exc @@ -102,18 +89,15 @@ function translateEntry(schemaVersion: string, entry: ExceptionListItemSchema) { function translateToEndpointExceptions( exc: FoundExceptionListItemSchema, schemaVersion: string -): TranslatedExceptionList { - const translatedList: TranslatedExceptionList = { - type: exc.type, - entries: [], - }; +): TranslatedEntry[] { + const translatedList: TranslatedEntry[] = []; if (schemaVersion === '1.0.0') { // Transform to endpoint format exc.data.forEach((list) => { list.entries.forEach((entry) => { // TODO case sensitive? - translatedList.entries.push(translateEntry(entry)); + translatedList.push(translateEntry(schemaVersion, entry)); }); }); } else { @@ -125,6 +109,41 @@ function translateToEndpointExceptions( return translatedList; } +function translateEntry(schemaVersion: string, entry: Entry | EntryNested): TranslatedEntry { + let translatedEntry: TranslatedEntry; + if (entry.type === 'nested') { + const e = (entry as unknown) as EntryNested; + const nestedEntries: TranslatedEntry[] = []; + e.entries.forEach((nestedEntry) => { + nestedEntries.push(translateEntry(schemaVersion, nestedEntry)); + }); + translatedEntry = { + entries: nestedEntries, + field: e.field, + type: 'nested', + } as TranslatedEntryNested; + } else if (entry.type === 'match') { + const e = (entry as unknown) as EntryMatch; + translatedEntry = { + field: e.field, + operator: e.operator, + type: 'exact_cased', // todo + value: e.value, + } as TranslatedEntryMatch; + } else if (entry.type === 'match_any') { + const e = (entry as unknown) as EntryMatchAny; + translatedEntry = { + field: e.field, + operator: e.operator, + type: 'exact_cased_any', // todo + value: e.value, + } as TranslatedEntryMatchAny; + } else { + throw new Error(); + } + return translatedEntry; +} + /** * Compresses the exception list */ diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts index 0820aaaeddd9f..f7121d61e73a1 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts @@ -57,3 +57,10 @@ export const translatedExceptionList = t.exact( }) ); export type TranslatedExceptionList = t.TypeOf; + +export const finalExceptionList = t.exact( + t.type({ + exceptions_list: t.array(translatedEntry), + }) +); +export type FinalTranslatedExceptionList = t.TypeOf; From da802791db2fdd88545479d43e1dd7212fea8c8f Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 23 Jun 2020 21:16:19 -0400 Subject: [PATCH 064/106] More tests and mocks --- .../endpoint/lib/artifacts/lists.test.ts | 6 +- .../server/endpoint/lib/artifacts/lists.ts | 55 +++++------ .../endpoint/lib/artifacts/manifest.test.ts | 96 +++++++++++++++++++ .../server/endpoint/lib/artifacts/manifest.ts | 33 ++++--- .../endpoint/schemas/artifacts/index.ts | 2 + .../endpoint/schemas/artifacts/lists.mock.ts | 27 ++++++ .../schemas/artifacts/saved_objects.mock.ts | 22 +++-- .../manifest_manager/manifest_manager.ts | 8 +- yarn.lock | 16 ---- 9 files changed, 194 insertions(+), 71 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts index 4b90be591556c..d25e8a64cebb3 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts @@ -8,7 +8,7 @@ import { ExceptionListClient } from '../../../../../lists/server'; import { listMock } from '../../../../../lists/server/mocks'; import { getFoundExceptionListItemSchemaMock } from '../../../../../lists/common/schemas/response/found_exception_list_item_schema.mock'; import { getExceptionListItemSchemaMock } from '../../../../../lists/common/schemas/response/exception_list_item_schema.mock'; -import { GetFullEndpointExceptionList } from './lists'; +import { getFullEndpointExceptionList } from './lists'; describe('buildEventTypeSignal', () => { let mockExceptionClient: ExceptionListClient; @@ -44,7 +44,7 @@ describe('buildEventTypeSignal', () => { const first = getFoundExceptionListItemSchemaMock(); mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); - const resp = await GetFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); + const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); expect(resp).toEqual(expectedEndpointExceptions); }); @@ -64,7 +64,7 @@ describe('buildEventTypeSignal', () => { .mockReturnValueOnce(first) .mockReturnValueOnce(second) .mockReturnValueOnce(third); - const resp = await GetFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); + const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); expect(resp.exceptions_list.length).toEqual(6); }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts index 13f0db98928f8..1a02906946712 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts @@ -23,30 +23,30 @@ import { TranslatedEntryNested, FinalTranslatedExceptionList, } from '../../schemas'; +import { ArtifactConstants } from './common'; -// export async function buildArtifact( -// exceptionListClient: ExceptionListClient, -// os: string, -// schemaVersion: string -// ): InternalArtifactSchema { -// const exceptions = await GetFullEndpointExceptionList(exceptionListClient, os, schemaVersion); -// const compressedExceptions: Buffer = await CompressExceptionList(exceptions); - -// const sha256 = createHash('sha256') -// .update(compressedExceptions.toString('utf8'), 'utf8') -// .digest('hex'); - -// return { -// identifier: `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-${os}-${schemaVersion}`, -// sha256, -// encoding: 'xz', -// created: Date.now(), -// body: compressedExceptions.toString('binary'), -// size: Buffer.from(JSON.stringify(exceptions)).byteLength, -// }; -// } - -export async function GetFullEndpointExceptionList( +export async function buildArtifact( + exceptions: FinalTranslatedExceptionList, + os: string, + schemaVersion: string +): InternalArtifactSchema { + const compressedExceptions: Buffer = await compressExceptionList(exceptions); + + const sha256 = createHash('sha256') + .update(compressedExceptions.toString('utf8'), 'utf8') + .digest('hex'); + + return { + identifier: `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-${os}-${schemaVersion}`, + sha256, + encoding: 'xz', + created: Date.now(), + body: compressedExceptions.toString('binary'), + size: Buffer.from(JSON.stringify(exceptions)).byteLength, + }; +} + +export async function getFullEndpointExceptionList( eClient: ExceptionListClient, os: string, // TODO: make type schemaVersion: string @@ -127,7 +127,7 @@ function translateEntry(schemaVersion: string, entry: Entry | EntryNested): Tran translatedEntry = { field: e.field, operator: e.operator, - type: 'exact_cased', // todo + type: e.field.endsWith('.text') ? 'exact_caseless' : 'exact_cased', value: e.value, } as TranslatedEntryMatch; } else if (entry.type === 'match_any') { @@ -135,11 +135,12 @@ function translateEntry(schemaVersion: string, entry: Entry | EntryNested): Tran translatedEntry = { field: e.field, operator: e.operator, - type: 'exact_cased_any', // todo + type: e.field.endsWith('.text') ? 'exact_caseless_any' : 'exact_cased_any', value: e.value, } as TranslatedEntryMatchAny; } else { - throw new Error(); + // TODO: log and continue? + throw new Error(`Invalid type encountered: ${entry.type}`); } return translatedEntry; } @@ -147,7 +148,7 @@ function translateEntry(schemaVersion: string, entry: Entry | EntryNested): Tran /** * Compresses the exception list */ -function CompressExceptionList(exceptionList: TranslatedExceptionList): Promise { +function compressExceptionList(exceptionList: TranslatedExceptionList): Promise { return lzma.compress(JSON.stringify(exceptionList), (res: Buffer) => { return res; }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts index 41bc2aa258807..de5e560a9f4f1 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts @@ -3,3 +3,99 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ + +import { getInternalArtifactMock } from '../../schemas'; +import { Manifest } from './manifest'; +import { ManifestEntry } from './manifest_entry'; + +describe('manifest', () => { + describe('Manifest object sanity checks', () => { + test('Can create manifest with valid schema version', () => { + const manifest = new Manifest(Date.now(), '1.0.0'); + expect(manifest).toBeInstanceOf(Manifest); + }); + + test('Cannot create manifest with invalid schema version', () => { + expect(() => { + new Manifest(Date.now(), 'abcd'); + }).toThrow(); + }); + + test('Manifest transforms correctly to expected endpoint format', async () => { + const manifest = new Manifest(Date.now(), '1.0.0'); + manifest.addEntry(await getInternalArtifactMock('linux', '1.0.0')); + manifest.addEntry(await getInternalArtifactMock('macos', '1.0.0')); + manifest.addEntry(await getInternalArtifactMock('windows', '1.0.0')); + manifest.setVersion('abcd'); + expect(manifest.toEndpointFormat()).toStrictEqual({ + artifacts: { + 'endpoint-allowlist-linux-1.0.0': { + sha256: '222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', + size: 268, + url: + '/api/endpoint/allowlist/download/endpoint-allowlist-linux-1.0.0/222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', + }, + 'endpoint-allowlist-macos-1.0.0': { + sha256: '222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', + size: 268, + url: + '/api/endpoint/allowlist/download/endpoint-allowlist-macos-1.0.0/222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', + }, + 'endpoint-allowlist-windows-1.0.0': { + sha256: '222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', + size: 268, + url: + '/api/endpoint/allowlist/download/endpoint-allowlist-windows-1.0.0/222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', + }, + }, + manifestVersion: 'abcd', + schemaVersion: '1.0.0', + }); + }); + + test('Manifest cannot be converted to endpoint format without a version', async () => { + const manifest = new Manifest(Date.now(), '1.0.0'); + manifest.addEntry(await getInternalArtifactMock('linux', '1.0.0')); + manifest.addEntry(await getInternalArtifactMock('macos', '1.0.0')); + manifest.addEntry(await getInternalArtifactMock('windows', '1.0.0')); + expect(manifest.toEndpointFormat).toThrow(); + }); + + test('Manifest transforms correctly to expected saved object format', async () => { + const now = Date.now(); + const manifest = new Manifest(now, '1.0.0'); + manifest.addEntry(await getInternalArtifactMock('linux', '1.0.0')); + manifest.addEntry(await getInternalArtifactMock('macos', '1.0.0')); + manifest.addEntry(await getInternalArtifactMock('windows', '1.0.0')); + manifest.setVersion('abcd'); + expect(manifest.toSavedObject()).toStrictEqual({ + created: now, + ids: [ + 'endpoint-allowlist-linux-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', + 'endpoint-allowlist-macos-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', + 'endpoint-allowlist-windows-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', + ], + }); + }); + + test('Manifest returns diffs since supplied manifest', async () => { + // TODO + }); + + test('Manifest returns data for given artifact', async () => { + // TODO + }); + + test('Manifest returns entries map', async () => { + // TODO + }); + + test('Manifest returns true if contains artifact', async () => { + // TODO + }); + + test('Manifest can be created from list of artifacts', async () => { + // TODO + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts index 2a89c1965ede2..c5ab0423f2ac9 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts @@ -4,13 +4,15 @@ * you may not use this file except in compliance with the Elastic License. */ -import { ManifestEntry } from './manifest_entry'; +import { validate } from '../../../../common/validate'; import { InternalArtifactSchema, InternalManifestSchema, ManifestSchema, - ManifestSchemaVersion, + manifestSchema, + manifestSchemaVersion, } from '../../schemas/artifacts'; +import { ManifestEntry } from './manifest_entry'; export interface ManifestDiff { type: string; @@ -28,7 +30,12 @@ export class Manifest { constructor(created: Date, schemaVersion: string) { this.created = created; this.entries = {}; - this.schemaVersion = schemaVersion; + + const [validated, errors] = validate(schemaVersion, manifestSchemaVersion); + if (errors != null) { + throw new Error(`Invalid manifest version: ${schemaVersion}`); + } + this.schemaVersion = validated; } public static fromArtifacts( @@ -71,13 +78,6 @@ export class Manifest { return this.entries[artifactId].getArtifact(); } - public copy(): Manifest { - const manifest = new Manifest(this.created, this.schemaVersion); - manifest.entries = { ...this.entries }; - manifest.version = this.version; - return manifest; - } - public diff(manifest: Manifest): ManifestDiff[] { const diffs: ManifestDiff[] = []; @@ -97,9 +97,9 @@ export class Manifest { } public toEndpointFormat(): ManifestSchema { - const manifestObj: object = { - manifestVersion: 'todo', - schemaVersion: 'todo', + const manifestObj = { + manifestVersion: this.version, + schemaVersion: this.schemaVersion, artifacts: {}, }; @@ -107,7 +107,12 @@ export class Manifest { manifestObj.artifacts[entry.getIdentifier()] = entry.getRecord(); } - return manifestObj as ManifestSchema; + const [validated, errors] = validate(manifestObj, manifestSchema); + if (errors != null) { + throw new Error(errors); + } + + return validated; } public toSavedObject(): InternalManifestSchema { diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/index.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/index.ts index b538a9b3328cc..213152ff33b36 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/index.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/index.ts @@ -6,7 +6,9 @@ export * from './common'; export * from './lists'; +export * from './lists.mock'; export * from './manifest_schema'; export * from './request'; export * from './response'; export * from './saved_objects'; +export * from './saved_objects.mock'; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.mock.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.mock.ts index 41bc2aa258807..0147a53838381 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.mock.ts @@ -3,3 +3,30 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ + +import { FinalTranslatedExceptionList } from '../../lib/artifacts/lists'; + +export const getTranslatedExceptionListMock = (): FinalTranslatedExceptionList => { + return { + exceptions_list: [ + { + entries: [ + { + field: 'some.not.nested.field', + operator: 'included', + type: 'exact_cased', + value: 'some value', + }, + ], + field: 'some.field', + type: 'nested', + }, + { + field: 'some.not.nested.field', + operator: 'included', + type: 'exact_cased', + value: 'some value', + }, + ], + }; +}; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts index e23b627f76180..e7e006050f601 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts @@ -5,18 +5,20 @@ */ import { InternalArtifactSchema, InternalManifestSchema } from './saved_objects'; +import { getTranslatedExceptionListMock } from './lists.mock'; +import { buildArtifact } from '../../lib/artifacts/lists'; +import { ArtifactConstants } from './common'; -// TODO: os type -export const getInternalArtifactMock = (os?: string): InternalArtifactSchema => ({ - identifier: '', - sha256: '', - encoding: '', - created: '', - body: '', - size: '', -}); +export const getInternalArtifactMock = async (os?: string): Promise => { + const osParam = os === undefined ? 'linux' : os; + return buildArtifact(getTranslatedExceptionListMock(os), osParam, '1.0.0'); +}; -export const getInternalArtifactsMock = (): InternalArtifactSchema[] => [{}, {}]; +export const getInternalArtifactsMock = async (): Promise => { + return ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS.map(async (os) => { + await buildArtifact(getTranslatedExceptionListMock(os), os, '1.0.0'); + }); +}; export const getInternalManifestMock = (): InternalManifestSchema => ({ created: Date.now(), diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts index 445d5944ef6e5..e9b7676b68a22 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts @@ -11,6 +11,7 @@ import { ManifestConstants, Manifest, buildArtifact, + getFullEndpointExceptionList, } from '../../../lib/artifacts'; import { InternalArtifactSchema, InternalManifestSchema } from '../../../schemas/artifacts'; import { ArtifactClient } from '../artifact_client'; @@ -60,7 +61,12 @@ export class ManifestManager { const artifacts: InternalArtifactSchema[] = []; for (const os of ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS) { - const artifact = buildArtifact(this.exceptionListClient, os, schemaVersion); + const exceptionList = await getFullEndpointExceptionList( + this.exceptionListClient, + os, + schemaVersion + ); + const artifact = await buildArtifact(exceptionList, os, schemaVersion); artifacts.push(artifact); } diff --git a/yarn.lock b/yarn.lock index 827b94aaae6cf..07b4b697f1243 100644 --- a/yarn.lock +++ b/yarn.lock @@ -22585,22 +22585,6 @@ node-pre-gyp@^0.11.0: semver "^5.3.0" tar "^4" -node-pre-gyp@^0.11.0: - version "0.11.0" - resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.11.0.tgz#db1f33215272f692cd38f03238e3e9b47c5dd054" - integrity sha512-TwWAOZb0j7e9eGaf9esRx3ZcLaE5tQ2lvYy1pb5IAaG1a2e2Kv5Lms1Y4hpj+ciXJRofIxxlt5haeQ/2ANeE0Q== - dependencies: - detect-libc "^1.0.2" - mkdirp "^0.5.1" - needle "^2.2.1" - nopt "^4.0.1" - npm-packlist "^1.1.6" - npmlog "^4.0.2" - rc "^1.2.7" - rimraf "^2.6.1" - semver "^5.3.0" - tar "^4" - node-preload@^0.2.1: version "0.2.1" resolved "https://registry.yarnpkg.com/node-preload/-/node-preload-0.2.1.tgz#c03043bb327f417a18fee7ab7ee57b408a144301" From 2e6e65123931e4ef56f39c0bac5dd6df15e610a0 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 23 Jun 2020 23:04:00 -0400 Subject: [PATCH 065/106] even more tests and mocks --- .../endpoint/lib/artifacts/manifest.test.ts | 95 +++++++++++++++---- .../lib/artifacts/manifest_entry.test.ts | 58 +++++++++++ .../schemas/artifacts/saved_objects.mock.ts | 26 +++-- .../artifacts/artifact_client.test.ts | 5 + .../artifacts/manifest_client.test.ts | 5 + .../manifest_manager/manifest_manager.test.ts | 11 +++ 6 files changed, 175 insertions(+), 25 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.test.ts diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts index de5e560a9f4f1..4636070edc757 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts @@ -4,12 +4,42 @@ * you may not use this file except in compliance with the Elastic License. */ -import { getInternalArtifactMock } from '../../schemas'; +import { + InternalArtifactSchema, + getInternalArtifactMock, + getInternalArtifactMockWithDiffs, +} from '../../schemas'; import { Manifest } from './manifest'; import { ManifestEntry } from './manifest_entry'; describe('manifest', () => { describe('Manifest object sanity checks', () => { + const artifacts: InternalArtifactSchema[] = []; + let now: Date; + let manifest1: Manifest; + let manifest2: Manifest; + + beforeAll(async () => { + const artifactLinux = await getInternalArtifactMock('linux', '1.0.0'); + const artifactMacos = await getInternalArtifactMock('macos', '1.0.0'); + const artifactWindows = await getInternalArtifactMock('windows', '1.0.0'); + artifacts.push(artifactLinux); + artifacts.push(artifactMacos); + artifacts.push(artifactWindows); + + manifest1 = new Manifest(now, '1.0.0'); + manifest1.addEntry(artifactLinux); + manifest1.addEntry(artifactMacos); + manifest1.addEntry(artifactWindows); + manifest1.setVersion('abcd'); + + const newArtifactLinux = await getInternalArtifactMockWithDiffs('linux', '1.0.0'); + manifest2 = new Manifest(Date.now(), '1.0.0'); + manifest2.addEntry(newArtifactLinux); + manifest2.addEntry(artifactMacos); + manifest2.addEntry(artifactWindows); + }); + test('Can create manifest with valid schema version', () => { const manifest = new Manifest(Date.now(), '1.0.0'); expect(manifest).toBeInstanceOf(Manifest); @@ -22,12 +52,7 @@ describe('manifest', () => { }); test('Manifest transforms correctly to expected endpoint format', async () => { - const manifest = new Manifest(Date.now(), '1.0.0'); - manifest.addEntry(await getInternalArtifactMock('linux', '1.0.0')); - manifest.addEntry(await getInternalArtifactMock('macos', '1.0.0')); - manifest.addEntry(await getInternalArtifactMock('windows', '1.0.0')); - manifest.setVersion('abcd'); - expect(manifest.toEndpointFormat()).toStrictEqual({ + expect(manifest1.toEndpointFormat()).toStrictEqual({ artifacts: { 'endpoint-allowlist-linux-1.0.0': { sha256: '222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', @@ -62,13 +87,7 @@ describe('manifest', () => { }); test('Manifest transforms correctly to expected saved object format', async () => { - const now = Date.now(); - const manifest = new Manifest(now, '1.0.0'); - manifest.addEntry(await getInternalArtifactMock('linux', '1.0.0')); - manifest.addEntry(await getInternalArtifactMock('macos', '1.0.0')); - manifest.addEntry(await getInternalArtifactMock('windows', '1.0.0')); - manifest.setVersion('abcd'); - expect(manifest.toSavedObject()).toStrictEqual({ + expect(manifest1.toSavedObject()).toStrictEqual({ created: now, ids: [ 'endpoint-allowlist-linux-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', @@ -79,23 +98,61 @@ describe('manifest', () => { }); test('Manifest returns diffs since supplied manifest', async () => { - // TODO + const diffs = manifest2.diff(manifest1); + expect(diffs).toEqual([ + { + id: + 'endpoint-allowlist-linux-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', + type: 'delete', + }, + { + id: + 'endpoint-allowlist-linux-1.0.0-03114bf3dc2258f0def5beaf675242b68b428c96eefab5f6c5533f0d8e4deb0b', + type: 'add', + }, + ]); }); test('Manifest returns data for given artifact', async () => { - // TODO + const artifact = artifacts[0]; + const returned = manifest1.getArtifact(`${artifact.identifier}-${artifact.sha256}`); + expect(returned).toEqual(artifact); }); test('Manifest returns entries map', async () => { - // TODO + const entries = manifest1.getEntries(); + const keys = Object.keys(entries); + expect(keys).toEqual([ + 'endpoint-allowlist-linux-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', + 'endpoint-allowlist-macos-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', + 'endpoint-allowlist-windows-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', + ]); }); test('Manifest returns true if contains artifact', async () => { - // TODO + const found = manifest1.contains( + 'endpoint-allowlist-macos-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466' + ); + expect(found).toEqual(true); }); test('Manifest can be created from list of artifacts', async () => { - // TODO + const manifest = Manifest.fromArtifacts(artifacts, '1.0.0'); + expect( + manifest.contains( + 'endpoint-allowlist-linux-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466' + ) + ).toEqual(true); + expect( + manifest.contains( + 'endpoint-allowlist-macos-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466' + ) + ).toEqual(true); + expect( + manifest.contains( + 'endpoint-allowlist-windows-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466' + ) + ).toEqual(true); }); }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts index 41bc2aa258807..dd28fb5f405df 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts @@ -3,3 +3,61 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ + +import { InternalArtifactSchema, getInternalArtifactMock } from '../../schemas'; +import { ManifestEntry } from './manifest_entry'; + +describe('manifest_entry', () => { + describe('ManifestEntry object sanity checks', () => { + let artifact: InternalArtifactSchema; + let manifestEntry: ManifestEntry; + + beforeAll(async () => { + artifact = await getInternalArtifactMock('windows', '1.0.0'); + manifestEntry = new ManifestEntry(artifact); + }); + + test('Can create manifest entry', async () => { + expect(manifestEntry).toBeInstanceOf(ManifestEntry); + }); + + test('Correct doc_id is returned', async () => { + expect(manifestEntry.getDocId()).toEqual( + 'endpoint-allowlist-windows-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466' + ); + }); + + test('Correct identifier is returned', async () => { + expect(manifestEntry.getIdentifier()).toEqual('endpoint-allowlist-windows-1.0.0'); + }); + + test('Correct sha256 is returned', async () => { + expect(manifestEntry.getSha256()).toEqual( + '222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466' + ); + }); + + test('Correct size is returned', async () => { + expect(manifestEntry.getSize()).toEqual(268); + }); + + test('Correct url is returned', async () => { + expect(manifestEntry.getUrl()).toEqual( + '/api/endpoint/allowlist/download/endpoint-allowlist-windows-1.0.0/222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466' + ); + }); + + test('Correct artifact is returned', async () => { + expect(manifestEntry.getArtifact()).toEqual(artifact); + }); + + test('Correct record is returned', async () => { + expect(manifestEntry.getRecord()).toEqual({ + sha256: '222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', + size: 268, + url: + '/api/endpoint/allowlist/download/endpoint-allowlist-windows-1.0.0/222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', + }); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts index e7e006050f601..747ac35a97c3d 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts @@ -9,14 +9,28 @@ import { getTranslatedExceptionListMock } from './lists.mock'; import { buildArtifact } from '../../lib/artifacts/lists'; import { ArtifactConstants } from './common'; -export const getInternalArtifactMock = async (os?: string): Promise => { - const osParam = os === undefined ? 'linux' : os; - return buildArtifact(getTranslatedExceptionListMock(os), osParam, '1.0.0'); +export const getInternalArtifactMock = async ( + os: string, + schemaVersion: string +): Promise => { + return buildArtifact(getTranslatedExceptionListMock(), os, schemaVersion); }; -export const getInternalArtifactsMock = async (): Promise => { - return ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS.map(async (os) => { - await buildArtifact(getTranslatedExceptionListMock(os), os, '1.0.0'); +export const getInternalArtifactMockWithDiffs = async ( + os: string, + schemaVersion: string +): Promise => { + const mock = getTranslatedExceptionListMock(); + mock.exceptions_list.pop(); + return buildArtifact(mock, os, schemaVersion); +}; + +export const getInternalArtifactsMock = async ( + os: string, + schemaVersion: string +): Promise => { + return ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS.map(async () => { + await buildArtifact(getTranslatedExceptionListMock(), os, schemaVersion); }); }; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts new file mode 100644 index 0000000000000..41bc2aa258807 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts @@ -0,0 +1,5 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.test.ts new file mode 100644 index 0000000000000..41bc2aa258807 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.test.ts @@ -0,0 +1,5 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts index 41bc2aa258807..2884d7650dc02 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts @@ -3,3 +3,14 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ + +import { ManifestManager } from './manifest_manager'; +import { ManifestManagerMock } from './manifest_manager.mock'; + +describe('manifest_manager', () => { + describe('ManifestManager sanity checks', () => { + beforeAll(async () => {}); + + test('Can do a test', () => {}); + }); +}); From 673f35bdcf9a7bf6e3abfbf5aa1f2ec42d7f73b3 Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Wed, 24 Jun 2020 10:18:31 -0400 Subject: [PATCH 066/106] Merging both refactors --- .../server/endpoint/lib/artifacts/lists.ts | 105 ++++++++++-------- 1 file changed, 60 insertions(+), 45 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts index 1a02906946712..43f58761bc631 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts @@ -5,8 +5,9 @@ */ import { createHash } from 'crypto'; - import lzma from 'lzma-native'; +import { validate } from '../../../../common/validate'; + import { Entry, EntryNested, @@ -16,20 +17,21 @@ import { import { FoundExceptionListItemSchema } from '../../../../../lists/common/schemas/response/found_exception_list_item_schema'; import { ExceptionListClient } from '../../../../../lists/server'; import { - TranslatedExceptionList, TranslatedEntry, TranslatedEntryMatch, TranslatedEntryMatchAny, TranslatedEntryNested, FinalTranslatedExceptionList, + InternalArtifactSchema, + finalExceptionList, } from '../../schemas'; -import { ArtifactConstants } from './common'; +import { ArtifactConstants } from '.'; export async function buildArtifact( exceptions: FinalTranslatedExceptionList, os: string, schemaVersion: string -): InternalArtifactSchema { +): Promise { const compressedExceptions: Buffer = await compressExceptionList(exceptions); const sha256 = createHash('sha256') @@ -48,7 +50,7 @@ export async function buildArtifact( export async function getFullEndpointExceptionList( eClient: ExceptionListClient, - os: string, // TODO: make type + os: string, schemaVersion: string ): Promise { const exceptions: FinalTranslatedExceptionList = { exceptions_list: [] }; // todo type @@ -79,7 +81,11 @@ export async function getFullEndpointExceptionList( } } while (numResponses > 0); - return exceptions; + const [validated, errors] = validate(exceptions, finalExceptionList); + if (errors != null) { + throw new Error(errors); + } + return validated as FinalTranslatedExceptionList; } /** @@ -93,56 +99,65 @@ function translateToEndpointExceptions( const translatedList: TranslatedEntry[] = []; if (schemaVersion === '1.0.0') { - // Transform to endpoint format exc.data.forEach((list) => { list.entries.forEach((entry) => { - // TODO case sensitive? - translatedList.push(translateEntry(schemaVersion, entry)); + const tEntry = translateEntry(schemaVersion, entry); + if (tEntry !== undefined) { + translatedList.push(tEntry); + } }); }); } else { throw new Error('unsupported schemaVersion'); } - - // TODO: validate - return translatedList; } -function translateEntry(schemaVersion: string, entry: Entry | EntryNested): TranslatedEntry { - let translatedEntry: TranslatedEntry; - if (entry.type === 'nested') { - const e = (entry as unknown) as EntryNested; - const nestedEntries: TranslatedEntry[] = []; - e.entries.forEach((nestedEntry) => { - nestedEntries.push(translateEntry(schemaVersion, nestedEntry)); - }); - translatedEntry = { - entries: nestedEntries, - field: e.field, - type: 'nested', - } as TranslatedEntryNested; - } else if (entry.type === 'match') { - const e = (entry as unknown) as EntryMatch; - translatedEntry = { - field: e.field, - operator: e.operator, - type: e.field.endsWith('.text') ? 'exact_caseless' : 'exact_cased', - value: e.value, - } as TranslatedEntryMatch; - } else if (entry.type === 'match_any') { - const e = (entry as unknown) as EntryMatchAny; - translatedEntry = { - field: e.field, - operator: e.operator, - type: e.field.endsWith('.text') ? 'exact_caseless_any' : 'exact_cased_any', - value: e.value, - } as TranslatedEntryMatchAny; - } else { - // TODO: log and continue? - throw new Error(`Invalid type encountered: ${entry.type}`); +function translateEntry( + schemaVersion: string, + entry: Entry | EntryNested +): TranslatedEntry | undefined { + let translatedEntry; + switch (entry.type) { + case 'nested': { + const e = (entry as unknown) as EntryNested; + const nestedEntries: TranslatedEntry[] = []; + for (const nestedEntry of e.entries) { + const translation = translateEntry(schemaVersion, nestedEntry); + if (translation !== undefined) { + nestedEntries.push(translation); + } + } + translatedEntry = { + entries: nestedEntries, + field: e.field, + type: 'nested', + } as TranslatedEntryNested; + break; + } + case 'match': { + const e = (entry as unknown) as EntryMatch; + translatedEntry = { + field: e.field, + operator: e.operator, + type: e.field.endsWith('.text') ? 'exact_caseless' : 'exact_cased', + value: e.value, + } as TranslatedEntryMatch; + break; + } + case 'match_any': + { + const e = (entry as unknown) as EntryMatchAny; + translatedEntry = { + field: e.field, + operator: e.operator, + type: e.field.endsWith('.text') ? 'exact_caseless_any' : 'exact_cased_any', + value: e.value, + } as TranslatedEntryMatchAny; + } + break; } - return translatedEntry; + return translatedEntry || undefined; } /** From 53df066e7d4c871ec24072371aa89af703f4715b Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Wed, 24 Jun 2020 11:11:19 -0400 Subject: [PATCH 067/106] Adding more tests for the convertion layer --- .../endpoint/lib/artifacts/lists.test.ts | 82 +++++++++++++++++++ .../server/endpoint/lib/artifacts/lists.ts | 16 ++-- .../endpoint/schemas/artifacts/lists.ts | 4 +- 3 files changed, 92 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts index d25e8a64cebb3..8d463503a0621 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts @@ -9,6 +9,7 @@ import { listMock } from '../../../../../lists/server/mocks'; import { getFoundExceptionListItemSchemaMock } from '../../../../../lists/common/schemas/response/found_exception_list_item_schema.mock'; import { getExceptionListItemSchemaMock } from '../../../../../lists/common/schemas/response/exception_list_item_schema.mock'; import { getFullEndpointExceptionList } from './lists'; +import { EntriesArray } from '../../../../../lists/common/schemas/types/entries'; describe('buildEventTypeSignal', () => { let mockExceptionClient: ExceptionListClient; @@ -48,6 +49,87 @@ describe('buildEventTypeSignal', () => { expect(resp).toEqual(expectedEndpointExceptions); }); + test('it should convert simple fields', async () => { + const testEntries: EntriesArray = [ + { field: 'server.domain', operator: 'included', type: 'match', value: 'DOMAIN' }, + { field: 'server.ip', operator: 'included', type: 'match', value: '192.168.1.1' }, + { field: 'host.hostname', operator: 'included', type: 'match', value: 'estc' }, + ]; + + const expectedEndpointExceptions = { + exceptions_list: [ + { + field: 'server.domain', + operator: 'included', + type: 'exact_cased', + value: 'DOMAIN', + }, + { + field: 'server.ip', + operator: 'included', + type: 'exact_cased', + value: '192.168.1.1', + }, + { + field: 'host.hostname', + operator: 'included', + type: 'exact_cased', + value: 'estc', + }, + ], + }; + + const first = getFoundExceptionListItemSchemaMock(); + first.data[0].entries = testEntries; + mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); + + const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); + expect(resp).toEqual(expectedEndpointExceptions); + }); + + test('it should convert fields case sensitive', async () => { + const testEntries: EntriesArray = [ + { field: 'server.domain.text', operator: 'included', type: 'match', value: 'DOMAIN' }, + { field: 'server.ip', operator: 'included', type: 'match', value: '192.168.1.1' }, + { + field: 'host.hostname.text', + operator: 'included', + type: 'match_any', + value: ['estc', 'kibana'], + }, + ]; + + const expectedEndpointExceptions = { + exceptions_list: [ + { + field: 'server.domain.text', + operator: 'included', + type: 'exact_caseless', + value: 'DOMAIN', + }, + { + field: 'server.ip', + operator: 'included', + type: 'exact_cased', + value: '192.168.1.1', + }, + { + field: 'host.hostname.text', + operator: 'included', + type: 'exact_caseless_any', + value: ['estc', 'kibana'], + }, + ], + }; + + const first = getFoundExceptionListItemSchemaMock(); + first.data[0].entries = testEntries; + mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); + + const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); + expect(resp).toEqual(expectedEndpointExceptions); + }); + test('it should convert the exception lists response to the proper endpoint format while paging', async () => { // The first call returns one exception const first = getFoundExceptionListItemSchemaMock(); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts index 43f58761bc631..5e454e49a781c 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts @@ -21,14 +21,14 @@ import { TranslatedEntryMatch, TranslatedEntryMatchAny, TranslatedEntryNested, - FinalTranslatedExceptionList, + WrappedTranslatedExceptionList, InternalArtifactSchema, - finalExceptionList, + wrappedExceptionList, } from '../../schemas'; import { ArtifactConstants } from '.'; export async function buildArtifact( - exceptions: FinalTranslatedExceptionList, + exceptions: WrappedTranslatedExceptionList, os: string, schemaVersion: string ): Promise { @@ -52,8 +52,8 @@ export async function getFullEndpointExceptionList( eClient: ExceptionListClient, os: string, schemaVersion: string -): Promise { - const exceptions: FinalTranslatedExceptionList = { exceptions_list: [] }; // todo type +): Promise { + const exceptions: WrappedTranslatedExceptionList = { exceptions_list: [] }; let numResponses = 0; let page = 1; @@ -81,11 +81,11 @@ export async function getFullEndpointExceptionList( } } while (numResponses > 0); - const [validated, errors] = validate(exceptions, finalExceptionList); + const [validated, errors] = validate(exceptions, wrappedExceptionList); if (errors != null) { throw new Error(errors); } - return validated as FinalTranslatedExceptionList; + return validated as WrappedTranslatedExceptionList; } /** @@ -163,7 +163,7 @@ function translateEntry( /** * Compresses the exception list */ -function compressExceptionList(exceptionList: TranslatedExceptionList): Promise { +function compressExceptionList(exceptionList: WrappedTranslatedExceptionList): Promise { return lzma.compress(JSON.stringify(exceptionList), (res: Buffer) => { return res; }); diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts index f7121d61e73a1..7a85f6a8a96ad 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts @@ -58,9 +58,9 @@ export const translatedExceptionList = t.exact( ); export type TranslatedExceptionList = t.TypeOf; -export const finalExceptionList = t.exact( +export const wrappedExceptionList = t.exact( t.type({ exceptions_list: t.array(translatedEntry), }) ); -export type FinalTranslatedExceptionList = t.TypeOf; +export type WrappedTranslatedExceptionList = t.TypeOf; From d7fa7b9422f88cbcdede1f1ee5a1f2d7bbde5040 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Wed, 24 Jun 2020 11:26:42 -0400 Subject: [PATCH 068/106] fix conflicts --- x-pack/plugins/security_solution/server/config.ts | 2 -- .../server/endpoint/lib/artifacts/lists.ts | 4 ++-- .../server/endpoint/lib/artifacts/manifest.test.ts | 9 ++++----- .../server/endpoint/lib/artifacts/manifest.ts | 7 +++++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/security_solution/server/config.ts b/x-pack/plugins/security_solution/server/config.ts index 4f51ecd53d193..f7d961ae3ec5c 100644 --- a/x-pack/plugins/security_solution/server/config.ts +++ b/x-pack/plugins/security_solution/server/config.ts @@ -29,8 +29,6 @@ export const configSchema = schema.object({ from: schema.string({ defaultValue: 'now-15m' }), to: schema.string({ defaultValue: 'now' }), }), - // TODO: why can't we remove this? :) - packagerTaskInterval: schema.string({ defaultValue: '60s' }), }); export const createConfig$ = (context: PluginInitializerContext) => diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts index 5e454e49a781c..77505e00f8c58 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts @@ -17,15 +17,15 @@ import { import { FoundExceptionListItemSchema } from '../../../../../lists/common/schemas/response/found_exception_list_item_schema'; import { ExceptionListClient } from '../../../../../lists/server'; import { + InternalArtifactSchema, TranslatedEntry, TranslatedEntryMatch, TranslatedEntryMatchAny, TranslatedEntryNested, WrappedTranslatedExceptionList, - InternalArtifactSchema, wrappedExceptionList, } from '../../schemas'; -import { ArtifactConstants } from '.'; +import { ArtifactConstants } from './common'; export async function buildArtifact( exceptions: WrappedTranslatedExceptionList, diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts index 4636070edc757..7139dadf94a0c 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts @@ -10,7 +10,6 @@ import { getInternalArtifactMockWithDiffs, } from '../../schemas'; import { Manifest } from './manifest'; -import { ManifestEntry } from './manifest_entry'; describe('manifest', () => { describe('Manifest object sanity checks', () => { @@ -34,20 +33,20 @@ describe('manifest', () => { manifest1.setVersion('abcd'); const newArtifactLinux = await getInternalArtifactMockWithDiffs('linux', '1.0.0'); - manifest2 = new Manifest(Date.now(), '1.0.0'); + manifest2 = new Manifest(new Date(), '1.0.0'); manifest2.addEntry(newArtifactLinux); manifest2.addEntry(artifactMacos); manifest2.addEntry(artifactWindows); }); test('Can create manifest with valid schema version', () => { - const manifest = new Manifest(Date.now(), '1.0.0'); + const manifest = new Manifest(new Date(), '1.0.0'); expect(manifest).toBeInstanceOf(Manifest); }); test('Cannot create manifest with invalid schema version', () => { expect(() => { - new Manifest(Date.now(), 'abcd'); + new Manifest(new Date(), 'abcd'); }).toThrow(); }); @@ -79,7 +78,7 @@ describe('manifest', () => { }); test('Manifest cannot be converted to endpoint format without a version', async () => { - const manifest = new Manifest(Date.now(), '1.0.0'); + const manifest = new Manifest(new Date(), '1.0.0'); manifest.addEntry(await getInternalArtifactMock('linux', '1.0.0')); manifest.addEntry(await getInternalArtifactMock('macos', '1.0.0')); manifest.addEntry(await getInternalArtifactMock('windows', '1.0.0')); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts index c5ab0423f2ac9..654d6473494ef 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts @@ -31,7 +31,10 @@ export class Manifest { this.created = created; this.entries = {}; - const [validated, errors] = validate(schemaVersion, manifestSchemaVersion); + const [validated, errors] = validate( + (schemaVersion as unknown) as object, + manifestSchemaVersion + ); if (errors != null) { throw new Error(`Invalid manifest version: ${schemaVersion}`); } @@ -42,7 +45,7 @@ export class Manifest { artifacts: InternalArtifactSchema[], schemaVersion: string ): Manifest { - const manifest = new Manifest(Date.now(), schemaVersion); + const manifest = new Manifest(new Date(), schemaVersion); artifacts.forEach((artifact) => { manifest.addEntry(artifact); }); From 44da0a5fb16dcf3cb6340a7c1f58e440c3a02e63 Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Wed, 24 Jun 2020 11:33:11 -0400 Subject: [PATCH 069/106] Adding lzma types --- package.json | 1 + x-pack/plugins/security_solution/package.json | 1 + yarn.lock | 7 +++++++ 3 files changed, 9 insertions(+) diff --git a/package.json b/package.json index 3eaa1fb05e906..efc329d552826 100644 --- a/package.json +++ b/package.json @@ -144,6 +144,7 @@ "@kbn/test-subj-selector": "0.2.1", "@kbn/ui-framework": "1.0.0", "@kbn/ui-shared-deps": "1.0.0", + "@types/lzma-native": "^4.0.0", "JSONStream": "1.3.5", "abortcontroller-polyfill": "^1.4.0", "accept": "3.0.2", diff --git a/x-pack/plugins/security_solution/package.json b/x-pack/plugins/security_solution/package.json index 312203854bd0f..b3cdf56c7256a 100644 --- a/x-pack/plugins/security_solution/package.json +++ b/x-pack/plugins/security_solution/package.json @@ -18,6 +18,7 @@ "dependencies": { "lodash": "^4.17.15", "lzma-native": "6.0.0", + "@types/lzma-native": "4.0.0", "querystring": "^0.2.0", "redux-devtools-extension": "^2.13.8", "@types/seedrandom": ">=2.0.0 <4.0.0" diff --git a/yarn.lock b/yarn.lock index 07b4b697f1243..6ad430fb6fc3d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5347,6 +5347,13 @@ resolved "https://registry.yarnpkg.com/@types/lru-cache/-/lru-cache-5.1.0.tgz#57f228f2b80c046b4a1bd5cac031f81f207f4f03" integrity sha512-RaE0B+14ToE4l6UqdarKPnXwVDuigfFv+5j9Dze/Nqr23yyuqdNvzcZi3xB+3Agvi5R4EOgAksfv3lXX4vBt9w== +"@types/lzma-native@4.0.0", "@types/lzma-native@^4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@types/lzma-native/-/lzma-native-4.0.0.tgz#86440e70b1431f2bc1a470eb0ea8bcfc5788fe14" + integrity sha512-9mBLyFWJe8/whIRjGwca5pbXv7ANzPj7KNgsgCU/l5z8xcyIe6LNae+iKdT5rjnXcXGOwyW/PKYt0N6+7N5QlA== + dependencies: + "@types/node" "*" + "@types/mapbox-gl@^1.9.1": version "1.9.1" resolved "https://registry.yarnpkg.com/@types/mapbox-gl/-/mapbox-gl-1.9.1.tgz#78b62f8a1ead78bc525a4c1db84bb71fa0fcc579" From 7ce4f286fb2b55de279f9bfaefcb26b7e3fdddd7 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Wed, 24 Jun 2020 11:50:07 -0400 Subject: [PATCH 070/106] Bug fixes --- .../services/artifacts/manifest_manager/manifest_manager.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts index e9b7676b68a22..0cd2ca501a2df 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts @@ -42,7 +42,7 @@ export class ManifestManager { } private async dispatch(manifest: Manifest) { - const manifestClient = this.getManifestClient(manifest.getSchemaVersion()); + const manifestClient = await this.getManifestClient(manifest.getSchemaVersion()); // TODO: dispatch and only update if successful @@ -75,7 +75,7 @@ export class ManifestManager { } private async getLastDispatchedManifest(schemaVersion: string): Promise { - const manifestClient = this.getManifestClient(schemaVersion); + const manifestClient = await this.getManifestClient(schemaVersion); let manifestSo: InternalManifestSchema; try { From 2a6103f962ae8d5b9c82763e370205a1129be225 Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Wed, 24 Jun 2020 11:52:22 -0400 Subject: [PATCH 071/106] lint --- .../endpoint/lib/artifacts/lists.test.ts | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts index 8d463503a0621..d29a6ad53bdcd 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts @@ -130,6 +130,38 @@ describe('buildEventTypeSignal', () => { expect(resp).toEqual(expectedEndpointExceptions); }); + test('it should ignore unsupported entries', async () => { + // Lists and exists are not supported by the Endpoint + const testEntries: EntriesArray = [ + { field: 'server.domain', operator: 'included', type: 'match', value: 'DOMAIN' }, + { + field: 'server.domain', + operator: 'included', + type: 'list', + value: ['lists', 'not', 'supported'], + }, + { field: 'server.ip', operator: 'included', type: 'exists' }, + ]; + + const expectedEndpointExceptions = { + exceptions_list: [ + { + field: 'server.domain', + operator: 'included', + type: 'exact_cased', + value: 'DOMAIN', + }, + ], + }; + + const first = getFoundExceptionListItemSchemaMock(); + first.data[0].entries = testEntries; + mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); + + const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); + expect(resp).toEqual(expectedEndpointExceptions); + }); + test('it should convert the exception lists response to the proper endpoint format while paging', async () => { // The first call returns one exception const first = getFoundExceptionListItemSchemaMock(); @@ -149,4 +181,13 @@ describe('buildEventTypeSignal', () => { const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); expect(resp.exceptions_list.length).toEqual(6); }); + + test('it should handle no exceptions', async () => { + const exceptionsResponse = getFoundExceptionListItemSchemaMock(); + exceptionsResponse.data = []; + exceptionsResponse.total = 0; + mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(exceptionsResponse); + const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); + expect(resp.exceptions_list.length).toEqual(0); + }); }); From 5a52d1b355a319642b9e6d3eafb4e5468c1f4d47 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Wed, 24 Jun 2020 12:54:09 -0400 Subject: [PATCH 072/106] resolve some type errors --- .../server/endpoint/lib/artifacts/lists.ts | 6 +++-- .../endpoint/lib/artifacts/manifest.test.ts | 4 ++-- .../server/endpoint/lib/artifacts/manifest.ts | 13 +++++----- .../artifacts/download_exception_list.test.ts | 9 +++---- .../endpoint/routes/metadata/metadata.test.ts | 2 ++ .../endpoint/routes/policy/handlers.test.ts | 2 ++ .../schemas/artifacts/saved_objects.mock.ts | 3 +-- .../artifacts/artifact_client.mock.ts | 8 +++---- .../artifacts/manifest_client.mock.ts | 18 +++++++++----- .../manifest_manager/manifest_manager.mock.ts | 24 ++++++++++++++----- .../manifest_manager/manifest_manager.test.ts | 2 +- .../manifest_manager/manifest_manager.ts | 8 +++---- 12 files changed, 62 insertions(+), 37 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts index 77505e00f8c58..417f2f41b4e04 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts @@ -92,7 +92,7 @@ export async function getFullEndpointExceptionList( * Translates Exception list items to Exceptions the endpoint can understand * @param exc */ -function translateToEndpointExceptions( +export function translateToEndpointExceptions( exc: FoundExceptionListItemSchema, schemaVersion: string ): TranslatedEntry[] { @@ -163,7 +163,9 @@ function translateEntry( /** * Compresses the exception list */ -function compressExceptionList(exceptionList: WrappedTranslatedExceptionList): Promise { +export function compressExceptionList( + exceptionList: WrappedTranslatedExceptionList +): Promise { return lzma.compress(JSON.stringify(exceptionList), (res: Buffer) => { return res; }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts index 7139dadf94a0c..8c94f9e94c80f 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts @@ -14,7 +14,7 @@ import { Manifest } from './manifest'; describe('manifest', () => { describe('Manifest object sanity checks', () => { const artifacts: InternalArtifactSchema[] = []; - let now: Date; + const now = new Date(); let manifest1: Manifest; let manifest2: Manifest; @@ -87,7 +87,7 @@ describe('manifest', () => { test('Manifest transforms correctly to expected saved object format', async () => { expect(manifest1.toSavedObject()).toStrictEqual({ - created: now, + created: now.getTime(), ids: [ 'endpoint-allowlist-linux-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', 'endpoint-allowlist-macos-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts index 654d6473494ef..7a4f2be24251e 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts @@ -9,6 +9,7 @@ import { InternalArtifactSchema, InternalManifestSchema, ManifestSchema, + ManifestSchemaVersion, manifestSchema, manifestSchemaVersion, } from '../../schemas/artifacts'; @@ -22,7 +23,7 @@ export interface ManifestDiff { export class Manifest { private created: Date; private entries: Record; - private schemaVersion: string; + private schemaVersion: ManifestSchemaVersion; // For concurrency control private version: string | undefined; @@ -43,7 +44,7 @@ export class Manifest { public static fromArtifacts( artifacts: InternalArtifactSchema[], - schemaVersion: string + schemaVersion: ManifestSchemaVersion ): Manifest { const manifest = new Manifest(new Date(), schemaVersion); artifacts.forEach((artifact) => { @@ -52,7 +53,7 @@ export class Manifest { return manifest; } - public getSchemaVersion(): string { + public getSchemaVersion(): ManifestSchemaVersion { return this.schemaVersion; } @@ -100,7 +101,7 @@ export class Manifest { } public toEndpointFormat(): ManifestSchema { - const manifestObj = { + const manifestObj: ManifestSchema = { manifestVersion: this.version, schemaVersion: this.schemaVersion, artifacts: {}, @@ -115,12 +116,12 @@ export class Manifest { throw new Error(errors); } - return validated; + return validated as ManifestSchema; } public toSavedObject(): InternalManifestSchema { return { - created: this.created, + created: this.created.getTime(), ids: Object.keys(this.entries), }; } diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts index 4b5fbc809fa1a..d6cbe48517486 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts @@ -22,12 +22,13 @@ import { loggingServiceMock, } from 'src/core/server/mocks'; import { ExceptionsCache } from '../../lib/artifacts/cache'; -import { CompressExceptionList } from '../../lib/artifacts/lists'; +import { compressExceptionList } from '../../lib/artifacts/lists'; import { ArtifactConstants } from '../../lib/artifacts'; import { registerDownloadExceptionListRoute } from './download_exception_list'; import { EndpointAppContextService } from '../../endpoint_app_context_services'; import { createMockAgentService } from '../../mocks'; import { createMockConfig } from '../../../lib/detection_engine/routes/__mocks__'; +import { getManifestManagerMock } from '../../services/artifacts'; const mockArtifactName = `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-windows-1.0.0`; const expectedEndpointExceptions = { @@ -73,7 +74,7 @@ describe('test alerts route', () => { endpointAppContextService.start({ agentService: createMockAgentService(), - manifestManager: undefined, + manifestManager: getManifestManagerMock(), }); registerDownloadExceptionListRoute( @@ -94,7 +95,7 @@ describe('test alerts route', () => { params: { sha256: '123456' }, }); - const mockCompressedArtifact = await CompressExceptionList(expectedEndpointExceptions); + const mockCompressedArtifact = await compressExceptionList(expectedEndpointExceptions); const mockArtifact = { id: '2468', @@ -183,7 +184,7 @@ describe('test alerts route', () => { }); // Add to the download cache - const mockCompressedArtifact = await CompressExceptionList(expectedEndpointExceptions); + const mockCompressedArtifact = await compressExceptionList(expectedEndpointExceptions); const cacheKey = `${mockArtifactName}-${mockSha}`; cache.set(cacheKey, mockCompressedArtifact.toString('binary')); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata.test.ts index ba51a3b6aa92e..11db530d18808 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata.test.ts @@ -31,6 +31,7 @@ import { createMockAgentService, createRouteHandlerContext } from '../../mocks'; import { AgentService } from '../../../../../ingest_manager/server'; import Boom from 'boom'; import { EndpointAppContextService } from '../../endpoint_app_context_services'; +import { getManifestManagerMock } from '../../services/artifacts/manifest_manager/manifest_manager.mock'; import { createMockConfig } from '../../../lib/detection_engine/routes/__mocks__'; import { EndpointDocGenerator } from '../../../../common/endpoint/generate_data'; @@ -60,6 +61,7 @@ describe('test endpoint route', () => { endpointAppContextService = new EndpointAppContextService(); endpointAppContextService.start({ agentService: mockAgentService, + manifestManager: getManifestManagerMock(), }); registerEndpointRoutes(routerMock, { diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/policy/handlers.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/policy/handlers.test.ts index 6c1f0a206ffaa..6138a283478a9 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/policy/handlers.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/policy/handlers.test.ts @@ -19,6 +19,7 @@ import { } from '../../../../../../../src/core/server/mocks'; import { AgentService } from '../../../../../ingest_manager/server/services'; import { SearchResponse } from 'elasticsearch'; +import { getManifestManagerMock } from '../../services/artifacts/manifest_manager/manifest_manager.mock'; import { GetHostPolicyResponse, HostPolicyResponse } from '../../../../common/endpoint/types'; import { EndpointDocGenerator } from '../../../../common/endpoint/generate_data'; import { createMockConfig } from '../../../lib/detection_engine/routes/__mocks__'; @@ -38,6 +39,7 @@ describe('test policy response handler', () => { mockAgentService = createMockAgentService(); endpointAppContextService.start({ agentService: mockAgentService, + manifestManager: getManifestManagerMock(), }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts index 747ac35a97c3d..a1b768b2e69ff 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts @@ -6,8 +6,7 @@ import { InternalArtifactSchema, InternalManifestSchema } from './saved_objects'; import { getTranslatedExceptionListMock } from './lists.mock'; -import { buildArtifact } from '../../lib/artifacts/lists'; -import { ArtifactConstants } from './common'; +import { ArtifactConstants, buildArtifact } from '../../lib/artifacts'; export const getInternalArtifactMock = async ( os: string, diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.mock.ts index 07d2731312591..2c676dc2ae5bd 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.mock.ts @@ -5,10 +5,10 @@ */ import { ArtifactClient } from './artifact_client'; -import { getInternalArtifactSchemaMock } from '../../schemas'; +import { getInternalArtifactMock } from '../../schemas'; export class ArtifactClientMock extends ArtifactClient { - public getArtifact = jest.fn().mockResolvedValue(getInternalArtifactSchemaMock()); - public createArtifact = jest.fn().mockResolvedValue(getInternalArtifactSchemaMock()); - public deleteArtifact = jest.fn().mockResolvedValue(getInternalArtifactSchemaMock()); + public getArtifact = jest.fn().mockResolvedValue(getInternalArtifactMock()); + public createArtifact = jest.fn().mockResolvedValue(getInternalArtifactMock()); + public deleteArtifact = jest.fn().mockResolvedValue(getInternalArtifactMock()); } diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.mock.ts index 5bf37ffea5556..7ed580721ff10 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.mock.ts @@ -4,12 +4,18 @@ * you may not use this file except in compliance with the Elastic License. */ -import { ManifestClient } from './manifest_service'; -import { getInternalManifestSchemaMock } from '../../schemas'; +import { savedObjectsClientMock } from '../../../../../../../src/core/server/mocks'; + +import { ManifestClient } from './manifest_client'; +import { getInternalManifestMock } from '../../schemas'; export class ManifestClientMock extends ManifestClient { - public createManifest = jest.fn().mockResolvedValue(getInternalManifestSchemaMock()); - public getManifest = jest.fn().mockResolvedValue(getInternalManifestSchemaMock()); - public updateManifest = jest.fn().mockResolvedValue(getInternalManifestSchemaMock()); - public deleteManifest = jest.fn().mockResolvedValue(getInternalManifestSchemaMock()); + public createManifest = jest.fn().mockResolvedValue(getInternalManifestMock()); + public getManifest = jest.fn().mockResolvedValue(getInternalManifestMock()); + public updateManifest = jest.fn().mockResolvedValue(getInternalManifestMock()); + public deleteManifest = jest.fn().mockResolvedValue(getInternalManifestMock()); } + +export const getManifestClientMock = (): ManifestClientMock => { + return new ManifestClientMock(savedObjectsClientMock.create(), '1.0.0'); +}; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts index d165cec605e28..ba241bf65946f 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts @@ -4,9 +4,21 @@ * you may not use this file except in compliance with the Elastic License. */ -import { ManifestManager } from './artifact_client'; +// TODO +// eslint-disable-next-line @kbn/eslint/no-restricted-paths +import { loggerMock } from '../../../../../../../../src/core/server/logging/logger.mock'; + +import { savedObjectsClientMock } from '../../../../../../../../src/core/server/mocks'; +import { Manifest } from '../../../lib/artifacts'; +import { ManifestManager } from './manifest_manager'; + +// TODO +// eslint-disable-next-line @kbn/eslint/no-restricted-paths +import { getExceptionListClientMock } from '../../../../../../lists/server/services/exception_lists/exception_list_client.mock'; import { getInternalArtifactMock, getInternalArtifactsMock } from '../../../schemas'; +import { getArtifactClientMock } from '../artifact_client.mock'; +import { getManifestClientMock } from '../manifest_client.mock'; function mockBuildExceptionListArtifacts() { // mock buildArtifactFunction @@ -14,21 +26,21 @@ function mockBuildExceptionListArtifacts() { // getInternalArtifactsMock() } -/* export class ManifestManagerMock extends ManifestManager { private buildExceptionListArtifacts = jest .fn() .mockResolvedValue(mockBuildExceptionListArtifacts()); - private getLastDispatchedManifest = jest.fn().mockResolvedValue(getManifestMock()); + private getLastDispatchedManifest = jest + .fn() + .mockResolvedValue(new Manifest(new Date(), '1.0.0')); private getManifestClient = jest.fn().mockValue(getManifestClientMock()); } -export const getManifestManagerMock = (): ManifestManager => { - const mock = new ManifestManager({ +export const getManifestManagerMock = (): ManifestManagerMock => { + return new ManifestManagerMock({ artifactClient: getArtifactClientMock(), exceptionListClient: getExceptionListClientMock(), savedObjectsClient: savedObjectsClientMock.create(), logger: loggerMock.create(), }); }; -*/ diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts index 2884d7650dc02..08ed343995411 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts @@ -5,7 +5,7 @@ */ import { ManifestManager } from './manifest_manager'; -import { ManifestManagerMock } from './manifest_manager.mock'; +import { getManifestManagerMock } from './manifest_manager.mock'; describe('manifest_manager', () => { describe('ManifestManager sanity checks', () => { diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts index 0cd2ca501a2df..2caa02bfe7417 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts @@ -37,12 +37,12 @@ export class ManifestManager { this.logger = context.logger; } - private async getManifestClient(schemaVersion: string): ManifestClient { + private getManifestClient(schemaVersion: string): ManifestClient { return new ManifestClient(this.savedObjectsClient, schemaVersion); } private async dispatch(manifest: Manifest) { - const manifestClient = await this.getManifestClient(manifest.getSchemaVersion()); + const manifestClient = this.getManifestClient(manifest.getSchemaVersion()); // TODO: dispatch and only update if successful @@ -75,7 +75,7 @@ export class ManifestManager { } private async getLastDispatchedManifest(schemaVersion: string): Promise { - const manifestClient = await this.getManifestClient(schemaVersion); + const manifestClient = this.getManifestClient(schemaVersion); let manifestSo: InternalManifestSchema; try { @@ -115,7 +115,7 @@ export class ManifestManager { if (oldManifest === null) { if (createInitial) { // TODO: implement this when ready to integrate with Paul's code - oldManifest = new Manifest(Date.now(), ManifestConstants.SCHEMA_VERSION); // create empty manifest + oldManifest = new Manifest(new Date(), ManifestConstants.SCHEMA_VERSION); // create empty manifest } else { this.logger.debug('Manifest does not exist yet. Waiting...'); return; From 2e59e15fc10bb7bf868efb00767890fff165335d Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Wed, 24 Jun 2020 14:02:07 -0400 Subject: [PATCH 073/106] Adding back in cache --- .../server/endpoint/lib/artifacts/cache.test.ts | 5 +++-- .../server/endpoint/lib/artifacts/cache.ts | 12 ++---------- .../server/endpoint/lib/artifacts/lists.ts | 4 +++- .../server/endpoint/lib/artifacts/task.ts | 2 -- .../routes/artifacts/download_exception_list.test.ts | 9 +++++---- .../routes/artifacts/download_exception_list.ts | 1 - .../server/endpoint/schemas/artifacts/lists.mock.ts | 4 ++-- .../artifacts/manifest_manager/manifest_manager.ts | 7 ++++++- x-pack/plugins/security_solution/server/plugin.ts | 4 ++-- 9 files changed, 23 insertions(+), 25 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts index 4149ff4a81623..f74e58e319457 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts @@ -10,7 +10,8 @@ describe('ExceptionsCache tests', () => { let cache: ExceptionsCache; beforeEach(() => { - cache = new ExceptionsCache(100); + jest.clearAllMocks(); + cache = new ExceptionsCache(); }); test('it should cache', async () => { @@ -33,7 +34,7 @@ describe('ExceptionsCache tests', () => { // Clean will remove all entries from the cache that have not been called by `get` since the last time it was cleaned cache.clean(); - // Need to call clean again to simulate a ttl period has gone by without `test` being requested + // Need to call clean again to simulate a ttl period has gone by without `test` being cache.clean(); const cacheRespCleaned = cache.get('test'); expect(cacheRespCleaned).toEqual(undefined); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts index fa0002b383a28..c0a8234eb5b40 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts @@ -4,22 +4,13 @@ * you may not use this file except in compliance with the Elastic License. */ -const DEFAULT_TTL = 1000; - export class ExceptionsCache { private cache: Map; private requested: string[]; - private ttl: number; - constructor(ttl: number) { + constructor() { this.cache = new Map(); this.requested = []; - this.ttl = ttl ? ttl : DEFAULT_TTL; - this.startClean(); - } - - private startClean() { - setInterval(() => this.clean, this.ttl); } clean() { @@ -32,6 +23,7 @@ export class ExceptionsCache { } set(id: string, body: string) { + this.clean(); this.cache.set(id, body); } diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts index 77505e00f8c58..89c8d870b1239 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts @@ -163,7 +163,9 @@ function translateEntry( /** * Compresses the exception list */ -function compressExceptionList(exceptionList: WrappedTranslatedExceptionList): Promise { +export function compressExceptionList( + exceptionList: WrappedTranslatedExceptionList +): Promise { return lzma.compress(JSON.stringify(exceptionList), (res: Buffer) => { return res; }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts index 0f74f0116662b..5ba0cf786d105 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts @@ -10,7 +10,6 @@ import { TaskManagerStartContract, } from '../../../../../../plugins/task_manager/server'; import { EndpointAppContext } from '../../types'; -import { ExceptionsCache } from './cache'; const PackagerTaskConstants = { TIMEOUT: '1m', @@ -29,7 +28,6 @@ interface PackagerTaskRunner { interface PackagerTaskContext { endpointAppContext: EndpointAppContext; taskManager: TaskManagerSetupContract; - cache: ExceptionsCache; } interface PackagerTaskRunnerContext { diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts index 4b5fbc809fa1a..60afd3328e056 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts @@ -22,7 +22,7 @@ import { loggingServiceMock, } from 'src/core/server/mocks'; import { ExceptionsCache } from '../../lib/artifacts/cache'; -import { CompressExceptionList } from '../../lib/artifacts/lists'; +import { compressExceptionList } from '../../lib/artifacts/lists'; import { ArtifactConstants } from '../../lib/artifacts'; import { registerDownloadExceptionListRoute } from './download_exception_list'; import { EndpointAppContextService } from '../../endpoint_app_context_services'; @@ -69,7 +69,8 @@ describe('test alerts route', () => { mockClusterClient.asScoped.mockReturnValue(mockScopedClient); routerMock = httpServiceMock.createRouter(); endpointAppContextService = new EndpointAppContextService(); - cache = new ExceptionsCache(10000); // TODO + cache = new ExceptionsCache(); + loggingServiceMock.createSetupContract(); endpointAppContextService.start({ agentService: createMockAgentService(), @@ -94,7 +95,7 @@ describe('test alerts route', () => { params: { sha256: '123456' }, }); - const mockCompressedArtifact = await CompressExceptionList(expectedEndpointExceptions); + const mockCompressedArtifact = await compressExceptionList(expectedEndpointExceptions); const mockArtifact = { id: '2468', @@ -183,7 +184,7 @@ describe('test alerts route', () => { }); // Add to the download cache - const mockCompressedArtifact = await CompressExceptionList(expectedEndpointExceptions); + const mockCompressedArtifact = await compressExceptionList(expectedEndpointExceptions); const cacheKey = `${mockArtifactName}-${mockSha}`; cache.set(cacheKey, mockCompressedArtifact.toString('binary')); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts index 86f64fd3baa9c..7a1bc8181932e 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts @@ -43,7 +43,6 @@ export function registerDownloadExceptionListRoute( // TODO: authenticate api key // https://github.com/elastic/kibana/issues/69329 - // PR: https://github.com/elastic/kibana/pull/69650 const buildAndValidateResponse = (artName: string, body: string): object => { const artifact = { diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.mock.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.mock.ts index 0147a53838381..7354b5fd0ec4d 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.mock.ts @@ -4,9 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ -import { FinalTranslatedExceptionList } from '../../lib/artifacts/lists'; +import { WrappedTranslatedExceptionList } from './lists'; -export const getTranslatedExceptionListMock = (): FinalTranslatedExceptionList => { +export const getTranslatedExceptionListMock = (): WrappedTranslatedExceptionList => { return { exceptions_list: [ { diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts index 0cd2ca501a2df..c8ae6c42bd571 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts @@ -12,6 +12,7 @@ import { Manifest, buildArtifact, getFullEndpointExceptionList, + ExceptionsCache, } from '../../../lib/artifacts'; import { InternalArtifactSchema, InternalManifestSchema } from '../../../schemas/artifacts'; import { ArtifactClient } from '../artifact_client'; @@ -22,6 +23,7 @@ export interface ManifestManagerContext { artifactClient: ArtifactClient; exceptionListClient: ExceptionListClient; logger: Logger; + cache: ExceptionsCache; } export class ManifestManager { @@ -29,15 +31,17 @@ export class ManifestManager { private exceptionListClient: ExceptionListClient; private savedObjectsClient: SavedObjectsClient; private logger: Logger; + private cache: ExceptionsCache; constructor(context: ManifestManagerContext) { this.artifactClient = context.artifactClient; this.exceptionListClient = context.exceptionListClient; this.savedObjectsClient = context.savedObjectsClient; this.logger = context.logger; + this.cache = context.cache; } - private async getManifestClient(schemaVersion: string): ManifestClient { + private async getManifestClient(schemaVersion: string): Promise { return new ManifestClient(this.savedObjectsClient, schemaVersion); } @@ -140,6 +144,7 @@ export class ManifestManager { const artifact = newManifest.getArtifact(diff.id); try { await this.artifactClient.createArtifact(artifact); + this.cache.set(`${artifact.identifier}-${artifact.sha256}`, artifact.body); } catch (err) { if (err.status === 409) { // This artifact already existed... diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index cf52b79f73737..63de317288a89 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -88,7 +88,7 @@ export class Plugin implements IPlugin Date: Wed, 24 Jun 2020 14:48:10 -0400 Subject: [PATCH 074/106] Fixing download test --- .../artifacts/download_exception_list.test.ts | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts index 4305631957a8c..201212383f090 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts @@ -19,7 +19,7 @@ import { savedObjectsClientMock, httpServiceMock, httpServerMock, - loggingServiceMock, + loggingSystemMock, } from 'src/core/server/mocks'; import { ExceptionsCache } from '../../lib/artifacts/cache'; import { compressExceptionList } from '../../lib/artifacts/lists'; @@ -28,20 +28,23 @@ import { registerDownloadExceptionListRoute } from './download_exception_list'; import { EndpointAppContextService } from '../../endpoint_app_context_services'; import { createMockAgentService } from '../../mocks'; import { createMockConfig } from '../../../lib/detection_engine/routes/__mocks__'; -import { getManifestManagerMock } from '../../services/artifacts'; +import { getManifestManagerMock } from '../../services/artifacts/manifest_manager/manifest_manager.mock'; +import { WrappedTranslatedExceptionList } from '../../schemas/artifacts/lists'; const mockArtifactName = `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-windows-1.0.0`; -const expectedEndpointExceptions = { +const expectedEndpointExceptions: WrappedTranslatedExceptionList = { exceptions_list: [ { entries: [ { - entry: { exact_caseless: 'Elastic, N.V.' }, + type: 'exact_caseless', + value: 'Elastic, N.V.', field: 'actingProcess.file.signer', operator: 'included', }, { - entry: { exact_caseless_any: ['process', 'malware'] }, + type: 'exact_caseless_any', + value: ['process', 'malware'], field: 'event.category', operator: 'included', }, @@ -71,17 +74,16 @@ describe('test alerts route', () => { routerMock = httpServiceMock.createRouter(); endpointAppContextService = new EndpointAppContextService(); cache = new ExceptionsCache(); - loggingServiceMock.createSetupContract(); endpointAppContextService.start({ agentService: createMockAgentService(), - manifestManager: getManifestManagerMock(), + manifestManager: undefined, }); registerDownloadExceptionListRoute( routerMock, { - logFactory: loggingServiceMock.create(), + logFactory: loggingSystemMock.create(), service: endpointAppContextService, config: () => Promise.resolve(createMockConfig()), }, From d23a7bb315240b65e54439d6e663bcb834ddaf7c Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Wed, 24 Jun 2020 15:09:38 -0400 Subject: [PATCH 075/106] Changing cache to be sized --- .../endpoint/lib/artifacts/cache.test.ts | 25 +++++++++--------- .../server/endpoint/lib/artifacts/cache.ts | 26 +++++++++---------- .../artifacts/download_exception_list.test.ts | 2 +- .../manifest_manager/manifest_manager.ts | 2 +- .../security_solution/server/plugin.ts | 2 +- 5 files changed, 29 insertions(+), 28 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts index f74e58e319457..d4006faced210 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts @@ -11,7 +11,7 @@ describe('ExceptionsCache tests', () => { beforeEach(() => { jest.clearAllMocks(); - cache = new ExceptionsCache(); + cache = new ExceptionsCache(3); }); test('it should cache', async () => { @@ -27,16 +27,17 @@ describe('ExceptionsCache tests', () => { }); test('it should handle cache clean', async () => { - cache.set('test', 'body'); - const cacheResp = cache.get('test'); - expect(cacheResp).toEqual('body'); - - // Clean will remove all entries from the cache that have not been called by `get` since the last time it was cleaned - cache.clean(); - - // Need to call clean again to simulate a ttl period has gone by without `test` being - cache.clean(); - const cacheRespCleaned = cache.get('test'); - expect(cacheRespCleaned).toEqual(undefined); + cache.set('1', 'a'); + cache.set('2', 'b'); + cache.set('3', 'c'); + const cacheResp = cache.get('1'); + expect(cacheResp).toEqual('a'); + + cache.set('4', 'd'); + const secondResp = cache.get('1'); + expect(secondResp).toEqual(undefined); + expect(cache.get('2')).toEqual('b'); + expect(cache.get('3')).toEqual('c'); + expect(cache.get('4')).toEqual('d'); }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts index c0a8234eb5b40..6e0c8f73531ce 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts @@ -4,31 +4,31 @@ * you may not use this file except in compliance with the Elastic License. */ +const DEFAULT_MAX_SIZE = 10; + export class ExceptionsCache { private cache: Map; - private requested: string[]; + private queue: string[]; + private maxSize: number; - constructor() { + constructor(maxSize: number) { this.cache = new Map(); - this.requested = []; + this.queue = []; + this.maxSize = maxSize || DEFAULT_MAX_SIZE; } - clean() { - for (const v of this.cache.keys()) { - if (!this.requested.includes(v)) { - this.cache.delete(v); + set(id: string, body: string) { + if (this.queue.length + 1 > this.maxSize) { + const entry = this.queue.shift(); + if (entry !== undefined) { + this.cache.delete(entry); } } - this.requested = []; - } - - set(id: string, body: string) { - this.clean(); + this.queue.push(id); this.cache.set(id, body); } get(id: string): string | undefined { - this.requested.push(id); return this.cache.get(id); } } diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts index 201212383f090..5355c444fad59 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts @@ -73,7 +73,7 @@ describe('test alerts route', () => { mockClusterClient.asScoped.mockReturnValue(mockScopedClient); routerMock = httpServiceMock.createRouter(); endpointAppContextService = new EndpointAppContextService(); - cache = new ExceptionsCache(); + cache = new ExceptionsCache(5); endpointAppContextService.start({ agentService: createMockAgentService(), diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts index 9bebed90920df..88903e7f117c9 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts @@ -41,7 +41,7 @@ export class ManifestManager { this.cache = context.cache; } - private async getManifestClient(schemaVersion: string): Promise { + private getManifestClient(schemaVersion: string): ManifestClient { return new ManifestClient(this.savedObjectsClient, schemaVersion); } diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 63de317288a89..ecaccb6f404bb 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -88,7 +88,7 @@ export class Plugin implements IPlugin Date: Wed, 24 Jun 2020 19:57:54 -0400 Subject: [PATCH 076/106] Fix manifest manager initialization --- .../server/endpoint/endpoint_app_context_services.ts | 1 + x-pack/plugins/security_solution/server/plugin.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts index 6babb3e713742..f01f8c2e42d70 100644 --- a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts +++ b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts @@ -23,6 +23,7 @@ export class EndpointAppContextService { public start(dependencies: EndpointAppContextServiceStartContract) { this.agentService = dependencies.agentService; + this.manifestManager = dependencies.manifestManager; dependencies.registerIngestCallback('datasourceCreate', handleDatasourceCreate); } diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 62bea3e6c5f3e..bc6da8f06492e 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -258,6 +258,7 @@ export class Plugin implements IPlugin Date: Wed, 24 Jun 2020 23:38:38 -0400 Subject: [PATCH 077/106] Hook up datasource service --- .../endpoint/endpoint_app_context_services.ts | 7 +- .../server/endpoint/ingest_integration.ts | 79 +++++++----- .../server/endpoint/lib/artifacts/task.ts | 9 +- .../manifest_manager/manifest_manager.ts | 116 +++++++++++------- .../security_solution/server/plugin.ts | 11 ++ 5 files changed, 143 insertions(+), 79 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts index f01f8c2e42d70..7a72585e6e2e6 100644 --- a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts +++ b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ import { AgentService, IngestManagerStartContract } from '../../../ingest_manager/server'; -import { handleDatasourceCreate } from './ingest_integration'; +import { getDatasourceCreateCallback } from './ingest_integration'; export type EndpointAppContextServiceStartContract = Pick< IngestManagerStartContract, @@ -24,7 +24,10 @@ export class EndpointAppContextService { public start(dependencies: EndpointAppContextServiceStartContract) { this.agentService = dependencies.agentService; this.manifestManager = dependencies.manifestManager; - dependencies.registerIngestCallback('datasourceCreate', handleDatasourceCreate); + dependencies.registerIngestCallback( + 'datasourceCreate', + getDatasourceCreateCallback(this.manifestManager) + ); } public stop() {} diff --git a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts index 6ff0949311587..93a3f19d6aca4 100644 --- a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts +++ b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts @@ -7,43 +7,58 @@ import { factory as policyConfigFactory } from '../../common/endpoint/models/policy_config'; import { NewPolicyData } from '../../common/endpoint/types'; import { NewDatasource } from '../../../ingest_manager/common/types/models'; +import { ManifestManager } from './services/artifacts'; /** * Callback to handle creation of Datasources in Ingest Manager - * @param newDatasource */ -export const handleDatasourceCreate = async ( - newDatasource: NewDatasource -): Promise => { - // We only care about Endpoint datasources - if (newDatasource.package?.name !== 'endpoint') { - return newDatasource; - } - - // We cast the type here so that any changes to the Endpoint specific data - // follow the types/schema expected - let updatedDatasource = newDatasource as NewPolicyData; - - // Until we get the Default Policy Configuration in the Endpoint package, - // we will add it here manually at creation time. - // @ts-ignore - if (newDatasource.inputs.length === 0) { - updatedDatasource = { - ...newDatasource, - inputs: [ - { - type: 'endpoint', - enabled: true, - streams: [], - config: { - policy: { - value: policyConfigFactory(), +export const getDatasourceCreateCallback = ( + manifestManager: ManifestManager +): ((newDatasource: NewDatasource) => Promise) => { + const handleDatasourceCreate = async (newDatasource: NewDatasource): Promise => { + // We only care about Endpoint datasources + if (newDatasource.package?.name !== 'endpoint') { + return newDatasource; + } + + const manifestState = await manifestManager.refresh({ initialize: true }); + + // We cast the type here so that any changes to the Endpoint specific data + // follow the types/schema expected + let updatedDatasource = newDatasource as NewPolicyData; + // updatedDatasource['artifact_manifest'] = manifestManager. + // console.log(JSON.stringify(updatedDatasource.inputs[0].config)); + + // Until we get the Default Policy Configuration in the Endpoint package, + // we will add it here manually at creation time. + // @ts-ignore + if (newDatasource.inputs.length === 0) { + updatedDatasource = { + ...newDatasource, + artifact_manifest: manifestState.manifest.toEndpointFormat(), + inputs: [ + { + type: 'endpoint', + enabled: true, + streams: [], + config: { + policy: { + value: policyConfigFactory(), + }, }, }, - }, - ], - }; - } + ], + }; + } + + // console.log(updatedDatasource); + + try { + return updatedDatasource; + } finally { + await manifestManager.commit(manifestState); + } + }; - return updatedDatasource; + return handleDatasourceCreate; }; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts index 5ba0cf786d105..c450dbb6becfa 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts @@ -59,11 +59,14 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { } try { - // TODO: change this to 'false' when we hook up the ingestManager callback - await manifestManager.refresh(true); + const manifestState = await manifestManager.refresh(); + if (manifestState != null) { + if (await manifestManager.dispatch(manifestState)) { + await manifestManager.commit(manifestState); + } + } } catch (err) { logger.error(err); - logger.debug('Manifest not created yet, nothing to do.'); } }; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts index c140c0e3cd35d..abbd6c1d5c187 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts @@ -5,6 +5,7 @@ */ import { Logger, SavedObjectsClient } from '../../../../../../../../src/core/server'; +import { DatasourceServiceInterface } from '../../../../../../ingest_manager/server'; import { ExceptionListClient } from '../../../../../../lists/server'; import { ArtifactConstants, @@ -22,13 +23,24 @@ export interface ManifestManagerContext { savedObjectsClient: SavedObjectsClient; artifactClient: ArtifactClient; exceptionListClient: ExceptionListClient; + datasourceService: DatasourceServiceInterface; logger: Logger; cache: ExceptionsCache; } +export interface ManifestRefreshOpts { + initialize?: boolean; +} + +export interface NewManifestState { + manifest: Manifest; + diffs: ManifestDiff[]; +} + export class ManifestManager { private artifactClient: ArtifactClient; private exceptionListClient: ExceptionListClient; + private datasourceService: DatasourceServiceInterface; private savedObjectsClient: SavedObjectsClient; private logger: Logger; private cache: ExceptionsCache; @@ -45,20 +57,6 @@ export class ManifestManager { return new ManifestClient(this.savedObjectsClient, schemaVersion); } - private async dispatch(manifest: Manifest) { - const manifestClient = this.getManifestClient(manifest.getSchemaVersion()); - - // TODO: dispatch and only update if successful - - if (manifest.getVersion() === undefined) { - await manifestClient.createManifest(manifest.toSavedObject()); - } else { - await manifestClient.updateManifest(manifest.toSavedObject(), { - version: manifest.getVersion(), - }); - } - } - private async buildExceptionListArtifacts( schemaVersion: string ): Promise { @@ -105,29 +103,19 @@ export class ManifestManager { } } - public async refresh(createInitial: boolean) { + public async refresh(opts: ManifestRefreshOpts): Promise { let oldManifest: Manifest; // Get the last-dispatched manifest - try { - oldManifest = await this.getLastDispatchedManifest(ManifestConstants.SCHEMA_VERSION); - } catch (err) { - this.logger.error(err); - return; - } + oldManifest = await this.getLastDispatchedManifest(ManifestConstants.SCHEMA_VERSION); - if (oldManifest === null) { - if (createInitial) { - // TODO: implement this when ready to integrate with Paul's code - oldManifest = new Manifest(new Date(), ManifestConstants.SCHEMA_VERSION); // create empty manifest - } else { - this.logger.debug('Manifest does not exist yet. Waiting...'); - return; - } + if (oldManifest === null && opts.initialize) { + oldManifest = new Manifest(new Date(), ManifestConstants.SCHEMA_VERSION); // create empty manifest + } else if (oldManifest == null) { + this.logger.debug('Manifest does not exist yet. Waiting...'); + return null; } - this.logger.debug(oldManifest); - // Build new exception list artifacts const artifacts = await this.buildExceptionListArtifacts(ArtifactConstants.SCHEMA_VERSION); @@ -145,7 +133,7 @@ export class ManifestManager { try { await this.artifactClient.createArtifact(artifact); // Cache the compressed body of the artifact - this.cache.set(`${artifact.identifier}-${artifact.sha256}`, artifact.body); + this.cache.set(diff.id, artifact.body); } catch (err) { if (err.status === 409) { // This artifact already existed... @@ -157,19 +145,63 @@ export class ManifestManager { } }, this); - // Dispatch manifest if new - if (diffs.length > 0) { - try { - this.logger.debug('Dispatching new manifest'); - await this.dispatch(newManifest); - } catch (err) { - this.logger.error(err); - return; - } + return { + manifest: newManifest, + diffs, + }; + } + + /** + * Dispatches the manifest by writing it to the endpoint datasource. + * + * @return {boolean} True if dispatched. + */ + public async dispatch(manifestState: NewManifestState): boolean { + if (manifestState.diffs.length > 0) { + this.logger.info(`Dispatching new manifest with diffs: ${manifestState.diffs}`); + // + // Datasource: + // + // { name: 'endpoint-1', + // description: '', + // config_id: 'c7fe80d0-b677-11ea-8bb2-09d7226f2862', + // enabled: true, + // output_id: '', + // inputs: + // [ { type: 'endpoint', enabled: true, streams: [], config: [Object] } ], + // namespace: 'default', + // package: + // { name: 'endpoint', title: 'Elastic Endpoint', version: '0.5.0' } } + // + // TODO: write to config + // await this.datasourceService.get('the-datasource-id'); + // OR + // await this.datasourceService.list('the-datasource-by-config_id'); + // THEN + // await this.datasourceService.update(...args); + // + return true; + } else { + this.logger.debug('No manifest diffs [no-op]'); + } + + return false; + } + + public async commit(manifestState: NewManifestState) { + const manifestClient = this.getManifestClient(manifestState.manifest.getSchemaVersion()); + + // Commit the new manifest + if (manifestState.manifest.getVersion() === undefined) { + await manifestClient.createManifest(manifestState.manifest.toSavedObject()); + } else { + await manifestClient.updateManifest(manifestState.manifest.toSavedObject(), { + version: manifestState.manifest.getVersion(), + }); } // Clean up old artifacts - diffs.forEach(async (diff) => { + manifestState.diffs.forEach(async (diff) => { try { if (diff.type === 'delete') { await this.artifactClient.deleteArtifact(diff.id); diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index bc6da8f06492e..2e8b92f309021 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -251,6 +251,7 @@ export class Plugin implements IPlugin Date: Thu, 25 Jun 2020 11:58:48 -0400 Subject: [PATCH 078/106] Fix download tests --- x-pack/plugins/lists/server/mocks.ts | 2 +- .../endpoint/endpoint_app_context_services.ts | 2 ++ .../server/endpoint/mocks.ts | 31 ++----------------- .../artifacts/download_exception_list.test.ts | 11 +++---- .../artifacts/artifact_client.mock.ts | 6 ++++ .../manifest_manager/manifest_manager.mock.ts | 26 ++++++++-------- 6 files changed, 29 insertions(+), 49 deletions(-) diff --git a/x-pack/plugins/lists/server/mocks.ts b/x-pack/plugins/lists/server/mocks.ts index aad4a25a900a1..ba565216fe431 100644 --- a/x-pack/plugins/lists/server/mocks.ts +++ b/x-pack/plugins/lists/server/mocks.ts @@ -18,6 +18,6 @@ const createSetupMock = (): jest.Mocked => { export const listMock = { createSetup: createSetupMock, - getExceptionList: getExceptionListClientMock, + getExceptionListClient: getExceptionListClientMock, getListClient: getListClientMock, }; diff --git a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts index 7a72585e6e2e6..e23d9981d21ce 100644 --- a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts +++ b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts @@ -5,11 +5,13 @@ */ import { AgentService, IngestManagerStartContract } from '../../../ingest_manager/server'; import { getDatasourceCreateCallback } from './ingest_integration'; +import { ManifestManager } from './services/artifacts'; export type EndpointAppContextServiceStartContract = Pick< IngestManagerStartContract, 'agentService' > & { + manifestManager: ManifestManager; registerIngestCallback: IngestManagerStartContract['registerExternalCallback']; }; diff --git a/x-pack/plugins/security_solution/server/endpoint/mocks.ts b/x-pack/plugins/security_solution/server/endpoint/mocks.ts index f32060081d20e..3ecc0f25d265d 100644 --- a/x-pack/plugins/security_solution/server/endpoint/mocks.ts +++ b/x-pack/plugins/security_solution/server/endpoint/mocks.ts @@ -14,6 +14,7 @@ import { } from '../../../ingest_manager/server'; import { EndpointAppContextServiceStartContract } from './endpoint_app_context_services'; import { createDatasourceServiceMock } from '../../../ingest_manager/server/mocks'; +import { getManifestManagerMock } from './services/artifacts/manifest_manager/manifest_manager.mock'; /** * Crates a mocked input contract for the `EndpointAppContextService#start()` method @@ -23,6 +24,7 @@ export const createMockEndpointAppContextServiceStartContract = (): jest.Mocked< > => { return { agentService: createMockAgentService(), + manifestManager: getManifestManagerMock(), registerIngestCallback: jest.fn< ReturnType, Parameters @@ -43,35 +45,6 @@ export const createMockAgentService = (): jest.Mocked => { }; }; -/** - * Creates a mock ArtifactClient - */ -export const createMockArtifactClient = (): jest.Mocked => { - return { - getArtifact: 'TODO', - createArtifact: 'TODO', - deleteArtifact: 'TODO', - }; -}; - -/** - * Creates a mock ManifestManager - */ -export const createMockManifestManager = (): jest.Mocked => { - return { - refresh: 'TODO', - }; -}; - -/** - * Creates a mock ExceptionListClient - */ -export const createMockExceptionListClient = (): jest.Mocked => { - return { - findExceptionListItem: 'TODO', - }; -}; - /** * Creates a mock IndexPatternService for use in tests that need to interact with the Ingest Manager's * ESIndexPatternService. diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts index e902a3eb28a5a..baaa8cb387d02 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts @@ -26,7 +26,10 @@ import { compressExceptionList } from '../../lib/artifacts/lists'; import { ArtifactConstants } from '../../lib/artifacts'; import { registerDownloadExceptionListRoute } from './download_exception_list'; import { EndpointAppContextService } from '../../endpoint_app_context_services'; -import { createMockAgentService } from '../../mocks'; +import { + createMockAgentService, + createMockEndpointAppContextServiceStartContract, +} from '../../mocks'; import { createMockConfig } from '../../../lib/detection_engine/routes/__mocks__'; import { WrappedTranslatedExceptionList } from '../../schemas/artifacts/lists'; @@ -73,11 +76,7 @@ describe('test alerts route', () => { routerMock = httpServiceMock.createRouter(); endpointAppContextService = new EndpointAppContextService(); cache = new ExceptionsCache(5); - - endpointAppContextService.start({ - agentService: createMockAgentService(), - manifestManager: undefined, - }); + endpointAppContextService.start(createMockEndpointAppContextServiceStartContract()); registerDownloadExceptionListRoute( routerMock, diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.mock.ts index 2c676dc2ae5bd..442356afe6354 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.mock.ts @@ -4,6 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ +import { savedObjectsClientMock } from '../../../../../../../src/core/server/mocks'; + import { ArtifactClient } from './artifact_client'; import { getInternalArtifactMock } from '../../schemas'; @@ -12,3 +14,7 @@ export class ArtifactClientMock extends ArtifactClient { public createArtifact = jest.fn().mockResolvedValue(getInternalArtifactMock()); public deleteArtifact = jest.fn().mockResolvedValue(getInternalArtifactMock()); } + +export const getArtifactClientMock = (): ArtifactClientMock => { + return new ArtifactClientMock(savedObjectsClientMock.create()); +}; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts index ba241bf65946f..cf5f9a1ec4d4f 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts @@ -4,22 +4,22 @@ * you may not use this file except in compliance with the Elastic License. */ -// TODO -// eslint-disable-next-line @kbn/eslint/no-restricted-paths -import { loggerMock } from '../../../../../../../../src/core/server/logging/logger.mock'; +import { Logger } from '../../../../../../../../src/core/server'; +import { + loggingSystemMock, + savedObjectsClientMock, +} from '../../../../../../../../src/core/server/mocks'; -import { savedObjectsClientMock } from '../../../../../../../../src/core/server/mocks'; -import { Manifest } from '../../../lib/artifacts'; -import { ManifestManager } from './manifest_manager'; - -// TODO -// eslint-disable-next-line @kbn/eslint/no-restricted-paths -import { getExceptionListClientMock } from '../../../../../../lists/server/services/exception_lists/exception_list_client.mock'; +import { listMock } from '../../../../../../lists/server/mocks'; +import { Manifest } from '../../../lib/artifacts'; import { getInternalArtifactMock, getInternalArtifactsMock } from '../../../schemas'; + import { getArtifactClientMock } from '../artifact_client.mock'; import { getManifestClientMock } from '../manifest_client.mock'; +import { ManifestManager } from './manifest_manager'; + function mockBuildExceptionListArtifacts() { // mock buildArtifactFunction // pass in OS, mock implementation of ExceptionListItemSchemaMock more than once @@ -33,14 +33,14 @@ export class ManifestManagerMock extends ManifestManager { private getLastDispatchedManifest = jest .fn() .mockResolvedValue(new Manifest(new Date(), '1.0.0')); - private getManifestClient = jest.fn().mockValue(getManifestClientMock()); + private getManifestClient = jest.fn().mockReturnValue(getManifestClientMock()); } export const getManifestManagerMock = (): ManifestManagerMock => { return new ManifestManagerMock({ artifactClient: getArtifactClientMock(), - exceptionListClient: getExceptionListClientMock(), + exceptionListClient: listMock.getExceptionListClient(), savedObjectsClient: savedObjectsClientMock.create(), - logger: loggerMock.create(), + logger: loggingSystemMock.create().get() as jest.Mocked, }); }; From e7b229d14fd66227f13305cfb3187f300b5539db Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Thu, 25 Jun 2020 16:12:26 -0400 Subject: [PATCH 079/106] Incremental progress --- .../common/endpoint/schema/common.ts | 22 +++++++++ .../endpoint/schema/manifest.ts} | 0 .../common/endpoint/types.ts | 2 + .../server/endpoint/ingest_integration.ts | 45 +++++++++---------- .../endpoint/lib/artifacts/lists.test.ts | 2 +- .../endpoint/lib/artifacts/manifest.test.ts | 4 +- .../server/endpoint/lib/artifacts/manifest.ts | 10 ++--- .../lib/artifacts/manifest_entry.test.ts | 3 +- .../endpoint/lib/artifacts/manifest_entry.ts | 3 +- .../server/endpoint/lib/artifacts/task.ts | 2 +- .../server/endpoint/mocks.ts | 4 -- .../artifacts/download_exception_list.test.ts | 5 +-- .../endpoint/routes/metadata/metadata.test.ts | 1 - .../endpoint/routes/policy/handlers.test.ts | 1 - .../endpoint/schemas/artifacts/common.ts | 16 ------- .../endpoint/schemas/artifacts/index.ts | 3 -- .../schemas/artifacts/manifest_schema.mock.ts | 5 --- .../request/download_artifact_schema.ts | 2 +- .../schemas/artifacts/saved_objects.ts | 3 +- .../artifacts/artifact_client.mock.ts | 2 +- .../artifacts/manifest_client.mock.ts | 2 +- .../manifest_manager/manifest_manager.mock.ts | 37 +++++++++++++++ .../manifest_manager/manifest_manager.ts | 30 +++++++++++-- 23 files changed, 127 insertions(+), 77 deletions(-) create mode 100644 x-pack/plugins/security_solution/common/endpoint/schema/common.ts rename x-pack/plugins/security_solution/{server/endpoint/schemas/artifacts/manifest_schema.ts => common/endpoint/schema/manifest.ts} (100%) delete mode 100644 x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/manifest_schema.mock.ts diff --git a/x-pack/plugins/security_solution/common/endpoint/schema/common.ts b/x-pack/plugins/security_solution/common/endpoint/schema/common.ts new file mode 100644 index 0000000000000..7f8c938d54feb --- /dev/null +++ b/x-pack/plugins/security_solution/common/endpoint/schema/common.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; + +export const identifier = t.string; + +export const manifestVersion = t.string; + +export const manifestSchemaVersion = t.keyof({ + '1.0.0': null, +}); +export type ManifestSchemaVersion = t.TypeOf; + +export const sha256 = t.string; + +export const size = t.number; + +export const url = t.string; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/manifest_schema.ts b/x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts similarity index 100% rename from x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/manifest_schema.ts rename to x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts diff --git a/x-pack/plugins/security_solution/common/endpoint/types.ts b/x-pack/plugins/security_solution/common/endpoint/types.ts index f8cfb8f7c3bbc..6aecf012576c1 100644 --- a/x-pack/plugins/security_solution/common/endpoint/types.ts +++ b/x-pack/plugins/security_solution/common/endpoint/types.ts @@ -5,6 +5,7 @@ */ import { Datasource, NewDatasource } from '../../../ingest_manager/common'; +import { ManifestSchema } from './schema/manifest'; /** * Object that allows you to maintain stateful information in the location object across navigation events @@ -658,6 +659,7 @@ export type NewPolicyData = NewDatasource & { enabled: boolean; streams: []; config: { + artifact_manifest: ManifestSchema; policy: { value: PolicyConfig; }; diff --git a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts index 93a3f19d6aca4..928bea23d8ff7 100644 --- a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts +++ b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts @@ -21,38 +21,35 @@ export const getDatasourceCreateCallback = ( return newDatasource; } - const manifestState = await manifestManager.refresh({ initialize: true }); - // We cast the type here so that any changes to the Endpoint specific data // follow the types/schema expected let updatedDatasource = newDatasource as NewPolicyData; - // updatedDatasource['artifact_manifest'] = manifestManager. - // console.log(JSON.stringify(updatedDatasource.inputs[0].config)); - // Until we get the Default Policy Configuration in the Endpoint package, - // we will add it here manually at creation time. - // @ts-ignore - if (newDatasource.inputs.length === 0) { - updatedDatasource = { - ...newDatasource, - artifact_manifest: manifestState.manifest.toEndpointFormat(), - inputs: [ - { - type: 'endpoint', - enabled: true, - streams: [], - config: { - policy: { - value: policyConfigFactory(), + const manifestState = await manifestManager.refresh({ initialize: true }); + if (manifestState !== null) { + // Until we get the Default Policy Configuration in the Endpoint package, + // we will add it here manually at creation time. + // @ts-ignore + if (newDatasource.inputs.length === 0) { + updatedDatasource = { + ...newDatasource, + inputs: [ + { + type: 'endpoint', + enabled: true, + streams: [], + config: { + artifact_manifest: manifestState.manifest.toEndpointFormat(), + policy: { + value: policyConfigFactory(), + }, }, }, - }, - ], - }; + ], + }; + } } - // console.log(updatedDatasource); - try { return updatedDatasource; } finally { diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts index d29a6ad53bdcd..1804f1c012537 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts @@ -16,7 +16,7 @@ describe('buildEventTypeSignal', () => { beforeEach(() => { jest.clearAllMocks(); - mockExceptionClient = listMock.getExceptionList(); + mockExceptionClient = listMock.getExceptionListClient(); }); test('it should convert the exception lists response to the proper endpoint format', async () => { diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts index 8c94f9e94c80f..b03a3e017dee1 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts @@ -4,11 +4,11 @@ * you may not use this file except in compliance with the Elastic License. */ +import { InternalArtifactSchema } from '../../schemas'; import { - InternalArtifactSchema, getInternalArtifactMock, getInternalArtifactMockWithDiffs, -} from '../../schemas'; +} from '../../schemas/artifacts/saved_objects.mock'; import { Manifest } from './manifest'; describe('manifest', () => { diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts index 7a4f2be24251e..a3189726e4c17 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts @@ -5,14 +5,12 @@ */ import { validate } from '../../../../common/validate'; +import { InternalArtifactSchema, InternalManifestSchema } from '../../schemas/artifacts'; import { - InternalArtifactSchema, - InternalManifestSchema, - ManifestSchema, - ManifestSchemaVersion, - manifestSchema, manifestSchemaVersion, -} from '../../schemas/artifacts'; + ManifestSchemaVersion, +} from '../../../../common/endpoint/schema/common'; +import { ManifestSchema, manifestSchema } from '../../../../common/endpoint/schema/manifest'; import { ManifestEntry } from './manifest_entry'; export interface ManifestDiff { diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts index dd28fb5f405df..9d76332df9169 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts @@ -4,7 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ -import { InternalArtifactSchema, getInternalArtifactMock } from '../../schemas'; +import { InternalArtifactSchema } from '../../schemas'; +import { getInternalArtifactMock } from '../../schemas/artifacts/saved_objects.mock'; import { ManifestEntry } from './manifest_entry'; describe('manifest_entry', () => { diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts index d90ae54d89f83..f330054d15453 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts @@ -4,7 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ -import { InternalArtifactSchema, ManifestEntrySchema } from '../../schemas/artifacts'; +import { InternalArtifactSchema } from '../../schemas/artifacts'; +import { ManifestEntrySchema } from '../../../../common/endpoint/schema/manifest'; export class ManifestEntry { private artifact: InternalArtifactSchema; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts index c450dbb6becfa..95e60f643eff7 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts @@ -60,7 +60,7 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { try { const manifestState = await manifestManager.refresh(); - if (manifestState != null) { + if (manifestState !== null) { if (await manifestManager.dispatch(manifestState)) { await manifestManager.commit(manifestState); } diff --git a/x-pack/plugins/security_solution/server/endpoint/mocks.ts b/x-pack/plugins/security_solution/server/endpoint/mocks.ts index 3ecc0f25d265d..de700058ea4be 100644 --- a/x-pack/plugins/security_solution/server/endpoint/mocks.ts +++ b/x-pack/plugins/security_solution/server/endpoint/mocks.ts @@ -32,10 +32,6 @@ export const createMockEndpointAppContextServiceStartContract = (): jest.Mocked< }; }; -import { ExceptionListClient } from '../../../lists/server'; - -import { ArtifactClient, ManifestManager } from './services'; - /** * Creates a mock AgentService */ diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts index baaa8cb387d02..7a06551fe5a1f 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts @@ -26,10 +26,7 @@ import { compressExceptionList } from '../../lib/artifacts/lists'; import { ArtifactConstants } from '../../lib/artifacts'; import { registerDownloadExceptionListRoute } from './download_exception_list'; import { EndpointAppContextService } from '../../endpoint_app_context_services'; -import { - createMockAgentService, - createMockEndpointAppContextServiceStartContract, -} from '../../mocks'; +import { createMockEndpointAppContextServiceStartContract } from '../../mocks'; import { createMockConfig } from '../../../lib/detection_engine/routes/__mocks__'; import { WrappedTranslatedExceptionList } from '../../schemas/artifacts/lists'; diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata.test.ts index b64bc069ce8d3..c04975fa8b28e 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata.test.ts @@ -33,7 +33,6 @@ import { } from '../../mocks'; import Boom from 'boom'; import { EndpointAppContextService } from '../../endpoint_app_context_services'; -import { getManifestManagerMock } from '../../services/artifacts/manifest_manager/manifest_manager.mock'; import { createMockConfig } from '../../../lib/detection_engine/routes/__mocks__'; import { EndpointDocGenerator } from '../../../../common/endpoint/generate_data'; diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/policy/handlers.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/policy/handlers.test.ts index d847a26b33518..16af3a95bc72d 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/policy/handlers.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/policy/handlers.test.ts @@ -21,7 +21,6 @@ import { savedObjectsClientMock, } from '../../../../../../../src/core/server/mocks'; import { SearchResponse } from 'elasticsearch'; -import { getManifestManagerMock } from '../../services/artifacts/manifest_manager/manifest_manager.mock'; import { GetHostPolicyResponse, HostPolicyResponse } from '../../../../common/endpoint/types'; import { EndpointDocGenerator } from '../../../../common/endpoint/generate_data'; import { createMockConfig } from '../../../lib/detection_engine/routes/__mocks__'; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/common.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/common.ts index 3b437a5ee0de8..b6a6f7b4af9dc 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/common.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/common.ts @@ -6,8 +6,6 @@ import * as t from 'io-ts'; -export const identifier = t.string; - export const body = t.string; export const created = t.number; // TODO: Make this into an ISO Date string check @@ -16,20 +14,6 @@ export const encoding = t.keyof({ xz: null, }); -export const manifestVersion = t.string; - -export const manifestSchemaVersion = t.keyof({ - '1.0.0': null, -}); - export const schemaVersion = t.keyof({ '1.0.0': null, }); - -export const sha256 = t.string; - -export const size = t.number; - -export const url = t.string; - -export type ManifestSchemaVersion = t.TypeOf; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/index.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/index.ts index 213152ff33b36..908fbb698adef 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/index.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/index.ts @@ -6,9 +6,6 @@ export * from './common'; export * from './lists'; -export * from './lists.mock'; -export * from './manifest_schema'; export * from './request'; export * from './response'; export * from './saved_objects'; -export * from './saved_objects.mock'; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/manifest_schema.mock.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/manifest_schema.mock.ts deleted file mode 100644 index 41bc2aa258807..0000000000000 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/manifest_schema.mock.ts +++ /dev/null @@ -1,5 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/request/download_artifact_schema.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/request/download_artifact_schema.ts index 6a6d15aae9759..7a194fdc7b5f4 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/request/download_artifact_schema.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/request/download_artifact_schema.ts @@ -5,7 +5,7 @@ */ import * as t from 'io-ts'; -import { identifier, sha256 } from '../common'; +import { identifier, sha256 } from '../../../../../common/endpoint/schema/common'; export const downloadArtifactRequestParamsSchema = t.exact( t.type({ diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts index 3473dfc952e45..2e71ef98387f1 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts @@ -5,7 +5,8 @@ */ import * as t from 'io-ts'; -import { body, created, encoding, identifier, sha256, size } from './common'; +import { identifier, sha256, size } from '../../../../common/endpoint/schema/common'; +import { body, created, encoding } from './common'; export const internalArtifactSchema = t.exact( t.type({ diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.mock.ts index 442356afe6354..d7a7f38473a04 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.mock.ts @@ -7,7 +7,7 @@ import { savedObjectsClientMock } from '../../../../../../../src/core/server/mocks'; import { ArtifactClient } from './artifact_client'; -import { getInternalArtifactMock } from '../../schemas'; +import { getInternalArtifactMock } from '../../schemas/artifacts/saved_objects.mock'; export class ArtifactClientMock extends ArtifactClient { public getArtifact = jest.fn().mockResolvedValue(getInternalArtifactMock()); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.mock.ts index 7ed580721ff10..61445cb407de2 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.mock.ts @@ -7,7 +7,7 @@ import { savedObjectsClientMock } from '../../../../../../../src/core/server/mocks'; import { ManifestClient } from './manifest_client'; -import { getInternalManifestMock } from '../../schemas'; +import { getInternalManifestMock } from '../../schemas/artifacts/saved_objects.mock'; export class ManifestClientMock extends ManifestClient { public createManifest = jest.fn().mockResolvedValue(getInternalManifestMock()); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts index cf5f9a1ec4d4f..86124ae88c897 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts @@ -4,12 +4,14 @@ * you may not use this file except in compliance with the Elastic License. */ +// eslint-disable-next-line max-classes-per-file import { Logger } from '../../../../../../../../src/core/server'; import { loggingSystemMock, savedObjectsClientMock, } from '../../../../../../../../src/core/server/mocks'; +import { DatasourceServiceInterface } from '../../../../../../ingest_manager/server'; import { listMock } from '../../../../../../lists/server/mocks'; import { Manifest } from '../../../lib/artifacts'; @@ -20,12 +22,46 @@ import { getManifestClientMock } from '../manifest_client.mock'; import { ManifestManager } from './manifest_manager'; +function getMockDatasource() { + return { + id: 'c6d16e42-c32d-4dce-8a88-113cfe276ad1', + inputs: [{}], + revision: 1, + version: 'abcd', // TODO: not yet implemented in ingest_manager (https://github.com/elastic/kibana/issues/69992) + updated_at: '2020-06-25T16:03:38.159292', + updated_by: 'kibana', + created_at: '2020-06-25T16:03:38.159292', + created_by: 'kibana', + }; +} + +// TODO +// eslint-disable-next-line max-classes-per-file +class DatasourceServiceMock extends DatasourceServiceInterface { + public create = jest.fn().mockResolvedValue(getMockDatasource()); + public get = jest.fn().mockResolvedValue(getMockDatasource()); + public getByIds = jest.fn().mockResolvedValue([getMockDatasource()]); + public list = jest.fn().mockResolvedValue({ + items: [getMockDatasource()], + total: 1, + page: 1, + perPage: 20, + }); + public update = jest.fn().mockResolvedValue(getMockDatasource()); +} + +function getDatasourceServiceMock() { + return new DatasourceServiceMock(); +} + function mockBuildExceptionListArtifacts() { // mock buildArtifactFunction // pass in OS, mock implementation of ExceptionListItemSchemaMock more than once // getInternalArtifactsMock() } +// TODO +// eslint-disable-next-line max-classes-per-file export class ManifestManagerMock extends ManifestManager { private buildExceptionListArtifacts = jest .fn() @@ -40,6 +76,7 @@ export const getManifestManagerMock = (): ManifestManagerMock => { return new ManifestManagerMock({ artifactClient: getArtifactClientMock(), exceptionListClient: listMock.getExceptionListClient(), + datasourceService: getDatasourceServiceMock(), savedObjectsClient: savedObjectsClientMock.create(), logger: loggingSystemMock.create().get() as jest.Mocked, }); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts index abbd6c1d5c187..b5c3c8ff0f445 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts @@ -48,6 +48,7 @@ export class ManifestManager { constructor(context: ManifestManagerContext) { this.artifactClient = context.artifactClient; this.exceptionListClient = context.exceptionListClient; + this.datasourceService = context.datasourceService; this.savedObjectsClient = context.savedObjectsClient; this.logger = context.logger; this.cache = context.cache; @@ -103,9 +104,18 @@ export class ManifestManager { } } - public async refresh(opts: ManifestRefreshOpts): Promise { + public async refresh(opts?: ManifestRefreshOpts): Promise { let oldManifest: Manifest; + /* + const result = await this.datasourceService.list( + this.savedObjectsClient, + { kuery: 'ingest-datasourcese.package.name:endpoint' }, + ); + console.log(result); + console.log(JSON.stringify(result)); + */ + // Get the last-dispatched manifest oldManifest = await this.getLastDispatchedManifest(ManifestConstants.SCHEMA_VERSION); @@ -156,9 +166,18 @@ export class ManifestManager { * * @return {boolean} True if dispatched. */ - public async dispatch(manifestState: NewManifestState): boolean { + public async dispatch(manifestState: NewManifestState | null): boolean { + if (manifestState === null) { + this.logger.debug('manifestState was null, aborting dispatch'); + return false; + } + if (manifestState.diffs.length > 0) { this.logger.info(`Dispatching new manifest with diffs: ${manifestState.diffs}`); + const result = await this.datasourceService.list(this.savedObjectsClient, { + kuery: 'ingest-datasources.package.name:endpoint', + }); + this.logger.debug(result); // // Datasource: // @@ -188,7 +207,12 @@ export class ManifestManager { return false; } - public async commit(manifestState: NewManifestState) { + public async commit(manifestState: NewManifestState | null) { + if (manifestState === null) { + this.logger.debug('manifestState was null, aborting commit'); + return; + } + const manifestClient = this.getManifestClient(manifestState.manifest.getSchemaVersion()); // Commit the new manifest From 53bb504df0c9f71426d15055af82d573c7717169 Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Thu, 25 Jun 2020 16:54:40 -0400 Subject: [PATCH 080/106] Adds integration with ingest manager for auth --- .../endpoint/endpoint_app_context_services.ts | 10 ++ .../server/endpoint/mocks.ts | 2 + .../artifacts/download_exception_list.test.ts | 106 ++++++++++++++++-- .../artifacts/download_exception_list.ts | 13 +++ .../manifest_manager/manifest_manager.ts | 1 + .../security_solution/server/plugin.ts | 1 + 6 files changed, 125 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts index e23d9981d21ce..1ef558f144d5a 100644 --- a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts +++ b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts @@ -3,6 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ +import { SavedObjectsServiceStart, KibanaRequest, SavedObjectsClient } from 'src/core/server'; import { AgentService, IngestManagerStartContract } from '../../../ingest_manager/server'; import { getDatasourceCreateCallback } from './ingest_integration'; import { ManifestManager } from './services/artifacts'; @@ -13,6 +14,7 @@ export type EndpointAppContextServiceStartContract = Pick< > & { manifestManager: ManifestManager; registerIngestCallback: IngestManagerStartContract['registerExternalCallback']; + savedObjectsStart: SavedObjectsServiceStart; }; /** @@ -22,10 +24,12 @@ export type EndpointAppContextServiceStartContract = Pick< export class EndpointAppContextService { private agentService: AgentService | undefined; private manifestManager: ManifestManager | undefined; + private savedObjectsStart: SavedObjectsServiceStart; public start(dependencies: EndpointAppContextServiceStartContract) { this.agentService = dependencies.agentService; this.manifestManager = dependencies.manifestManager; + this.savedObjectsStart = dependencies.savedObjectsStart; dependencies.registerIngestCallback( 'datasourceCreate', getDatasourceCreateCallback(this.manifestManager) @@ -44,4 +48,10 @@ export class EndpointAppContextService { public getManifestManager(): ManifestManager | undefined { return this.manifestManager; } + + public getScopedSavedObjects(req: KibanaRequest) { + return this.savedObjectsStart.getScopedClient(req, { + excludedWrappers: ['security'], + }); + } } diff --git a/x-pack/plugins/security_solution/server/endpoint/mocks.ts b/x-pack/plugins/security_solution/server/endpoint/mocks.ts index 3ecc0f25d265d..3324b1a680ad1 100644 --- a/x-pack/plugins/security_solution/server/endpoint/mocks.ts +++ b/x-pack/plugins/security_solution/server/endpoint/mocks.ts @@ -5,6 +5,7 @@ */ import { IScopedClusterClient, SavedObjectsClientContract } from 'kibana/server'; +import { savedObjectsServiceMock } from 'src/core/server/mocks'; import { xpackMocks } from '../../../../mocks'; import { @@ -24,6 +25,7 @@ export const createMockEndpointAppContextServiceStartContract = (): jest.Mocked< > => { return { agentService: createMockAgentService(), + savedObjectsStart: savedObjectsServiceMock.createStartContract(), manifestManager: getManifestManagerMock(), registerIngestCallback: jest.fn< ReturnType, diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts index baaa8cb387d02..57ea9c26d0b50 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts @@ -26,10 +26,7 @@ import { compressExceptionList } from '../../lib/artifacts/lists'; import { ArtifactConstants } from '../../lib/artifacts'; import { registerDownloadExceptionListRoute } from './download_exception_list'; import { EndpointAppContextService } from '../../endpoint_app_context_services'; -import { - createMockAgentService, - createMockEndpointAppContextServiceStartContract, -} from '../../mocks'; +import { createMockEndpointAppContextServiceStartContract } from '../../mocks'; import { createMockConfig } from '../../../lib/detection_engine/routes/__mocks__'; import { WrappedTranslatedExceptionList } from '../../schemas/artifacts/lists'; @@ -56,6 +53,25 @@ const expectedEndpointExceptions: WrappedTranslatedExceptionList = { }, ], }; +const mockIngestSOResponse = { + page: 1, + per_page: 100, + total: 1, + saved_objects: [ + { + id: 'agent1', + type: 'agent', + references: [], + score: 0, + attributes: { + active: true, + access_api_key_id: 'pedTuHIBTEDt93wW0Fhr', + }, + }, + ], +}; +const AuthHeader = 'ApiKey cGVkVHVISUJURUR0OTN3VzBGaHI6TnU1U0JtbHJSeC12Rm9qQWpoSHlUZw=='; + describe('test alerts route', () => { let routerMock: jest.Mocked; let mockClusterClient: jest.Mocked; @@ -66,6 +82,7 @@ describe('test alerts route', () => { let routeHandler: RequestHandler; let endpointAppContextService: EndpointAppContextService; let cache: ExceptionsCache; + let ingestSavedObjectClient: jest.Mocked; beforeEach(() => { mockClusterClient = elasticsearchServiceMock.createClusterClient(); @@ -76,7 +93,13 @@ describe('test alerts route', () => { routerMock = httpServiceMock.createRouter(); endpointAppContextService = new EndpointAppContextService(); cache = new ExceptionsCache(5); - endpointAppContextService.start(createMockEndpointAppContextServiceStartContract()); + const startContract = createMockEndpointAppContextServiceStartContract(); + + // The authentication with the Fleet Plugin needs a separate scoped SO Client + ingestSavedObjectClient = savedObjectsClientMock.create(); + ingestSavedObjectClient.find.mockReturnValue(Promise.resolve(mockIngestSOResponse)); + startContract.savedObjectsStart.getScopedClient.mockReturnValue(ingestSavedObjectClient); + endpointAppContextService.start(startContract); registerDownloadExceptionListRoute( routerMock, @@ -94,10 +117,13 @@ describe('test alerts route', () => { path: `/api/endpoint/allowlist/download/${mockArtifactName}/123456`, method: 'get', params: { sha256: '123456' }, + headers: { + authorization: AuthHeader, + }, }); + // Mock the SavedObjectsClient get response for fetching the artifact const mockCompressedArtifact = await compressExceptionList(expectedEndpointExceptions); - const mockArtifact = { id: '2468', type: 'test', @@ -112,11 +138,9 @@ describe('test alerts route', () => { size: 100, }, }; - const soFindResp: SavedObject = { ...mockArtifact, }; - mockSavedObjectClient.get.mockImplementationOnce(() => Promise.resolve(soFindResp)); [routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) => @@ -151,6 +175,9 @@ describe('test alerts route', () => { path: `/api/endpoint/allowlist/download/${mockArtifactName}/123456`, method: 'get', params: { sha256: '789' }, + headers: { + authorization: AuthHeader, + }, }); mockSavedObjectClient.get.mockImplementationOnce(() => @@ -182,6 +209,9 @@ describe('test alerts route', () => { path: `/api/endpoint/allowlist/download/${mockArtifactName}/${mockSha}`, method: 'get', params: { sha256: mockSha, identifier: mockArtifactName }, + headers: { + authorization: AuthHeader, + }, }); // Add to the download cache @@ -208,4 +238,64 @@ describe('test alerts route', () => { // The saved objects client should be bypassed as the cache will contain the download expect(mockSavedObjectClient.get.mock.calls.length).toEqual(0); }); + + it('should respond with a 401 if a valid API Token is not supplied', async () => { + const mockSha = '123456789'; + const mockRequest = httpServerMock.createKibanaRequest({ + path: `/api/endpoint/allowlist/download/${mockArtifactName}/${mockSha}`, + method: 'get', + params: { sha256: mockSha, identifier: mockArtifactName }, + }); + + [routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) => + path.startsWith('/api/endpoint/allowlist/download') + )!; + + await routeHandler( + ({ + core: { + savedObjects: { + client: mockSavedObjectClient, + }, + }, + } as unknown) as RequestHandlerContext, + mockRequest, + mockResponse + ); + expect(mockResponse.unauthorized).toBeCalled(); + }); + + it('should respond with a 404 if an agent cannot be linked to the API token', async () => { + const mockSha = '123456789'; + const mockRequest = httpServerMock.createKibanaRequest({ + path: `/api/endpoint/allowlist/download/${mockArtifactName}/${mockSha}`, + method: 'get', + params: { sha256: mockSha, identifier: mockArtifactName }, + headers: { + authorization: AuthHeader, + }, + }); + + // Mock the SavedObjectsClient find response for verifying the API token with no results + mockIngestSOResponse.saved_objects = []; + mockIngestSOResponse.total = 0; + ingestSavedObjectClient.find.mockReturnValue(Promise.resolve(mockIngestSOResponse)); + + [routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) => + path.startsWith('/api/endpoint/allowlist/download') + )!; + + await routeHandler( + ({ + core: { + savedObjects: { + client: mockSavedObjectClient, + }, + }, + } as unknown) as RequestHandlerContext, + mockRequest, + mockResponse + ); + expect(mockResponse.notFound).toBeCalled(); + }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts index 7a1bc8181932e..678dd5235f718 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts @@ -5,6 +5,8 @@ */ import { IRouter } from 'src/core/server'; +// eslint-disable-next-line @kbn/eslint/no-restricted-paths +import { authenticateAgentWithAccessToken } from '../../../../../ingest_manager/server/services/agents/authenticate'; import { validate } from '../../../../common/validate'; import { buildRouteValidation } from '../../../utils/build_validation/route_validation'; import { ArtifactConstants, ExceptionsCache } from '../../lib/artifacts'; @@ -60,6 +62,17 @@ export function registerDownloadExceptionListRoute( } }; + try { + const scopedSOClient = endpointContext.service.getScopedSavedObjects(req); + await authenticateAgentWithAccessToken(scopedSOClient, req); + } catch (err) { + if (err.output.statusCode === 401) { + return res.unauthorized(); + } else { + return res.notFound(); + } + } + const id = `${req.params.identifier}-${req.params.sha256}`; const cacheResp = cache.get(id); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts index abbd6c1d5c187..36126a36aea6f 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts @@ -108,6 +108,7 @@ export class ManifestManager { // Get the last-dispatched manifest oldManifest = await this.getLastDispatchedManifest(ManifestConstants.SCHEMA_VERSION); + // console.log(oldManifest); if (oldManifest === null && opts.initialize) { oldManifest = new Manifest(new Date(), ManifestConstants.SCHEMA_VERSION); // create empty manifest diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 2e8b92f309021..b058eac73d093 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -261,6 +261,7 @@ export class Plugin implements IPlugin Date: Thu, 25 Jun 2020 19:59:45 -0400 Subject: [PATCH 081/106] Update test fixture --- .../exceptions/api_feature/exception_list/data.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/test/functional/es_archives/security_solution/exceptions/api_feature/exception_list/data.json b/x-pack/test/functional/es_archives/security_solution/exceptions/api_feature/exception_list/data.json index 3991e1286b9f0..8f03dead04798 100644 --- a/x-pack/test/functional/es_archives/security_solution/exceptions/api_feature/exception_list/data.json +++ b/x-pack/test/functional/es_archives/security_solution/exceptions/api_feature/exception_list/data.json @@ -1,7 +1,7 @@ { "type": "doc", "value": { - "id": "securitySolution-exceptions-artifact:endpoint-allowlist-linux-1.0.0", + "id": "securitySolution-exceptions-artifact:endpoint-allowlist-linux-1.0.0-0fa0ba277627f7cc116e198bac3e524e9a8befdca1591a1dd4830dc7952b2103", "index": ".kibana_1", "source": { "references": [ @@ -24,7 +24,7 @@ { "type": "doc", "value": { - "id": "securitySolution-exceptions-artifact:endpoint-allowlist-windows-1.0.0", + "id": "securitySolution-exceptions-artifact:endpoint-allowlist-windows-1.0.0-1825fb19fcc6dc391cae0bc4a2e96dd7f728a0c3ae9e1469251ada67f9e1b975", "index": ".kibana_1", "source": { "references": [ From 484ad9974782286c5fb194f6bb1ec65715342531 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Thu, 25 Jun 2020 22:17:45 -0400 Subject: [PATCH 082/106] Add manifest dispatch --- .../server/endpoint/ingest_integration.ts | 4 +- .../server/endpoint/lib/artifacts/manifest.ts | 2 +- .../server/endpoint/lib/artifacts/task.ts | 1 + .../manifest_manager/manifest_manager.ts | 79 ++++++++++--------- 4 files changed, 47 insertions(+), 39 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts index 928bea23d8ff7..e5ee374f614fc 100644 --- a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts +++ b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts @@ -39,7 +39,9 @@ export const getDatasourceCreateCallback = ( enabled: true, streams: [], config: { - artifact_manifest: manifestState.manifest.toEndpointFormat(), + artifact_manifest: { + value: manifestState.manifest.toEndpointFormat(), + }, policy: { value: policyConfigFactory(), }, diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts index a3189726e4c17..533d66d9cf593 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts @@ -100,7 +100,7 @@ export class Manifest { public toEndpointFormat(): ManifestSchema { const manifestObj: ManifestSchema = { - manifestVersion: this.version, + manifestVersion: this.version ?? 'baseline', schemaVersion: this.schemaVersion, artifacts: {}, }; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts index 95e60f643eff7..8bc6893a622ab 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts @@ -63,6 +63,7 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { if (manifestState !== null) { if (await manifestManager.dispatch(manifestState)) { await manifestManager.commit(manifestState); + logger.debug(`Committed manifest ${manifestState.manifest.getVersion()}`); } } } catch (err) { diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts index acad5c2408fd6..5d6e844ee1ed2 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts @@ -7,6 +7,7 @@ import { Logger, SavedObjectsClient } from '../../../../../../../../src/core/server'; import { DatasourceServiceInterface } from '../../../../../../ingest_manager/server'; import { ExceptionListClient } from '../../../../../../lists/server'; +import { NewPolicyData } from '../../../../../common/endpoint/types'; import { ArtifactConstants, ManifestConstants, @@ -107,20 +108,11 @@ export class ManifestManager { public async refresh(opts?: ManifestRefreshOpts): Promise { let oldManifest: Manifest; - /* - const result = await this.datasourceService.list( - this.savedObjectsClient, - { kuery: 'ingest-datasourcese.package.name:endpoint' }, - ); - console.log(result); - console.log(JSON.stringify(result)); - */ - // Get the last-dispatched manifest oldManifest = await this.getLastDispatchedManifest(ManifestConstants.SCHEMA_VERSION); // console.log(oldManifest); - if (oldManifest === null && opts.initialize) { + if (oldManifest === null && opts !== undefined && opts.initialize) { oldManifest = new Manifest(new Date(), ManifestConstants.SCHEMA_VERSION); // create empty manifest } else if (oldManifest == null) { this.logger.debug('Manifest does not exist yet. Waiting...'); @@ -175,32 +167,41 @@ export class ManifestManager { if (manifestState.diffs.length > 0) { this.logger.info(`Dispatching new manifest with diffs: ${manifestState.diffs}`); - const result = await this.datasourceService.list(this.savedObjectsClient, { - kuery: 'ingest-datasources.package.name:endpoint', - }); - this.logger.debug(result); - // - // Datasource: - // - // { name: 'endpoint-1', - // description: '', - // config_id: 'c7fe80d0-b677-11ea-8bb2-09d7226f2862', - // enabled: true, - // output_id: '', - // inputs: - // [ { type: 'endpoint', enabled: true, streams: [], config: [Object] } ], - // namespace: 'default', - // package: - // { name: 'endpoint', title: 'Elastic Endpoint', version: '0.5.0' } } - // - // TODO: write to config - // await this.datasourceService.get('the-datasource-id'); - // OR - // await this.datasourceService.list('the-datasource-by-config_id'); - // THEN - // await this.datasourceService.update(...args); - // - return true; + + let paging = true; + let success = true; + + while (paging) { + const { items, total, page } = await this.datasourceService.list(this.savedObjectsClient, { + kuery: 'ingest-datasources.package.name:endpoint', + }); + + items.forEach(async (datasource) => { + const { id, revision, updated_at, updated_by, ...newDatasource } = datasource; + + if (newDatasource.inputs.length > 0) { + newDatasource.inputs[0].config.artifact_manifest = manifestState.manifest.toEndpointFormat(); + + this.datasourceService + .update(this.savedObjectsClient, id, newDatasource) + .then((response) => { + this.logger.debug(`Updated datasource ${id}`); + }) + .catch((err) => { + success = false; + this.logger.debug(`Error updating datasource ${id}`); + this.logger.error(err); + }); + } else { + success = false; + this.logger.debug(`Datasource ${id} has no inputs.`); + } + }, this); + + paging = page * items.length < total; + } + + return success; } else { this.logger.debug('No manifest diffs [no-op]'); } @@ -220,8 +221,12 @@ export class ManifestManager { if (manifestState.manifest.getVersion() === undefined) { await manifestClient.createManifest(manifestState.manifest.toSavedObject()); } else { + const version = manifestState.manifest.getVersion(); + if (version === 'baseline') { + throw new Error('Updating existing manifest with baseline version. Bad state.'); + } await manifestClient.updateManifest(manifestState.manifest.toSavedObject(), { - version: manifestState.manifest.getVersion(), + version, }); } From 16d68a699ebbb531a6a86150dc2a40cd3ed91851 Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Fri, 26 Jun 2020 11:34:21 -0400 Subject: [PATCH 083/106] Refactoring to use the same SO Client from ingest --- .../artifacts/download_exception_list.test.ts | 6 ++--- .../artifacts/download_exception_list.ts | 27 +++++++++---------- .../manifest_manager/manifest_manager.mock.ts | 2 +- 3 files changed, 17 insertions(+), 18 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts index 57ea9c26d0b50..9a8e8fe0dfcba 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts @@ -141,7 +141,7 @@ describe('test alerts route', () => { const soFindResp: SavedObject = { ...mockArtifact, }; - mockSavedObjectClient.get.mockImplementationOnce(() => Promise.resolve(soFindResp)); + ingestSavedObjectClient.get.mockImplementationOnce(() => Promise.resolve(soFindResp)); [routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) => path.startsWith('/api/endpoint/allowlist/download') @@ -180,7 +180,7 @@ describe('test alerts route', () => { }, }); - mockSavedObjectClient.get.mockImplementationOnce(() => + ingestSavedObjectClient.get.mockImplementationOnce(() => // eslint-disable-next-line prefer-promise-reject-errors Promise.reject({ output: { statusCode: 404 } }) ); @@ -236,7 +236,7 @@ describe('test alerts route', () => { ); expect(mockResponse.ok).toBeCalled(); // The saved objects client should be bypassed as the cache will contain the download - expect(mockSavedObjectClient.get.mock.calls.length).toEqual(0); + expect(ingestSavedObjectClient.get.mock.calls.length).toEqual(0); }); it('should respond with a 401 if a valid API Token is not supplied', async () => { diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts index 678dd5235f718..8f4304de14c8c 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts @@ -40,11 +40,20 @@ export function registerDownloadExceptionListRoute( options: { tags: [] }, }, async (context, req, res) => { - const soClient = context.core.savedObjects.client; + let scopedSOClient; const logger = endpointContext.logFactory.get('download_exception_list'); - // TODO: authenticate api key - // https://github.com/elastic/kibana/issues/69329 + // The ApiKey must be associated with an enrolled Fleet agent + try { + scopedSOClient = endpointContext.service.getScopedSavedObjects(req); + await authenticateAgentWithAccessToken(scopedSOClient, req); + } catch (err) { + if (err.output.statusCode === 401) { + return res.unauthorized(); + } else { + return res.notFound(); + } + } const buildAndValidateResponse = (artName: string, body: string): object => { const artifact = { @@ -62,16 +71,6 @@ export function registerDownloadExceptionListRoute( } }; - try { - const scopedSOClient = endpointContext.service.getScopedSavedObjects(req); - await authenticateAgentWithAccessToken(scopedSOClient, req); - } catch (err) { - if (err.output.statusCode === 401) { - return res.unauthorized(); - } else { - return res.notFound(); - } - } const id = `${req.params.identifier}-${req.params.sha256}`; const cacheResp = cache.get(id); @@ -81,7 +80,7 @@ export function registerDownloadExceptionListRoute( return buildAndValidateResponse(req.params.identifier, cacheResp); } else { logger.debug(`Cache MISS artifact ${id}`); - return soClient + return scopedSOClient .get(ArtifactConstants.SAVED_OBJECT_TYPE, id) .then((artifact) => { cache.set(id, artifact.attributes.body); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts index 86124ae88c897..d066a615263ec 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts @@ -37,7 +37,7 @@ function getMockDatasource() { // TODO // eslint-disable-next-line max-classes-per-file -class DatasourceServiceMock extends DatasourceServiceInterface { +class DatasourceServiceMock { public create = jest.fn().mockResolvedValue(getMockDatasource()); public get = jest.fn().mockResolvedValue(getMockDatasource()); public getByIds = jest.fn().mockResolvedValue([getMockDatasource()]); From a8c3a405c769232aeb6f1e985847df0ac9749cd5 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Fri, 26 Jun 2020 12:17:30 -0400 Subject: [PATCH 084/106] bug fixes --- .../server/endpoint/ingest_integration.ts | 8 +-- .../server/endpoint/lib/artifacts/task.ts | 28 ++++---- .../manifest_manager/manifest_manager.ts | 64 +++++++++++-------- 3 files changed, 56 insertions(+), 44 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts index e5ee374f614fc..3cc0471b2df5c 100644 --- a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts +++ b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts @@ -25,8 +25,8 @@ export const getDatasourceCreateCallback = ( // follow the types/schema expected let updatedDatasource = newDatasource as NewPolicyData; - const manifestState = await manifestManager.refresh({ initialize: true }); - if (manifestState !== null) { + const wrappedManifest = await manifestManager.refresh({ initialize: true }); + if (wrappedManifest !== null) { // Until we get the Default Policy Configuration in the Endpoint package, // we will add it here manually at creation time. // @ts-ignore @@ -40,7 +40,7 @@ export const getDatasourceCreateCallback = ( streams: [], config: { artifact_manifest: { - value: manifestState.manifest.toEndpointFormat(), + value: wrappedManifest.manifest.toEndpointFormat(), }, policy: { value: policyConfigFactory(), @@ -55,7 +55,7 @@ export const getDatasourceCreateCallback = ( try { return updatedDatasource; } finally { - await manifestManager.commit(manifestState); + await manifestManager.commit(wrappedManifest); } }; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts index 8bc6893a622ab..fea3ffa585232 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts @@ -39,9 +39,7 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { return `${PackagerTaskConstants.TYPE}:${PackagerTaskConstants.VERSION}`; }; - const logger = context.endpointAppContext.logFactory.get( - `endpoint_manifest_refresh_${getTaskId()}` - ); + const logger = context.endpointAppContext.logFactory.get(getTaskId()); const run = async (taskId: string) => { // Check that this task is current @@ -58,17 +56,21 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { return; } - try { - const manifestState = await manifestManager.refresh(); - if (manifestState !== null) { - if (await manifestManager.dispatch(manifestState)) { - await manifestManager.commit(manifestState); - logger.debug(`Committed manifest ${manifestState.manifest.getVersion()}`); + manifestManager + .refresh() + .then((wrappedManifest) => { + if (wrappedManifest !== null) { + return manifestManager.dispatch(wrappedManifest); } - } - } catch (err) { - logger.error(err); - } + }) + .then((wrappedManifest) => { + if (wrappedManifest !== null) { + return manifestManager.commit(wrappedManifest); + } + }) + .catch((err) => { + logger.error(err); + }); }; const getTaskRunner = (runnerContext: PackagerTaskRunnerContext): PackagerTaskRunner => { diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts index 5d6e844ee1ed2..7672c2134dd35 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts @@ -33,7 +33,7 @@ export interface ManifestRefreshOpts { initialize?: boolean; } -export interface NewManifestState { +export interface WrappedManifest { manifest: Manifest; diffs: ManifestDiff[]; } @@ -105,7 +105,7 @@ export class ManifestManager { } } - public async refresh(opts?: ManifestRefreshOpts): Promise { + public async refresh(opts?: ManifestRefreshOpts): Promise { let oldManifest: Manifest; // Get the last-dispatched manifest @@ -130,7 +130,7 @@ export class ManifestManager { const diffs = newManifest.diff(oldManifest); // Create new artifacts - diffs.forEach(async (diff) => { + for (const diff of diffs) { if (diff.type === 'add') { const artifact = newManifest.getArtifact(diff.id); try { @@ -146,7 +146,7 @@ export class ManifestManager { } } } - }, this); + } return { manifest: newManifest, @@ -157,16 +157,23 @@ export class ManifestManager { /** * Dispatches the manifest by writing it to the endpoint datasource. * - * @return {boolean} True if dispatched. + * @return {WrappedManifest | null} WrappedManifest if all dispatched, else null */ - public async dispatch(manifestState: NewManifestState | null): boolean { - if (manifestState === null) { - this.logger.debug('manifestState was null, aborting dispatch'); - return false; + public async dispatch(wrappedManifest: WrappedManifest | null): WrappedManifest | null { + if (wrappedManifest === null) { + this.logger.debug('wrappedManifest was null, aborting dispatch'); + return null; + } + + function showDiffs(diffs: ManifestDiff[]) { + return diffs.map((diff) => { + const op = diff.type === 'add' ? '(+)' : '(-)'; + return `${op}${diff.id}`; + }); } - if (manifestState.diffs.length > 0) { - this.logger.info(`Dispatching new manifest with diffs: ${manifestState.diffs}`); + if (wrappedManifest.diffs.length > 0) { + this.logger.info(`Dispatching new manifest with diffs: ${showDiffs(wrappedManifest.diffs)}`); let paging = true; let success = true; @@ -176,13 +183,13 @@ export class ManifestManager { kuery: 'ingest-datasources.package.name:endpoint', }); - items.forEach(async (datasource) => { + for (const datasource of items) { const { id, revision, updated_at, updated_by, ...newDatasource } = datasource; if (newDatasource.inputs.length > 0) { - newDatasource.inputs[0].config.artifact_manifest = manifestState.manifest.toEndpointFormat(); + newDatasource.inputs[0].config.artifact_manifest = wrappedManifest.manifest.toEndpointFormat(); - this.datasourceService + await this.datasourceService .update(this.savedObjectsClient, id, newDatasource) .then((response) => { this.logger.debug(`Updated datasource ${id}`); @@ -196,49 +203,52 @@ export class ManifestManager { success = false; this.logger.debug(`Datasource ${id} has no inputs.`); } - }, this); + } paging = page * items.length < total; } - return success; + return success ? wrappedManifest : null; } else { this.logger.debug('No manifest diffs [no-op]'); } - return false; + return null; } - public async commit(manifestState: NewManifestState | null) { - if (manifestState === null) { - this.logger.debug('manifestState was null, aborting commit'); + public async commit(wrappedManifest: WrappedManifest | null) { + if (wrappedManifest === null) { + this.logger.debug('wrappedManifest was null, aborting commit'); return; } - const manifestClient = this.getManifestClient(manifestState.manifest.getSchemaVersion()); + const manifestClient = this.getManifestClient(wrappedManifest.manifest.getSchemaVersion()); // Commit the new manifest - if (manifestState.manifest.getVersion() === undefined) { - await manifestClient.createManifest(manifestState.manifest.toSavedObject()); + if (wrappedManifest.manifest.getVersion() === undefined) { + await manifestClient.createManifest(wrappedManifest.manifest.toSavedObject()); } else { - const version = manifestState.manifest.getVersion(); + const version = wrappedManifest.manifest.getVersion(); if (version === 'baseline') { throw new Error('Updating existing manifest with baseline version. Bad state.'); } - await manifestClient.updateManifest(manifestState.manifest.toSavedObject(), { + await manifestClient.updateManifest(wrappedManifest.manifest.toSavedObject(), { version, }); } + this.logger.info(`Commited manifest ${wrappedManifest.manifest.getVersion()}`); + // Clean up old artifacts - manifestState.diffs.forEach(async (diff) => { + for (const diff of wrappedManifest.diffs) { try { if (diff.type === 'delete') { await this.artifactClient.deleteArtifact(diff.id); + this.logger.info(`Cleaned up artifact ${diff.id}`); } } catch (err) { this.logger.error(err); } - }, this); + } } } From 5f2e539d45a90feb8ea0bede2e0021ae884a5070 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Fri, 26 Jun 2020 13:47:08 -0400 Subject: [PATCH 085/106] build renovate config --- renovate.json5 | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/renovate.json5 b/renovate.json5 index 1af155fcc645e..ecaa267e0868a 100644 --- a/renovate.json5 +++ b/renovate.json5 @@ -589,6 +589,14 @@ '@types/lru-cache', ], }, + { + groupSlug: 'lzma-native', + groupName: 'lzma-native related packages', + packageNames: [ + 'lzma-native', + '@types/lzma-native', + ], + }, { groupSlug: 'mapbox-gl', groupName: 'mapbox-gl related packages', From 6854602101d18d4ae6b4e0ed7f1b4f4a275bc193 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Fri, 26 Jun 2020 16:14:15 -0400 Subject: [PATCH 086/106] Fix endpoint_app_context_services tests --- .../endpoint/endpoint_app_context_services.test.ts | 11 +++++++++-- .../server/endpoint/endpoint_app_context_services.ts | 12 +++++++----- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts index ff09d67a972bc..85c949ae0a7f6 100644 --- a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts @@ -4,14 +4,21 @@ * you may not use this file except in compliance with the Elastic License. */ import { EndpointAppContextService } from './endpoint_app_context_services'; +import { httpServerMock } from '../../../../../src/core/server/mocks'; describe('test endpoint app context services', () => { it('should throw error on getAgentService if start is not called', async () => { const endpointAppContextService = new EndpointAppContextService(); expect(() => endpointAppContextService.getAgentService()).toThrow(Error); }); - it('should throw error on getManifestManager if start is not called', async () => { + it('should return undefined on getManifestManager if start is not called', async () => { const endpointAppContextService = new EndpointAppContextService(); - expect(() => endpointAppContextService.getManifestManager()).toThrow(Error); + expect(endpointAppContextService.getManifestManager()).toEqual(undefined); + }); + it('should return undefined on getScopedSavedObjects if start is not called', async () => { + const endpointAppContextService = new EndpointAppContextService(); + expect( + endpointAppContextService.getScopedSavedObjects(httpServerMock.createKibanaRequest()) + ).toEqual(undefined); }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts index 1ef558f144d5a..55d33e4923581 100644 --- a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts +++ b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts @@ -24,7 +24,7 @@ export type EndpointAppContextServiceStartContract = Pick< export class EndpointAppContextService { private agentService: AgentService | undefined; private manifestManager: ManifestManager | undefined; - private savedObjectsStart: SavedObjectsServiceStart; + private savedObjectsStart: SavedObjectsServiceStart | undefined; public start(dependencies: EndpointAppContextServiceStartContract) { this.agentService = dependencies.agentService; @@ -49,9 +49,11 @@ export class EndpointAppContextService { return this.manifestManager; } - public getScopedSavedObjects(req: KibanaRequest) { - return this.savedObjectsStart.getScopedClient(req, { - excludedWrappers: ['security'], - }); + public getScopedSavedObjects(req: KibanaRequest): SavedObjectsClient | undefined { + let client: SavedObjectsClient; + if (this.savedObjectsStart !== undefined) { + client = this.savedObjectsStart.getScopedClient(req, { excludedWrappers: ['security'] }); + } + return client; } } From 536a3b23d0d1807b48578db66c7abe81b6b187f2 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Fri, 26 Jun 2020 16:41:14 -0400 Subject: [PATCH 087/106] Only index the fields that are necessary for searching --- .../server/endpoint/lib/artifacts/saved_object_mappings.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts index f805c117e34e1..d8f7b2485e2a8 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts @@ -22,15 +22,19 @@ export const exceptionsArtifactSavedObjectMappings: SavedObjectsType['mappings'] }, encoding: { type: 'keyword', + index: false, }, created: { type: 'date', + index: false, }, body: { type: 'binary', + index: false, }, size: { type: 'long', + index: false, }, }, }; @@ -39,10 +43,12 @@ export const manifestSavedObjectMappings: SavedObjectsType['mappings'] = { properties: { created: { type: 'date', + index: false, }, // array of doc ids ids: { type: 'keyword', + index: false, }, }, }; From 4356e085ef9d585ba9085a8d2a1651fba5773adb Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Fri, 26 Jun 2020 17:15:05 -0400 Subject: [PATCH 088/106] Integ test progress --- .../apis/security_solution/exceptions.ts | 14 +- .../api_feature/exception_list/data.json | 180 +++++++++++++++--- 2 files changed, 168 insertions(+), 26 deletions(-) diff --git a/x-pack/test/api_integration/apis/security_solution/exceptions.ts b/x-pack/test/api_integration/apis/security_solution/exceptions.ts index 79b05ce1c6eda..9650318923d91 100644 --- a/x-pack/test/api_integration/apis/security_solution/exceptions.ts +++ b/x-pack/test/api_integration/apis/security_solution/exceptions.ts @@ -7,27 +7,35 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../ftr_provider_context'; +import { getSupertestWithoutAuth } from '../fleet/agents/services'; -export default function ({ getService }: FtrProviderContext) { +export default function (providerContext: FtrProviderContext) { + const { getService } = providerContext; const esArchiver = getService('esArchiver'); const supertest = getService('supertest'); + const supertestWithoutAuth = getSupertestWithoutAuth(providerContext); + const authKey = 'OFpuVDdISUJ3TEZ2a1VFUFFhVDM6TnU1U0JtbHJSeC12Rm9qQWpoSHlUZw=='; describe('artifact download', () => { before(() => esArchiver.load('security_solution/exceptions/api_feature/exception_list')); after(() => esArchiver.unload('security_solution/exceptions/api_feature/exception_list')); it('should fail to find artifact with invalid hash', async () => { - const { body } = await supertest + const { body } = await supertestWithoutAuth .get('/api/endpoint/allowlist/download/endpoint-allowlist-windows-1.0.0/abcd') + .set('kbn-xsrf', 'xxx') + .set('authorization', `ApiKey ${authKey}`) .send() .expect(404); }); it('should download an artifact with correct hash', async () => { - const { body } = await supertest + const { body } = await supertestWithoutAuth .get( '/api/endpoint/allowlist/download/endpoint-allowlist-windows-1.0.0/1825fb19fcc6dc391cae0bc4a2e96dd7f728a0c3ae9e1469251ada67f9e1b975' ) + .set('kbn-xsrf', 'xxx') + .set('authorization', `ApiKey ${authKey}`) .send() .expect(200); }); diff --git a/x-pack/test/functional/es_archives/security_solution/exceptions/api_feature/exception_list/data.json b/x-pack/test/functional/es_archives/security_solution/exceptions/api_feature/exception_list/data.json index 8f03dead04798..7f51fbd0c3503 100644 --- a/x-pack/test/functional/es_archives/security_solution/exceptions/api_feature/exception_list/data.json +++ b/x-pack/test/functional/es_archives/security_solution/exceptions/api_feature/exception_list/data.json @@ -1,22 +1,21 @@ { "type": "doc", "value": { - "id": "securitySolution-exceptions-artifact:endpoint-allowlist-linux-1.0.0-0fa0ba277627f7cc116e198bac3e524e9a8befdca1591a1dd4830dc7952b2103", - "index": ".kibana_1", + "id": "securitySolution:endpoint:exceptions-artifact:endpoint-allowlist-macos-1.0.0-1825fb19fcc6dc391cae0bc4a2e96dd7f728a0c3ae9e1469251ada67f9e1b975", + "index": ".kibana", "source": { "references": [ ], - "securitySolution-exceptions-artifact": { - "body": "ý7zXZ\u0000\u0000\u0001i\"Þ6\u0002\u0000!\u0001\u0016\u0000\u0000\u0000t/å£à\u0000ÿ\u0000©]\u0000=ˆˆ§Ã{\u0000Š&W“{Wï¶=\f‚‚söÊ\u001eæõ\u0014WI1dƒÛtÂ\u00185ý\u0012[.?=ÃB#­òW€ì¦\u0006)ä˜ÕVkÌâ\b´Ï; ãÍ¥Á’‚Ê“èÕ£ÅuŸÜ0ê\t\u0001\u0002ÿmQ\u001b«Ï_°+Ræ­H¦,x¶½OÂ\u0004P\u001f*YP#ƒÐ.0\r‹\u0002\u0000\u0000\u0000\u0000\u0001YZ", - "created": 1592414128607, + "securitySolution:endpoint:exceptions-artifact": { + "body": "ý7zXZ\u0000\u0000\u0001i\"Þ6\u0002\u0000!\u0001\u0016\u0000\u0000\u0000t/å£\u0001\u0000\u0015{\"exceptions_list\":[]}\u0000\u0000\u000052¤\u0000\u0001*\u0016RÌ9»B™\r\u0001\u0000\u0000\u0000\u0000\u0001YZ", + "created": 1593016187465, "encoding": "xz", - "name": "endpoint-allowlist-linux-1.0.0", - "schemaVersion": "1.0.0", - "sha256": "0fa0ba277627f7cc116e198bac3e524e9a8befdca1591a1dd4830dc7952b2103", - "size": 256 + "identifier": "endpoint-allowlist-macos-1.0.0", + "sha256": "1825fb19fcc6dc391cae0bc4a2e96dd7f728a0c3ae9e1469251ada67f9e1b975", + "size": 22 }, - "type": "securitySolution-exceptions-artifact", - "updated_at": "2020-06-17T17:15:28.607Z" + "type": "securitySolution:endpoint:exceptions-artifact", + "updated_at": "2020-06-24T16:29:47.584Z" } } } @@ -24,22 +23,157 @@ { "type": "doc", "value": { - "id": "securitySolution-exceptions-artifact:endpoint-allowlist-windows-1.0.0-1825fb19fcc6dc391cae0bc4a2e96dd7f728a0c3ae9e1469251ada67f9e1b975", - "index": ".kibana_1", + "id": "securitySolution:endpoint:exceptions-manifest:endpoint-manifest-1.0.0", + "index": ".kibana", "source": { "references": [ ], - "securitySolution-exceptions-artifact": { - "body": "ý7zXZ\u0000\u0000\u0001i\"Þ6\u0002\u0000!\u0001\u0016\u0000\u0000\u0000t/å£\u0001\u0000\u0015{\"exceptions_list\":[]}\u0000\u0000\u000052¤\u0000\u0001*\u0016RÌ9»B™\r\u0001\u0000\u0000\u0000\u0000\u0001YZ", - "created": 1592414129763, - "encoding": "xz", - "name": "endpoint-allowlist-windows-1.0.0", - "schemaVersion": "1.0.0", - "sha256": "1825fb19fcc6dc391cae0bc4a2e96dd7f728a0c3ae9e1469251ada67f9e1b975", - "size": 22 + "securitySolution:endpoint:exceptions-manifest": { + "created": 1593183699663, + "ids": [ + "endpoint-allowlist-linux-1.0.0-6b065fbf5d9ae82039ca20e2f3f1629d2cd9d8d3f3517d0f295f879bf939a269", + "endpoint-allowlist-macos-1.0.0-1825fb19fcc6dc391cae0bc4a2e96dd7f728a0c3ae9e1469251ada67f9e1b975", + "endpoint-allowlist-windows-1.0.0-dbba762118dadd31597eb2cf6a39ff5a45aa31b27c1671f8b4d1e403d2102a77" + ] + }, + "type": "securitySolution:endpoint:exceptions-manifest", + "updated_at": "2020-06-26T15:01:39.704Z" + } + } +} + +{ + "type": "doc", + "value": { + "id": "exception-list-agnostic:13a7ef40-b63b-11ea-ace9-591c8e572c76", + "index": ".kibana", + "source": { + "exception-list-agnostic": { + "_tags": [ + "endpoint", + "process", + "malware", + "os:linux" + ], + "created_at": "2020-06-24T16:52:23.689Z", + "created_by": "akahan", + "description": "This is a sample agnostic endpoint type exception", + "list_id": "endpoint_list", + "list_type": "list", + "name": "Sample Endpoint Exception List", + "tags": [ + "user added string for a tag", + "malware" + ], + "tie_breaker_id": "e3b20e6e-c023-4575-a033-47990115969c", + "type": "endpoint", + "updated_by": "akahan" + }, + "references": [ + ], + "type": "exception-list-agnostic", + "updated_at": "2020-06-24T16:52:23.732Z" + } + } +} + +{ + "type": "doc", + "value": { + "id": "exception-list-agnostic:679b95a0-b714-11ea-a4c9-0963ae39bc3d", + "index": ".kibana", + "source": { + "exception-list-agnostic": { + "_tags": [ + "os:windows" + ], + "comments": [ + ], + "created_at": "2020-06-25T18:48:05.326Z", + "created_by": "akahan", + "description": "This is a sample endpoint type exception", + "entries": [ + { + "field": "actingProcess.file.signer", + "operator": "included", + "type": "match", + "value": "Elastic, N.V." + }, + { + "field": "event.category", + "operator": "included", + "type": "match_any", + "value": [ + "process", + "malware" + ] + } + ], + "item_id": "61142b8f-5876-4709-9952-95160cd58f2f", + "list_id": "endpoint_list", + "list_type": "item", + "name": "Sample Endpoint Exception List", + "tags": [ + "user added string for a tag", + "malware" + ], + "tie_breaker_id": "b36176d2-bc75-4641-a8e3-e811c6bc30d8", + "type": "endpoint", + "updated_by": "akahan" }, - "type": "securitySolution-exceptions-artifact", - "updated_at": "2020-06-17T17:15:29.763Z" + "references": [ + ], + "type": "exception-list-agnostic", + "updated_at": "2020-06-25T18:48:05.369Z" } } } + +{ + "type": "doc", + "value": { + "id": "fleet-agents:a34d87c1-726e-4c30-b2ff-1b4b95f59d2a", + "index": ".kibana", + "source": { + "fleet-agents": { + "access_api_key_id": "8ZnT7HIBwLFvkUEPQaT3", + "active": true, + "config_id": "2dd2a110-b6f6-11ea-a66d-63cf082a3b58", + "enrolled_at": "2020-06-25T18:52:47.290Z", + "local_metadata": { + "os": "macos" + }, + "type": "PERMANENT", + "user_provided_metadata": { + "region": "us-east" + } + }, + "references": [ + ], + "type": "fleet-agents", + "updated_at": "2020-06-25T18:52:48.464Z" + } + } +} + +{ + "type": "doc", + "value": { + "id": "fleet-enrollment-api-keys:8178eb66-392f-4b76-9dc9-704ed1a5c56e", + "index": ".kibana", + "source": { + "fleet-enrollment-api-keys": { + "active": true, + "api_key": "8ZnT7HIBwLFvkUEPQaT3", + "api_key_id": "8ZnT7HIBwLFvkUEPQaT3", + "config_id": "2dd2a110-b6f6-11ea-a66d-63cf082a3b58", + "created_at": "2020-06-25T17:25:30.065Z", + "name": "Default (93aa98c8-d650-422e-aa7b-663dae3dff83)" + }, + "references": [ + ], + "type": "fleet-enrollment-api-keys", + "updated_at": "2020-06-25T17:25:30.114Z" + } + } +} \ No newline at end of file From 7f8d9d808b22b77d1a65a4dd5204b0fad6420d86 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Fri, 26 Jun 2020 18:10:25 -0400 Subject: [PATCH 089/106] mock and test city --- .../endpoint/lib/artifacts/cache.test.ts | 2 +- .../server/endpoint/lib/artifacts/cache.ts | 3 + .../server/endpoint/lib/artifacts/manifest.ts | 2 + .../lib/artifacts/manifest_entry.test.ts | 16 +-- .../endpoint/lib/artifacts/manifest_entry.ts | 2 +- .../endpoint/lib/artifacts/task.mock.ts | 23 +++++ .../endpoint/lib/artifacts/task.test.ts | 40 ++++++++ .../server/endpoint/lib/artifacts/task.ts | 98 ++++++++++--------- .../server/endpoint/mocks.ts | 18 +++- .../artifacts/download_exception_list.ts | 1 - .../artifacts/artifact_client.test.ts | 4 + .../artifacts/manifest_client.test.ts | 4 + .../security_solution/server/plugin.ts | 2 +- 13 files changed, 155 insertions(+), 60 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.mock.ts diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts index d4006faced210..5a0fb91345552 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts @@ -26,7 +26,7 @@ describe('ExceptionsCache tests', () => { expect(cacheResp).toEqual(undefined); }); - test('it should handle cache clean', async () => { + test('it should handle cache eviction', async () => { cache.set('1', 'a'); cache.set('2', 'b'); cache.set('3', 'c'); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts index 6e0c8f73531ce..b7a4c2feb6bf8 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts @@ -6,6 +6,9 @@ const DEFAULT_MAX_SIZE = 10; +/** + * FIFO cache implementation for artifact downloads. + */ export class ExceptionsCache { private cache: Map; private queue: string[]; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts index 533d66d9cf593..0ed2bc954dbb0 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts @@ -34,9 +34,11 @@ export class Manifest { (schemaVersion as unknown) as object, manifestSchemaVersion ); + if (errors != null) { throw new Error(`Invalid manifest version: ${schemaVersion}`); } + this.schemaVersion = validated; } diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts index 9d76332df9169..dda7e8e75443f 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts @@ -18,41 +18,41 @@ describe('manifest_entry', () => { manifestEntry = new ManifestEntry(artifact); }); - test('Can create manifest entry', async () => { + test('Can create manifest entry', () => { expect(manifestEntry).toBeInstanceOf(ManifestEntry); }); - test('Correct doc_id is returned', async () => { + test('Correct doc_id is returned', () => { expect(manifestEntry.getDocId()).toEqual( 'endpoint-allowlist-windows-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466' ); }); - test('Correct identifier is returned', async () => { + test('Correct identifier is returned', () => { expect(manifestEntry.getIdentifier()).toEqual('endpoint-allowlist-windows-1.0.0'); }); - test('Correct sha256 is returned', async () => { + test('Correct sha256 is returned', () => { expect(manifestEntry.getSha256()).toEqual( '222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466' ); }); - test('Correct size is returned', async () => { + test('Correct size is returned', () => { expect(manifestEntry.getSize()).toEqual(268); }); - test('Correct url is returned', async () => { + test('Correct url is returned', () => { expect(manifestEntry.getUrl()).toEqual( '/api/endpoint/allowlist/download/endpoint-allowlist-windows-1.0.0/222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466' ); }); - test('Correct artifact is returned', async () => { + test('Correct artifact is returned', () => { expect(manifestEntry.getArtifact()).toEqual(artifact); }); - test('Correct record is returned', async () => { + test('Correct record is returned', () => { expect(manifestEntry.getRecord()).toEqual({ sha256: '222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', size: 268, diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts index f330054d15453..f5fd23c978c32 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts @@ -40,9 +40,9 @@ export class ManifestEntry { public getRecord(): ManifestEntrySchema { return { - url: this.getUrl(), sha256: this.getSha256(), size: this.getSize(), + url: this.getUrl(), }; } } diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.mock.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.mock.ts new file mode 100644 index 0000000000000..bb79f2ffc892d --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.mock.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { taskManagerStartContract } from '../../../../../task_manager/server'; + +import { taskManagerMock } from '../../../../../task_manager/server/mocks'; + +import { createMockEndpointAppContext } from '../../mocks'; + +import { PackagerTaskScheduler, setupPackagerTask } from './task'; + +export const getMockPackagerTaskScheduler = ( + taskManagerStart: TaskManagerStartContract +): PackagerTaskScheduler => { + const packagerTask = setupPackagerTask({ + endpointAppContext: createMockEndpointAppContext(), + taskManager: taskManagerMock.createSetup(), + }); + return packagerTask.getTaskScheduler({ taskManager: taskManagerStart }); +}; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.test.ts index 41bc2aa258807..505c1eefd3a66 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.test.ts @@ -3,3 +3,43 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ + +import { taskManagerMock } from '../../../../../task_manager/server/mocks'; + +import { createMockEndpointAppContext } from '../../mocks'; + +import { getMockPackagerTaskScheduler } from './task.mock'; + +import { setupPackagerTask } from './task'; + +describe('task', () => { + describe('Periodic task sanity checks', () => { + test('setupPackagerTask runs and returns task scheduler', () => { + const packagerTask = setupPackagerTask({ + endpointAppContext: createMockEndpointAppContext(), + taskManager: taskManagerMock.createSetup(), + }); + expect(packagerTask).toHaveProperty('getTaskScheduler'); + }); + + test('task should be registered', () => { + const mockTaskManager = taskManagerMock.createSetup(); + const packagerTask = setupPackagerTask({ + endpointAppContext: createMockEndpointAppContext(), + taskManager: mockTaskManager, + }); + expect(mockTaskManager.registerTaskDefinitions).toHaveBeenCalled(); + }); + + test('task should be scheduled', async () => { + const mockTaskManager = taskManagerMock.createStart(); + const taskScheduler = getMockPackagerTaskScheduler(mockTaskManager); + await taskScheduler.run(); + expect(mockTaskManager.ensureScheduled).toHaveBeenCalled(); + }); + + test('task should run', async () => { + // TODO + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts index fea3ffa585232..de829f9329c04 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import { Logger } from '../../../../../../../src/kibana/server'; import { ConcreteTaskInstance, TaskManagerSetupContract, @@ -18,10 +19,10 @@ const PackagerTaskConstants = { }; export interface PackagerTask { - getTaskRunner: (context: PackagerTaskRunnerContext) => PackagerTaskRunner; + getTaskScheduler: (context: PackagerTaskSchedulerContext) => PackagerTaskScheduler; } -interface PackagerTaskRunner { +interface PackagerTaskScheduler { run: () => void; } @@ -30,55 +31,62 @@ interface PackagerTaskContext { taskManager: TaskManagerSetupContract; } -interface PackagerTaskRunnerContext { +interface PackagerTaskSchedulerContext { taskManager: TaskManagerStartContract; } -export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { - const getTaskId = (): string => { - return `${PackagerTaskConstants.TYPE}:${PackagerTaskConstants.VERSION}`; - }; +interface PackagerTaskRunnerContext extends PackagerTaskContext { + logger: Logger; + taskId: string; +} - const logger = context.endpointAppContext.logFactory.get(getTaskId()); +const getTaskId = (): string => { + return `${PackagerTaskConstants.TYPE}:${PackagerTaskConstants.VERSION}`; +}; - const run = async (taskId: string) => { - // Check that this task is current - if (taskId !== getTaskId()) { - // old task, return - logger.debug(`Outdated task running: ${taskId}`); - return; - } - - const manifestManager = context.endpointAppContext.service.getManifestManager(); - - if (manifestManager === undefined) { - logger.debug('Manifest Manager not available.'); - return; - } - - manifestManager - .refresh() - .then((wrappedManifest) => { - if (wrappedManifest !== null) { - return manifestManager.dispatch(wrappedManifest); - } - }) - .then((wrappedManifest) => { - if (wrappedManifest !== null) { - return manifestManager.commit(wrappedManifest); - } - }) - .catch((err) => { - logger.error(err); - }); - }; +export const runPackagerTask = async (context: PackagerTaskRunnerContext) => { + // Check that this task is current + if (context.taskId !== getTaskId()) { + // old task, return + context.logger.debug(`Outdated task running: ${context.taskId}`); + return; + } + + const manifestManager = context.endpointAppContext.service.getManifestManager(); - const getTaskRunner = (runnerContext: PackagerTaskRunnerContext): PackagerTaskRunner => { + if (manifestManager === undefined) { + context.logger.debug('Manifest Manager not available.'); + return; + } + + manifestManager + .refresh() + .then((wrappedManifest) => { + if (wrappedManifest !== null) { + return manifestManager.dispatch(wrappedManifest); + } + }) + .then((wrappedManifest) => { + if (wrappedManifest !== null) { + return manifestManager.commit(wrappedManifest); + } + }) + .catch((err) => { + context.logger.error(err); + }); +}; + +export const setupPackagerTask = (context: PackagerTaskContext): PackagerTask => { + const logger = context.endpointAppContext.logFactory.get(getTaskId()); + const taskId = getTaskId(); + + const getTaskScheduler = ( + schedulerContext: PackagerTaskSchedulerContext + ): PackagerTaskScheduler => { return { run: async () => { - const taskId = getTaskId(); try { - await runnerContext.taskManager.ensureScheduled({ + await schedulerContext.taskManager.ensureScheduled({ id: taskId, taskType: PackagerTaskConstants.TYPE, scope: ['securitySolution'], @@ -104,7 +112,7 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { createTaskRunner: ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => { return { run: async () => { - await run(taskInstance.id); + await runPackagerTask({ ...context, logger, taskId: taskInstance.id }); }, cancel: async () => {}, }; @@ -113,6 +121,6 @@ export function setupPackagerTask(context: PackagerTaskContext): PackagerTask { }); return { - getTaskRunner, + getTaskScheduler, }; -} +}; diff --git a/x-pack/plugins/security_solution/server/endpoint/mocks.ts b/x-pack/plugins/security_solution/server/endpoint/mocks.ts index c9c1e64f301f6..fe9d3366d398a 100644 --- a/x-pack/plugins/security_solution/server/endpoint/mocks.ts +++ b/x-pack/plugins/security_solution/server/endpoint/mocks.ts @@ -5,7 +5,7 @@ */ import { IScopedClusterClient, SavedObjectsClientContract } from 'kibana/server'; -import { savedObjectsServiceMock } from 'src/core/server/mocks'; +import { loggingSystemMock, savedObjectsServiceMock } from 'src/core/server/mocks'; import { xpackMocks } from '../../../../mocks'; import { @@ -13,12 +13,24 @@ import { IngestManagerStartContract, ExternalCallback, } from '../../../ingest_manager/server'; -import { EndpointAppContextServiceStartContract } from './endpoint_app_context_services'; import { createDatasourceServiceMock } from '../../../ingest_manager/server/mocks'; +import { createMockConfig } from '../lib/detection_engine/routes/__mocks__'; +import { EndpointAppContextServiceStartContract } from './endpoint_app_context_services'; import { getManifestManagerMock } from './services/artifacts/manifest_manager/manifest_manager.mock'; /** - * Crates a mocked input contract for the `EndpointAppContextService#start()` method + * Creates a mocked EndpointAppContext. + */ +export const createMockEndpointAppContext = () => { + return { + logFactory: loggingSystemMock.create(), + config: createMockConfig(), + service: createMockEndpointAppContextServiceStartContract(), + }; +}; + +/** + * Creates a mocked input contract for the `EndpointAppContextService#start()` method */ export const createMockEndpointAppContextServiceStartContract = (): jest.Mocked< EndpointAppContextServiceStartContract diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts index 8f4304de14c8c..21098e3226353 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts @@ -71,7 +71,6 @@ export function registerDownloadExceptionListRoute( } }; - const id = `${req.params.identifier}-${req.params.sha256}`; const cacheResp = cache.get(id); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts index 41bc2aa258807..3bd9c3e3dd66c 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts @@ -3,3 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ + +describe('artifact_client', () => { + describe('ArtifactClient sanity checks', () => {}); +}); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.test.ts index 41bc2aa258807..867b23617ff46 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.test.ts @@ -3,3 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ + +describe('manifest_client', () => { + describe('ManifestClient sanity checks', () => {}); +}); diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index b058eac73d093..ebedbbe4cbd22 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -266,7 +266,7 @@ export class Plugin implements IPlugin Date: Sat, 27 Jun 2020 22:47:22 -0400 Subject: [PATCH 090/106] Add task tests --- .../endpoint/lib/artifacts/task.mock.ts | 14 +- .../endpoint/lib/artifacts/task.test.ts | 53 ++++-- .../server/endpoint/lib/artifacts/task.ts | 170 ++++++++---------- .../server/endpoint/mocks.ts | 29 ++- .../manifest_manager/manifest_manager.mock.ts | 1 + .../security_solution/server/plugin.ts | 18 +- 6 files changed, 155 insertions(+), 130 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.mock.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.mock.ts index bb79f2ffc892d..409d48c3dbfd2 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.mock.ts @@ -10,14 +10,8 @@ import { taskManagerMock } from '../../../../../task_manager/server/mocks'; import { createMockEndpointAppContext } from '../../mocks'; -import { PackagerTaskScheduler, setupPackagerTask } from './task'; +import { ManifestTask } from './task'; -export const getMockPackagerTaskScheduler = ( - taskManagerStart: TaskManagerStartContract -): PackagerTaskScheduler => { - const packagerTask = setupPackagerTask({ - endpointAppContext: createMockEndpointAppContext(), - taskManager: taskManagerMock.createSetup(), - }); - return packagerTask.getTaskScheduler({ taskManager: taskManagerStart }); -}; +export class MockManifestTask extends ManifestTask { + private runTask = jest.fn(); +} diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.test.ts index 505c1eefd3a66..15200cacaacf7 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.test.ts @@ -5,26 +5,27 @@ */ import { taskManagerMock } from '../../../../../task_manager/server/mocks'; +import { TaskStatus } from '../../../../../task_manager/server'; import { createMockEndpointAppContext } from '../../mocks'; +import { getManifestManagerMock } from '../../services/artifacts/manifest_manager/manifest_manager.mock'; -import { getMockPackagerTaskScheduler } from './task.mock'; - -import { setupPackagerTask } from './task'; +import { ManifestTaskConstants, ManifestTask } from './task'; +import { MockManifestTask } from './task.mock'; describe('task', () => { describe('Periodic task sanity checks', () => { - test('setupPackagerTask runs and returns task scheduler', () => { - const packagerTask = setupPackagerTask({ + test('can create task', () => { + const manifestTask = new ManifestTask({ endpointAppContext: createMockEndpointAppContext(), taskManager: taskManagerMock.createSetup(), }); - expect(packagerTask).toHaveProperty('getTaskScheduler'); + expect(manifestTask).toBeInstanceOf(ManifestTask); }); test('task should be registered', () => { const mockTaskManager = taskManagerMock.createSetup(); - const packagerTask = setupPackagerTask({ + const manifestTask = new ManifestTask({ endpointAppContext: createMockEndpointAppContext(), taskManager: mockTaskManager, }); @@ -32,14 +33,42 @@ describe('task', () => { }); test('task should be scheduled', async () => { - const mockTaskManager = taskManagerMock.createStart(); - const taskScheduler = getMockPackagerTaskScheduler(mockTaskManager); - await taskScheduler.run(); - expect(mockTaskManager.ensureScheduled).toHaveBeenCalled(); + const mockTaskManagerSetup = taskManagerMock.createSetup(); + const manifestTask = new ManifestTask({ + endpointAppContext: createMockEndpointAppContext(), + taskManager: mockTaskManagerSetup, + }); + const mockTaskManagerStart = taskManagerMock.createStart(); + manifestTask.start({ taskManager: mockTaskManagerStart }); + expect(mockTaskManagerStart.ensureScheduled).toHaveBeenCalled(); }); test('task should run', async () => { - // TODO + const mockContext = createMockEndpointAppContext(); + const mockTaskManager = taskManagerMock.createSetup(); + const mockManifestTask = new MockManifestTask({ + endpointAppContext: mockContext, + taskManager: mockTaskManager, + }); + const mockTaskInstance = { + id: ManifestTaskConstants.TYPE, + runAt: new Date(), + attempts: 0, + ownerId: '', + status: TaskStatus.Running, + startedAt: new Date(), + scheduledAt: new Date(), + retryAt: new Date(), + params: {}, + state: {}, + taskType: ManifestTaskConstants.TYPE, + }; + const createTaskRunner = + mockTaskManager.registerTaskDefinitions.mock.calls[0][0][ManifestTaskConstants.TYPE] + .createTaskRunner; + const taskRunner = createTaskRunner({ taskInstance: mockTaskInstance }); + await taskRunner.run(); + expect(mockManifestTask.runTask).toHaveBeenCalled(); }); }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts index de829f9329c04..95c849ebe4e00 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts @@ -12,115 +12,97 @@ import { } from '../../../../../../plugins/task_manager/server'; import { EndpointAppContext } from '../../types'; -const PackagerTaskConstants = { +export const ManifestTaskConstants = { TIMEOUT: '1m', TYPE: 'securitySolution:endpoint:exceptions-packager', VERSION: '1.0.0', }; -export interface PackagerTask { - getTaskScheduler: (context: PackagerTaskSchedulerContext) => PackagerTaskScheduler; -} - -interface PackagerTaskScheduler { - run: () => void; -} - -interface PackagerTaskContext { +export interface ManifestTaskSetupContract { endpointAppContext: EndpointAppContext; taskManager: TaskManagerSetupContract; } -interface PackagerTaskSchedulerContext { +export interface ManifestTaskStartContract { taskManager: TaskManagerStartContract; } -interface PackagerTaskRunnerContext extends PackagerTaskContext { - logger: Logger; - taskId: string; -} - -const getTaskId = (): string => { - return `${PackagerTaskConstants.TYPE}:${PackagerTaskConstants.VERSION}`; -}; - -export const runPackagerTask = async (context: PackagerTaskRunnerContext) => { - // Check that this task is current - if (context.taskId !== getTaskId()) { - // old task, return - context.logger.debug(`Outdated task running: ${context.taskId}`); - return; - } - - const manifestManager = context.endpointAppContext.service.getManifestManager(); - - if (manifestManager === undefined) { - context.logger.debug('Manifest Manager not available.'); - return; - } - - manifestManager - .refresh() - .then((wrappedManifest) => { - if (wrappedManifest !== null) { - return manifestManager.dispatch(wrappedManifest); - } - }) - .then((wrappedManifest) => { - if (wrappedManifest !== null) { - return manifestManager.commit(wrappedManifest); - } - }) - .catch((err) => { - context.logger.error(err); - }); -}; - -export const setupPackagerTask = (context: PackagerTaskContext): PackagerTask => { - const logger = context.endpointAppContext.logFactory.get(getTaskId()); - const taskId = getTaskId(); - - const getTaskScheduler = ( - schedulerContext: PackagerTaskSchedulerContext - ): PackagerTaskScheduler => { - return { - run: async () => { - try { - await schedulerContext.taskManager.ensureScheduled({ - id: taskId, - taskType: PackagerTaskConstants.TYPE, - scope: ['securitySolution'], - schedule: { - // TODO: change this to '60s' before merging - interval: '5s', +export class ManifestTask { + private endpointAppContext: EndpointAppContext; + private logger: Logger; + + constructor(setupContract: ManifestTaskSetupContract) { + this.endpointAppContext = setupContract.endpointAppContext; + this.logger = this.endpointAppContext.logFactory.get(this.getTaskId()); + + setupContract.taskManager.registerTaskDefinitions({ + [ManifestTaskConstants.TYPE]: { + title: 'Security Solution Endpoint Exceptions Handler', + type: ManifestTaskConstants.TYPE, + timeout: ManifestTaskConstants.TIMEOUT, + createTaskRunner: ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => { + return { + run: async () => { + await this.runTask(taskInstance.id); }, - state: {}, - params: { version: PackagerTaskConstants.VERSION }, - }); - } catch (e) { - logger.debug(`Error scheduling task, received ${e.message}`); - } + cancel: async () => {}, + }; + }, }, - }; + }); + } + + public start = async (startContract: ManifestTaskStartContract) => { + try { + await startContract.taskManager.ensureScheduled({ + id: this.getTaskId(), + taskType: ManifestTaskConstants.TYPE, + scope: ['securitySolution'], + schedule: { + // TODO: change this to '60s' before merging + interval: '5s', + }, + state: {}, + params: { version: ManifestTaskConstants.VERSION }, + }); + } catch (e) { + this.logger.debug(`Error scheduling task, received ${e.message}`); + } }; - context.taskManager.registerTaskDefinitions({ - [PackagerTaskConstants.TYPE]: { - title: 'Security Solution Endpoint Exceptions Handler', - type: PackagerTaskConstants.TYPE, - timeout: PackagerTaskConstants.TIMEOUT, - createTaskRunner: ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => { - return { - run: async () => { - await runPackagerTask({ ...context, logger, taskId: taskInstance.id }); - }, - cancel: async () => {}, - }; - }, - }, - }); + private getTaskId = (): string => { + return `${ManifestTaskConstants.TYPE}:${ManifestTaskConstants.VERSION}`; + }; - return { - getTaskScheduler, + private runTask = async (taskId: string) => { + // Check that this task is current + if (taskId !== this.getTaskId()) { + // old task, return + this.logger.debug(`Outdated task running: ${taskId}`); + return; + } + + const manifestManager = this.endpointAppContext.service.getManifestManager(); + + if (manifestManager === undefined) { + this.logger.debug('Manifest Manager not available.'); + return; + } + + manifestManager + .refresh() + .then((wrappedManifest) => { + if (wrappedManifest !== null) { + return manifestManager.dispatch(wrappedManifest); + } + }) + .then((wrappedManifest) => { + if (wrappedManifest !== null) { + return manifestManager.commit(wrappedManifest); + } + }) + .catch((err) => { + this.logger.error(err); + }); }; -}; +} diff --git a/x-pack/plugins/security_solution/server/endpoint/mocks.ts b/x-pack/plugins/security_solution/server/endpoint/mocks.ts index fe9d3366d398a..de81eae059556 100644 --- a/x-pack/plugins/security_solution/server/endpoint/mocks.ts +++ b/x-pack/plugins/security_solution/server/endpoint/mocks.ts @@ -15,17 +15,38 @@ import { } from '../../../ingest_manager/server'; import { createDatasourceServiceMock } from '../../../ingest_manager/server/mocks'; import { createMockConfig } from '../lib/detection_engine/routes/__mocks__'; -import { EndpointAppContextServiceStartContract } from './endpoint_app_context_services'; -import { getManifestManagerMock } from './services/artifacts/manifest_manager/manifest_manager.mock'; +import { + EndpointAppContextService, + EndpointAppContextServiceStartContract, +} from './endpoint_app_context_services'; +import { + ManifestManagerMock, + getManifestManagerMock, +} from './services/artifacts/manifest_manager/manifest_manager.mock'; /** * Creates a mocked EndpointAppContext. */ -export const createMockEndpointAppContext = () => { +export const createMockEndpointAppContext = (mockManifestManager?: ManifestManagerMock) => { return { logFactory: loggingSystemMock.create(), config: createMockConfig(), - service: createMockEndpointAppContextServiceStartContract(), + service: createMockEndpointAppContextService(mockManifestManager), + }; +}; + +/** + * Creates a mocked EndpointAppContextService + */ +export const createMockEndpointAppContextService = ( + mockManifestManager?: ManifestManagerMock +): jest.Mocked => { + return { + start: jest.fn(), + stop: jest.fn(), + getAgentService: jest.fn(), + getManifestManager: mockManifestManager ?? jest.fn(), + getScopedSavedObjects: jest.fn(), }; }; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts index d066a615263ec..8ea1a9a793e8e 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts @@ -70,6 +70,7 @@ export class ManifestManagerMock extends ManifestManager { .fn() .mockResolvedValue(new Manifest(new Date(), '1.0.0')); private getManifestClient = jest.fn().mockReturnValue(getManifestClientMock()); + public refresh = jest.fn().mockResolvedValue(null); } export const getManifestManagerMock = (): ManifestManagerMock => { diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index ebedbbe4cbd22..bcdbc82ac6a1d 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -34,7 +34,7 @@ import { signalRulesAlertType } from './lib/detection_engine/signals/signal_rule import { rulesNotificationAlertType } from './lib/detection_engine/notifications/rules_notification_alert_type'; import { isNotificationAlertExecutor } from './lib/detection_engine/notifications/types'; import { hasListsFeature, listsEnvFeatureFlagName } from './lib/detection_engine/feature_flags'; -import { PackagerTask, setupPackagerTask, ExceptionsCache } from './endpoint/lib/artifacts'; +import { ManifestTask, ExceptionsCache } from './endpoint/lib/artifacts'; import { initSavedObjects, savedObjectTypes } from './saved_objects'; import { AppClientFactory } from './client'; import { createConfig$, ConfigType } from './config'; @@ -80,7 +80,7 @@ export class Plugin implements IPlugin Date: Sun, 28 Jun 2020 14:39:33 -0400 Subject: [PATCH 091/106] Tests for artifact_client and manifest_client --- .../endpoint/lib/artifacts/lists.test.ts | 2 +- .../artifacts/artifact_client.mock.ts | 16 +++-- .../artifacts/artifact_client.test.ts | 42 ++++++++++++- .../services/artifacts/artifact_client.ts | 6 +- .../artifacts/manifest_client.mock.ts | 17 +++--- .../artifacts/manifest_client.test.ts | 61 ++++++++++++++++++- .../services/artifacts/manifest_client.ts | 23 +++++-- .../manifest_manager/manifest_manager.mock.ts | 2 +- .../manifest_manager/manifest_manager.test.ts | 6 +- 9 files changed, 146 insertions(+), 29 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts index 1804f1c012537..8af4e6eb5abf5 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts @@ -8,8 +8,8 @@ import { ExceptionListClient } from '../../../../../lists/server'; import { listMock } from '../../../../../lists/server/mocks'; import { getFoundExceptionListItemSchemaMock } from '../../../../../lists/common/schemas/response/found_exception_list_item_schema.mock'; import { getExceptionListItemSchemaMock } from '../../../../../lists/common/schemas/response/exception_list_item_schema.mock'; -import { getFullEndpointExceptionList } from './lists'; import { EntriesArray } from '../../../../../lists/common/schemas/types/entries'; +import { getFullEndpointExceptionList } from './lists'; describe('buildEventTypeSignal', () => { let mockExceptionClient: ExceptionListClient; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.mock.ts index d7a7f38473a04..5e13ef54a478f 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.mock.ts @@ -7,14 +7,12 @@ import { savedObjectsClientMock } from '../../../../../../../src/core/server/mocks'; import { ArtifactClient } from './artifact_client'; -import { getInternalArtifactMock } from '../../schemas/artifacts/saved_objects.mock'; -export class ArtifactClientMock extends ArtifactClient { - public getArtifact = jest.fn().mockResolvedValue(getInternalArtifactMock()); - public createArtifact = jest.fn().mockResolvedValue(getInternalArtifactMock()); - public deleteArtifact = jest.fn().mockResolvedValue(getInternalArtifactMock()); -} - -export const getArtifactClientMock = (): ArtifactClientMock => { - return new ArtifactClientMock(savedObjectsClientMock.create()); +export const getArtifactClientMock = ( + savedObjectsClient?: typeof savedObjectsClientMock.create +): ArtifactClient => { + if (savedObjectsClient !== undefined) { + return new ArtifactClient(savedObjectsClient); + } + return new ArtifactClient(savedObjectsClientMock.create()); }; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts index 3bd9c3e3dd66c..f942485aec7d4 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts @@ -4,6 +4,46 @@ * you may not use this file except in compliance with the Elastic License. */ +import { savedObjectsClientMock } from '../../../../../../../src/core/server/mocks'; +import { ArtifactConstants } from '../../lib/artifacts'; +import { getInternalArtifactMock } from '../../schemas/artifacts/saved_objects.mock'; +import { getArtifactClientMock } from './artifact_client.mock'; +import { ArtifactClient } from './artifact_client'; + describe('artifact_client', () => { - describe('ArtifactClient sanity checks', () => {}); + describe('ArtifactClient sanity checks', () => { + test('can create ArtifactClient', () => { + const artifactClient = new ArtifactClient(savedObjectsClientMock.create()); + expect(artifactClient).toBeInstanceOf(ArtifactClient); + }); + + test('can get artifact', async () => { + const savedObjectsClient = savedObjectsClientMock.create(); + const artifactClient = getArtifactClientMock(savedObjectsClient); + await artifactClient.getArtifact('abcd'); + expect(savedObjectsClient.get).toHaveBeenCalled(); + }); + + test('can create artifact', async () => { + const savedObjectsClient = savedObjectsClientMock.create(); + const artifactClient = getArtifactClientMock(savedObjectsClient); + const artifact = await getInternalArtifactMock('linux', '1.0.0'); + await artifactClient.createArtifact(artifact); + expect(savedObjectsClient.create).toHaveBeenCalledWith( + ArtifactConstants.SAVED_OBJECT_TYPE, + artifact, + { id: artifactClient.getArtifactId(artifact) } + ); + }); + + test('can delete artifact', async () => { + const savedObjectsClient = savedObjectsClientMock.create(); + const artifactClient = getArtifactClientMock(savedObjectsClient); + await artifactClient.deleteArtifact('abcd'); + expect(savedObjectsClient.delete).toHaveBeenCalledWith( + ArtifactConstants.SAVED_OBJECT_TYPE, + 'abcd' + ); + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts index 1313adb916d86..47c76d643b018 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts @@ -15,6 +15,10 @@ export class ArtifactClient { this.savedObjectsClient = savedObjectsClient; } + public getArtifactId(artifact: InternalArtifactSchema) { + return `${artifact.identifier}-${artifact.sha256}`; + } + public async getArtifact(id: string): Promise> { return this.savedObjectsClient.get( ArtifactConstants.SAVED_OBJECT_TYPE, @@ -28,7 +32,7 @@ export class ArtifactClient { return this.savedObjectsClient.create( ArtifactConstants.SAVED_OBJECT_TYPE, artifact, - { id: `${artifact.identifier}-${artifact.sha256}` } + { id: this.getArtifactId(artifact) } ); } diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.mock.ts index 61445cb407de2..51c9122dfde24 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.mock.ts @@ -7,15 +7,12 @@ import { savedObjectsClientMock } from '../../../../../../../src/core/server/mocks'; import { ManifestClient } from './manifest_client'; -import { getInternalManifestMock } from '../../schemas/artifacts/saved_objects.mock'; -export class ManifestClientMock extends ManifestClient { - public createManifest = jest.fn().mockResolvedValue(getInternalManifestMock()); - public getManifest = jest.fn().mockResolvedValue(getInternalManifestMock()); - public updateManifest = jest.fn().mockResolvedValue(getInternalManifestMock()); - public deleteManifest = jest.fn().mockResolvedValue(getInternalManifestMock()); -} - -export const getManifestClientMock = (): ManifestClientMock => { - return new ManifestClientMock(savedObjectsClientMock.create(), '1.0.0'); +export const getManifestClientMock = ( + savedObjectsClient?: typeof savedObjectsClientMock.create +): ManifestClient => { + if (savedObjectsClient !== undefined) { + return new ManifestClient(savedObjectsClient, '1.0.0'); + } + return new ManifestClient(savedObjectsClientMock.create()); }; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.test.ts index 867b23617ff46..7bdf567f06e8e 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.test.ts @@ -4,6 +4,65 @@ * you may not use this file except in compliance with the Elastic License. */ +import { savedObjectsClientMock } from '../../../../../../../src/core/server/mocks'; +import { ManifestConstants } from '../../lib/artifacts'; +import { getInternalManifestMock } from '../../schemas/artifacts/saved_objects.mock'; +import { getManifestClientMock } from './manifest_client.mock'; +import { ManifestClient } from './manifest_client'; + describe('manifest_client', () => { - describe('ManifestClient sanity checks', () => {}); + describe('ManifestClient sanity checks', () => { + test('can create ManifestClient', () => { + const manifestClient = new ManifestClient(savedObjectsClientMock.create(), '1.0.0'); + expect(manifestClient).toBeInstanceOf(ManifestClient); + }); + + test('cannot create ManifestClient with invalid schema version', () => { + expect(() => { + new ManifestClient(savedObjectsClientMock.create(), 'invalid'); + }).toThrow(); + }); + + test('can get manifest', async () => { + const savedObjectsClient = savedObjectsClientMock.create(); + const manifestClient = getManifestClientMock(savedObjectsClient); + await manifestClient.getManifest(); + expect(savedObjectsClient.get).toHaveBeenCalled(); + }); + + test('can create manifest', async () => { + const savedObjectsClient = savedObjectsClientMock.create(); + const manifestClient = getManifestClientMock(savedObjectsClient); + const manifest = getInternalManifestMock(); + await manifestClient.createManifest(manifest); + expect(savedObjectsClient.create).toHaveBeenCalledWith( + ManifestConstants.SAVED_OBJECT_TYPE, + manifest, + { id: manifestClient.getManifestId() } + ); + }); + + test('can update manifest', async () => { + const savedObjectsClient = savedObjectsClientMock.create(); + const manifestClient = getManifestClientMock(savedObjectsClient); + const manifest = getInternalManifestMock(); + await manifestClient.updateManifest(manifest, { version: 'abcd' }); + expect(savedObjectsClient.update).toHaveBeenCalledWith( + ManifestConstants.SAVED_OBJECT_TYPE, + manifestClient.getManifestId(), + manifest, + { version: 'abcd' } + ); + }); + + test('can delete manifest', async () => { + const savedObjectsClient = savedObjectsClientMock.create(); + const manifestClient = getManifestClientMock(savedObjectsClient); + await manifestClient.deleteManifest(); + expect(savedObjectsClient.delete).toHaveBeenCalledWith( + ManifestConstants.SAVED_OBJECT_TYPE, + manifestClient.getManifestId() + ); + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.ts index 8641963caebbf..766d710a687b1 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.ts @@ -9,6 +9,11 @@ import { SavedObjectsClient, SavedObjectsUpdateResponse, } from '../../../../../../../src/core/server'; +import { validate } from '../../../../common/validate'; +import { + manifestSchemaVersion, + ManifestSchemaVersion, +} from '../../../../common/endpoint/schema/common'; import { ManifestConstants } from '../../lib/artifacts'; import { InternalManifestSchema } from '../../schemas/artifacts'; @@ -17,15 +22,25 @@ interface UpdateManifestOpts { } export class ManifestClient { - private schemaVersion: string; + private schemaVersion: ManifestSchemaVersion; private savedObjectsClient: SavedObjectsClient; - constructor(savedObjectsClient: SavedObjectsClient, schemaVersion: string) { + constructor(savedObjectsClient: SavedObjectsClient, schemaVersion: ManifestSchemaVersion) { this.savedObjectsClient = savedObjectsClient; - this.schemaVersion = schemaVersion; + + const [validated, errors] = validate( + (schemaVersion as unknown) as object, + manifestSchemaVersion + ); + + if (errors != null) { + throw new Error(`Invalid manifest version: ${schemaVersion}`); + } + + this.schemaVersion = validated; } - private getManifestId(): string { + public getManifestId(): string { return `endpoint-manifest-${this.schemaVersion}`; } diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts index 8ea1a9a793e8e..85c642ef04968 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts @@ -70,7 +70,7 @@ export class ManifestManagerMock extends ManifestManager { .fn() .mockResolvedValue(new Manifest(new Date(), '1.0.0')); private getManifestClient = jest.fn().mockReturnValue(getManifestClientMock()); - public refresh = jest.fn().mockResolvedValue(null); + // public refresh = jest.fn().mockResolvedValue(null); } export const getManifestManagerMock = (): ManifestManagerMock => { diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts index 08ed343995411..34cc507ac07c6 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts @@ -11,6 +11,10 @@ describe('manifest_manager', () => { describe('ManifestManager sanity checks', () => { beforeAll(async () => {}); - test('Can do a test', () => {}); + test('ManifestManager can refresh manifest', () => {}); + + test('ManifestManager can dispatch manifest', () => {}); + + test('ManifestManager can commit manifest', () => {}); }); }); From 2c8a53617cbb9459ec9da7002f5256d7d695f1af Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Sun, 28 Jun 2020 23:16:00 -0400 Subject: [PATCH 092/106] Add manifest_manager tests --- .../artifacts/manifest_client.mock.ts | 2 +- .../manifest_manager/manifest_manager.mock.ts | 65 ++++++++++++++---- .../manifest_manager/manifest_manager.test.ts | 68 +++++++++++++++++-- 3 files changed, 116 insertions(+), 19 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.mock.ts index 51c9122dfde24..8029d328c8229 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.mock.ts @@ -14,5 +14,5 @@ export const getManifestClientMock = ( if (savedObjectsClient !== undefined) { return new ManifestClient(savedObjectsClient, '1.0.0'); } - return new ManifestClient(savedObjectsClientMock.create()); + return new ManifestClient(savedObjectsClientMock.create(), '1.0.0'); }; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts index 85c642ef04968..028ec6d2c2303 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts @@ -12,9 +12,15 @@ import { } from '../../../../../../../../src/core/server/mocks'; import { DatasourceServiceInterface } from '../../../../../../ingest_manager/server'; +import { getFoundExceptionListItemSchemaMock } from '../../../../../../lists/common/schemas/response/found_exception_list_item_schema.mock'; import { listMock } from '../../../../../../lists/server/mocks'; -import { Manifest } from '../../../lib/artifacts'; +import { + ExceptionsCache, + Manifest, + buildArtifact, + getFullEndpointExceptionList, +} from '../../../lib/artifacts'; import { getInternalArtifactMock, getInternalArtifactsMock } from '../../../schemas'; import { getArtifactClientMock } from '../artifact_client.mock'; @@ -25,7 +31,11 @@ import { ManifestManager } from './manifest_manager'; function getMockDatasource() { return { id: 'c6d16e42-c32d-4dce-8a88-113cfe276ad1', - inputs: [{}], + inputs: [ + { + config: {}, + }, + ], revision: 1, version: 'abcd', // TODO: not yet implemented in ingest_manager (https://github.com/elastic/kibana/issues/69992) updated_at: '2020-06-25T16:03:38.159292', @@ -50,34 +60,61 @@ class DatasourceServiceMock { public update = jest.fn().mockResolvedValue(getMockDatasource()); } -function getDatasourceServiceMock() { +export function getDatasourceServiceMock() { return new DatasourceServiceMock(); } -function mockBuildExceptionListArtifacts() { - // mock buildArtifactFunction - // pass in OS, mock implementation of ExceptionListItemSchemaMock more than once - // getInternalArtifactsMock() +async function mockBuildExceptionListArtifacts( + os: string, + schemaVersion: string +): InternalArtifactSchema[] { + const mockExceptionClient = listMock.getExceptionListClient(); + const first = getFoundExceptionListItemSchemaMock(); + mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); + const exceptions = await getFullEndpointExceptionList(mockExceptionClient, os, schemaVersion); + return [await buildArtifact(exceptions, os, schemaVersion)]; } // TODO // eslint-disable-next-line max-classes-per-file export class ManifestManagerMock extends ManifestManager { - private buildExceptionListArtifacts = jest - .fn() - .mockResolvedValue(mockBuildExceptionListArtifacts()); + private buildExceptionListArtifacts = async () => { + return mockBuildExceptionListArtifacts('linux', '1.0.0'); + }; + private getLastDispatchedManifest = jest .fn() .mockResolvedValue(new Manifest(new Date(), '1.0.0')); + private getManifestClient = jest.fn().mockReturnValue(getManifestClientMock()); - // public refresh = jest.fn().mockResolvedValue(null); } -export const getManifestManagerMock = (): ManifestManagerMock => { +export const getManifestManagerMock = (opts: { + artifactClientMock?: ArtifactClient; + datasourceServiceMock?: DatasourceServiceMock; + manifestClientMock?: ManifestClient; +}): ManifestManagerMock => { + let artifactClient = getArtifactClientMock(); + if (opts?.artifactClientMock !== undefined) { + artifactClient = opts.artifactClientMock; + } + + let datasourceService = getDatasourceServiceMock(); + if (opts?.datasourceServiceMock !== undefined) { + datasourceService = opts.datasourceServiceMock; + } + + // TODO: use the manifestClient + let manifestClient = getManifestClientMock(); + if (opts?.manifestClientMock !== undefined) { + manifestClient = opts.manifestClientMock; + } + return new ManifestManagerMock({ - artifactClient: getArtifactClientMock(), + artifactClient, + cache: new ExceptionsCache(5), exceptionListClient: listMock.getExceptionListClient(), - datasourceService: getDatasourceServiceMock(), + datasourceService, savedObjectsClient: savedObjectsClientMock.create(), logger: loggingSystemMock.create().get() as jest.Mocked, }); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts index 34cc507ac07c6..ebc3380b98dc6 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts @@ -4,17 +4,77 @@ * you may not use this file except in compliance with the Elastic License. */ +import { savedObjectsClientMock } from '../../../../../../../../src/core/server/mocks'; +import { ArtifactConstants, ManifestConstants, Manifest } from '../../../lib/artifacts'; +import { getArtifactClientMock } from '../artifact_client.mock'; +import { getManifestClientMock } from '../manifest_client.mock'; import { ManifestManager } from './manifest_manager'; -import { getManifestManagerMock } from './manifest_manager.mock'; +import { getDatasourceServiceMock, getManifestManagerMock } from './manifest_manager.mock'; describe('manifest_manager', () => { describe('ManifestManager sanity checks', () => { beforeAll(async () => {}); - test('ManifestManager can refresh manifest', () => {}); + test('ManifestManager can refresh manifest', async () => { + const manifestManager = getManifestManagerMock(); + const manifestWrapper = await manifestManager.refresh(); + expect(manifestManager.getLastDispatchedManifest).toHaveBeenCalled(); + expect(manifestWrapper.diffs).toEqual([ + { + id: + 'endpoint-allowlist-linux-1.0.0-a0b2886af05849e1e7e7b05bd6e38ea2e2de6566bfb5f4bdbdeda8236de0ff5c', + type: 'add', + }, + ]); + expect(manifestWrapper.manifest).toBeInstanceOf(Manifest); + }); - test('ManifestManager can dispatch manifest', () => {}); + test('ManifestManager can dispatch manifest', async () => { + const datasourceService = getDatasourceServiceMock(); + const manifestManager = getManifestManagerMock({ datasourceServiceMock: datasourceService }); + const manifestWrapperRefresh = await manifestManager.refresh(); + const manifestWrapperDispatch = await manifestManager.dispatch(manifestWrapperRefresh); + expect(manifestWrapperRefresh).toEqual(manifestWrapperDispatch); + const entries = manifestWrapperDispatch.manifest.entries; + const artifact = Object.values(entries)[0].artifact; + expect(datasourceService.update.mock.calls[0][2].inputs[0].config.artifact_manifest).toEqual({ + manifestVersion: 'baseline', + schemaVersion: '1.0.0', + artifacts: { + [artifact.identifier]: { + sha256: artifact.sha256, + size: artifact.size, + url: `/api/endpoint/allowlist/download/${artifact.identifier}/${artifact.sha256}`, + }, + }, + }); + }); - test('ManifestManager can commit manifest', () => {}); + test('ManifestManager can commit manifest', async () => { + const savedObjectsClient = savedObjectsClientMock.create(); + const artifactClient = getArtifactClientMock(savedObjectsClient); + const manifestClient = getManifestClientMock(savedObjectsClient); + const manifestManager = getManifestManagerMock({ + artifactClientMock: artifactClient, + manifestClientMock: manifestClient, + }); + + const manifestWrapperRefresh = await manifestManager.refresh(); + const manifestWrapperDispatch = await manifestManager.dispatch(manifestWrapperRefresh); + const diff = { + id: 'abcd', + type: 'delete', + }; + manifestWrapperDispatch.diffs.push(diff); + + await manifestManager.commit(manifestWrapperDispatch); + + expect(savedObjectsClient.delete).toHaveBeenCalledWith( + ArtifactConstants.SAVED_OBJECT_TYPE, + 'abcd' + ); + + // TODO: check manifestClient.create/update + }); }); }); From da0b32f6453f0f946de0b404eadfbbfaf7291ec7 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Sun, 28 Jun 2020 23:50:10 -0400 Subject: [PATCH 093/106] minor refactor --- .../endpoint_app_context_services.test.ts | 4 +- .../endpoint/endpoint_app_context_services.ts | 2 +- .../server/endpoint/lib/index.ts | 5 --- .../server/endpoint/mocks.ts | 2 +- .../artifacts/download_exception_list.ts | 2 +- .../scripts/exceptions/check_env_variables.sh | 41 ------------------- .../scripts/exceptions/get_artifact_by_id.sh | 12 ------ .../server/scripts/exceptions/get_manifest.sh | 12 ------ 8 files changed, 5 insertions(+), 75 deletions(-) delete mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/index.ts delete mode 100644 x-pack/plugins/security_solution/server/scripts/exceptions/check_env_variables.sh delete mode 100644 x-pack/plugins/security_solution/server/scripts/exceptions/get_artifact_by_id.sh delete mode 100644 x-pack/plugins/security_solution/server/scripts/exceptions/get_manifest.sh diff --git a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts index 85c949ae0a7f6..ed1ccff90586d 100644 --- a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts @@ -15,10 +15,10 @@ describe('test endpoint app context services', () => { const endpointAppContextService = new EndpointAppContextService(); expect(endpointAppContextService.getManifestManager()).toEqual(undefined); }); - it('should return undefined on getScopedSavedObjects if start is not called', async () => { + it('should return undefined on getScopedSavedObjectsClient if start is not called', async () => { const endpointAppContextService = new EndpointAppContextService(); expect( - endpointAppContextService.getScopedSavedObjects(httpServerMock.createKibanaRequest()) + endpointAppContextService.getScopedSavedObjectsClient(httpServerMock.createKibanaRequest()) ).toEqual(undefined); }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts index 55d33e4923581..eff2a32738f23 100644 --- a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts +++ b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts @@ -49,7 +49,7 @@ export class EndpointAppContextService { return this.manifestManager; } - public getScopedSavedObjects(req: KibanaRequest): SavedObjectsClient | undefined { + public getScopedSavedObjectsClient(req: KibanaRequest): SavedObjectsClient | undefined { let client: SavedObjectsClient; if (this.savedObjectsStart !== undefined) { client = this.savedObjectsStart.getScopedClient(req, { excludedWrappers: ['security'] }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/index.ts b/x-pack/plugins/security_solution/server/endpoint/lib/index.ts deleted file mode 100644 index 41bc2aa258807..0000000000000 --- a/x-pack/plugins/security_solution/server/endpoint/lib/index.ts +++ /dev/null @@ -1,5 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ diff --git a/x-pack/plugins/security_solution/server/endpoint/mocks.ts b/x-pack/plugins/security_solution/server/endpoint/mocks.ts index de81eae059556..b6892ee5fb1b1 100644 --- a/x-pack/plugins/security_solution/server/endpoint/mocks.ts +++ b/x-pack/plugins/security_solution/server/endpoint/mocks.ts @@ -46,7 +46,7 @@ export const createMockEndpointAppContextService = ( stop: jest.fn(), getAgentService: jest.fn(), getManifestManager: mockManifestManager ?? jest.fn(), - getScopedSavedObjects: jest.fn(), + getScopedSavedObjectsClient: jest.fn(), }; }; diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts index 21098e3226353..404f1e37d1d82 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts @@ -45,7 +45,7 @@ export function registerDownloadExceptionListRoute( // The ApiKey must be associated with an enrolled Fleet agent try { - scopedSOClient = endpointContext.service.getScopedSavedObjects(req); + scopedSOClient = endpointContext.service.getScopedSavedObjectsClient(req); await authenticateAgentWithAccessToken(scopedSOClient, req); } catch (err) { if (err.output.statusCode === 401) { diff --git a/x-pack/plugins/security_solution/server/scripts/exceptions/check_env_variables.sh b/x-pack/plugins/security_solution/server/scripts/exceptions/check_env_variables.sh deleted file mode 100644 index 2f7644051debb..0000000000000 --- a/x-pack/plugins/security_solution/server/scripts/exceptions/check_env_variables.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/sh - -# -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License; -# you may not use this file except in compliance with the Elastic License. -# - -# Add this to the start of any scripts to detect if env variables are set - -set -e - -if [ -z "${ELASTICSEARCH_USERNAME}" ]; then - echo "Set ELASTICSEARCH_USERNAME in your environment" - exit 1 -fi - -if [ -z "${ELASTICSEARCH_PASSWORD}" ]; then - echo "Set ELASTICSEARCH_PASSWORD in your environment" - exit 1 -fi - -if [ -z "${ELASTICSEARCH_URL}" ]; then - echo "Set ELASTICSEARCH_URL in your environment" - exit 1 -fi - -if [ -z "${KIBANA_URL}" ]; then - echo "Set KIBANA_URL in your environment" - exit 1 -fi - -if [ -z "${TASK_MANAGER_INDEX}" ]; then - echo "Set TASK_MANAGER_INDEX in your environment" - exit 1 -fi - -if [ -z "${KIBANA_INDEX}" ]; then - echo "Set KIBANA_INDEX in your environment" - exit 1 -fi diff --git a/x-pack/plugins/security_solution/server/scripts/exceptions/get_artifact_by_id.sh b/x-pack/plugins/security_solution/server/scripts/exceptions/get_artifact_by_id.sh deleted file mode 100644 index bc012750dadcb..0000000000000 --- a/x-pack/plugins/security_solution/server/scripts/exceptions/get_artifact_by_id.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/sh - -# -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License; -# you may not use this file except in compliance with the Elastic License. -# - -set -e -./check_env_variables.sh - -# TODO diff --git a/x-pack/plugins/security_solution/server/scripts/exceptions/get_manifest.sh b/x-pack/plugins/security_solution/server/scripts/exceptions/get_manifest.sh deleted file mode 100644 index bc012750dadcb..0000000000000 --- a/x-pack/plugins/security_solution/server/scripts/exceptions/get_manifest.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/sh - -# -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License; -# you may not use this file except in compliance with the Elastic License. -# - -set -e -./check_env_variables.sh - -# TODO From 843f874c730540413fc7bacac0b383574cb26ee2 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Mon, 29 Jun 2020 00:16:11 -0400 Subject: [PATCH 094/106] Finish manifest_manager tests --- .../manifest_manager/manifest_manager.mock.ts | 36 ++++++++----------- .../manifest_manager/manifest_manager.test.ts | 16 ++++++--- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts index 028ec6d2c2303..804f2b4484718 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts @@ -5,11 +5,11 @@ */ // eslint-disable-next-line max-classes-per-file -import { Logger } from '../../../../../../../../src/core/server'; import { loggingSystemMock, savedObjectsClientMock, } from '../../../../../../../../src/core/server/mocks'; +import { Logger, SavedObjectsClient } from '../../../../../../../../src/core/server'; import { DatasourceServiceInterface } from '../../../../../../ingest_manager/server'; import { getFoundExceptionListItemSchemaMock } from '../../../../../../lists/common/schemas/response/found_exception_list_item_schema.mock'; @@ -25,6 +25,7 @@ import { getInternalArtifactMock, getInternalArtifactsMock } from '../../../sche import { getArtifactClientMock } from '../artifact_client.mock'; import { getManifestClientMock } from '../manifest_client.mock'; +import { ManifestClient } from '../manifest_client'; import { ManifestManager } from './manifest_manager'; @@ -45,8 +46,6 @@ function getMockDatasource() { }; } -// TODO -// eslint-disable-next-line max-classes-per-file class DatasourceServiceMock { public create = jest.fn().mockResolvedValue(getMockDatasource()); public get = jest.fn().mockResolvedValue(getMockDatasource()); @@ -75,8 +74,6 @@ async function mockBuildExceptionListArtifacts( return [await buildArtifact(exceptions, os, schemaVersion)]; } -// TODO -// eslint-disable-next-line max-classes-per-file export class ManifestManagerMock extends ManifestManager { private buildExceptionListArtifacts = async () => { return mockBuildExceptionListArtifacts('linux', '1.0.0'); @@ -86,36 +83,33 @@ export class ManifestManagerMock extends ManifestManager { .fn() .mockResolvedValue(new Manifest(new Date(), '1.0.0')); - private getManifestClient = jest.fn().mockReturnValue(getManifestClientMock()); + private getManifestClient = jest + .fn() + .mockReturnValue(getManifestClientMock(this.savedObjectsClient)); } export const getManifestManagerMock = (opts: { - artifactClientMock?: ArtifactClient; datasourceServiceMock?: DatasourceServiceMock; - manifestClientMock?: ManifestClient; + savedObjectsClientMock?: SavedObjectsClient; }): ManifestManagerMock => { - let artifactClient = getArtifactClientMock(); - if (opts?.artifactClientMock !== undefined) { - artifactClient = opts.artifactClientMock; - } - let datasourceService = getDatasourceServiceMock(); if (opts?.datasourceServiceMock !== undefined) { datasourceService = opts.datasourceServiceMock; } - // TODO: use the manifestClient - let manifestClient = getManifestClientMock(); - if (opts?.manifestClientMock !== undefined) { - manifestClient = opts.manifestClientMock; + let savedObjectsClient = savedObjectsClientMock.create(); + if (opts?.savedObjectsClientMock !== undefined) { + savedObjectsClient = opts.savedObjectsClientMock; } - return new ManifestManagerMock({ - artifactClient, + const manifestManager = new ManifestManagerMock({ + artifactClient: getArtifactClientMock(savedObjectsClient), cache: new ExceptionsCache(5), - exceptionListClient: listMock.getExceptionListClient(), datasourceService, - savedObjectsClient: savedObjectsClientMock.create(), + exceptionListClient: listMock.getExceptionListClient(), logger: loggingSystemMock.create().get() as jest.Mocked, + savedObjectsClient, }); + + return manifestManager; }; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts index ebc3380b98dc6..195c1746441bd 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts @@ -52,11 +52,8 @@ describe('manifest_manager', () => { test('ManifestManager can commit manifest', async () => { const savedObjectsClient = savedObjectsClientMock.create(); - const artifactClient = getArtifactClientMock(savedObjectsClient); - const manifestClient = getManifestClientMock(savedObjectsClient); const manifestManager = getManifestManagerMock({ - artifactClientMock: artifactClient, - manifestClientMock: manifestClient, + savedObjectsClientMock: savedObjectsClient, }); const manifestWrapperRefresh = await manifestManager.refresh(); @@ -69,12 +66,21 @@ describe('manifest_manager', () => { await manifestManager.commit(manifestWrapperDispatch); + // created new artifact + expect(savedObjectsClient.create.mock.calls[0][0]).toEqual( + ArtifactConstants.SAVED_OBJECT_TYPE + ); + + // deleted old artifact expect(savedObjectsClient.delete).toHaveBeenCalledWith( ArtifactConstants.SAVED_OBJECT_TYPE, 'abcd' ); - // TODO: check manifestClient.create/update + // committed new manifest + expect(savedObjectsClient.create.mock.calls[1][0]).toEqual( + ManifestConstants.SAVED_OBJECT_TYPE + ); }); }); }); From 0c46e88426d86b11c4f159649a22128440917bd1 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Mon, 29 Jun 2020 10:54:45 -0400 Subject: [PATCH 095/106] Type errors --- x-pack/plugins/lists/server/index.ts | 1 - .../security_solution/common/endpoint/generate_data.ts | 5 +++++ .../security_solution/common/endpoint/schema/manifest.ts | 4 ++-- .../pages/policy/store/policy_details/index.test.ts | 5 +++++ .../server/endpoint/lib/artifacts/manifest.test.ts | 4 ++-- .../server/endpoint/lib/artifacts/manifest.ts | 4 ++-- .../artifacts/manifest_manager/manifest_manager.test.ts | 4 ++-- 7 files changed, 18 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/lists/server/index.ts b/x-pack/plugins/lists/server/index.ts index d1b8ac939b153..8279095c6ffa8 100644 --- a/x-pack/plugins/lists/server/index.ts +++ b/x-pack/plugins/lists/server/index.ts @@ -12,7 +12,6 @@ import { ListPlugin } from './plugin'; // exporting these since its required at top level in siem plugin export { ExceptionListClient } from './services/exception_lists/exception_list_client'; export { ListClient } from './services/lists/list_client'; -export { ExceptionListClient } from './services/exception_lists/exception_list_client'; export { ListPluginSetup } from './types'; export const config = { schema: ConfigSchema }; diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts index 5af34b6a694e8..778edff507daa 100644 --- a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts @@ -975,6 +975,11 @@ export class EndpointDocGenerator { enabled: true, streams: [], config: { + artifact_manifest: { + manifest_version: 'baseline', + schema_version: '1.0.0', + artifacts: {}, + }, policy: { value: policyFactory(), }, diff --git a/x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts b/x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts index e1c5a989734b6..470e9b13ef78a 100644 --- a/x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts +++ b/x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts @@ -17,8 +17,8 @@ export const manifestEntrySchema = t.exact( export const manifestSchema = t.exact( t.type({ - manifestVersion, - schemaVersion: manifestSchemaVersion, + manifest_version: manifestVersion, + schema_version: manifestSchemaVersion, artifacts: t.record(identifier, manifestEntrySchema), }) ); diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts index 469b71854dfcc..aa451c021c6e4 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts +++ b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts @@ -41,6 +41,11 @@ describe('policy details: ', () => { enabled: true, streams: [], config: { + artifact_manifest: { + manifest_version: 'baseline', + schema_version: '1.0.0', + artifacts: {}, + }, policy: { value: policyConfigFactory(), }, diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts index b03a3e017dee1..f79e38b3c4d7d 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts @@ -72,8 +72,8 @@ describe('manifest', () => { '/api/endpoint/allowlist/download/endpoint-allowlist-windows-1.0.0/222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', }, }, - manifestVersion: 'abcd', - schemaVersion: '1.0.0', + manifest_version: 'abcd', + schema_version: '1.0.0', }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts index 0ed2bc954dbb0..08cb4ecd1ed4c 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts @@ -102,8 +102,8 @@ export class Manifest { public toEndpointFormat(): ManifestSchema { const manifestObj: ManifestSchema = { - manifestVersion: this.version ?? 'baseline', - schemaVersion: this.schemaVersion, + manifest_version: this.version ?? 'baseline', + schema_version: this.schemaVersion, artifacts: {}, }; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts index 195c1746441bd..35b332e30cbca 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts @@ -38,8 +38,8 @@ describe('manifest_manager', () => { const entries = manifestWrapperDispatch.manifest.entries; const artifact = Object.values(entries)[0].artifact; expect(datasourceService.update.mock.calls[0][2].inputs[0].config.artifact_manifest).toEqual({ - manifestVersion: 'baseline', - schemaVersion: '1.0.0', + manifest_version: 'baseline', + schema_version: '1.0.0', artifacts: { [artifact.identifier]: { sha256: artifact.sha256, From 3c09d7bbdb2986a6ed25032eec6e9b03c95bc14e Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Mon, 29 Jun 2020 17:52:17 -0400 Subject: [PATCH 096/106] Update integ test --- .../apis/security_solution/exceptions.ts | 53 ++++++++++++++++--- 1 file changed, 47 insertions(+), 6 deletions(-) diff --git a/x-pack/test/api_integration/apis/security_solution/exceptions.ts b/x-pack/test/api_integration/apis/security_solution/exceptions.ts index 9650318923d91..9a7e5c04dc5f0 100644 --- a/x-pack/test/api_integration/apis/security_solution/exceptions.ts +++ b/x-pack/test/api_integration/apis/security_solution/exceptions.ts @@ -7,24 +7,65 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../ftr_provider_context'; -import { getSupertestWithoutAuth } from '../fleet/agents/services'; +import { getSupertestWithoutAuth, setupIngest } from '../fleet/agents/services'; + +const exceptionListBody = { + list_id: 'endpoint_list', + _tags: ['endpoint', 'process', 'malware', 'os:linux'], + tags: ['user added string for a tag', 'malware'], + type: 'endpoint', + description: 'This is a sample agnostic endpoint type exception', + name: 'Sample Endpoint Exception List', + namespace_type: 'agnostic', +}; export default function (providerContext: FtrProviderContext) { const { getService } = providerContext; const esArchiver = getService('esArchiver'); const supertest = getService('supertest'); const supertestWithoutAuth = getSupertestWithoutAuth(providerContext); - const authKey = 'OFpuVDdISUJ3TEZ2a1VFUFFhVDM6TnU1U0JtbHJSeC12Rm9qQWpoSHlUZw=='; + let agentAccessAPIKey: string; describe('artifact download', () => { - before(() => esArchiver.load('security_solution/exceptions/api_feature/exception_list')); + setupIngest(providerContext); + before(async () => { + await esArchiver.load('security_solution/exceptions/api_feature/exception_list'); + const { body: enrollmentApiKeysResponse } = await supertest + .get(`/api/ingest_manager/fleet/enrollment-api-keys`) + .expect(200); + + expect(enrollmentApiKeysResponse.list).length(1); + const { body: enrollmentApiKeyResponse } = await supertest + .get( + `/api/ingest_manager/fleet/enrollment-api-keys/${enrollmentApiKeysResponse.list[0].id}` + ) + .expect(200); + + expect(enrollmentApiKeyResponse.item).to.have.key('api_key'); + const enrollmentAPIToken = enrollmentApiKeyResponse.item.api_key; + // 2. Enroll agent + const { body: enrollmentResponse } = await supertestWithoutAuth + .post(`/api/ingest_manager/fleet/agents/enroll`) + .set('kbn-xsrf', 'xxx') + .set('Authorization', `ApiKey ${enrollmentAPIToken}`) + .send({ + type: 'PERMANENT', + metadata: { + local: {}, + user_provided: {}, + }, + }) + .expect(200); + expect(enrollmentResponse.success).to.eql(true); + agentAccessAPIKey = enrollmentResponse.item.access_api_key; + }); after(() => esArchiver.unload('security_solution/exceptions/api_feature/exception_list')); it('should fail to find artifact with invalid hash', async () => { const { body } = await supertestWithoutAuth .get('/api/endpoint/allowlist/download/endpoint-allowlist-windows-1.0.0/abcd') .set('kbn-xsrf', 'xxx') - .set('authorization', `ApiKey ${authKey}`) + .set('authorization', `ApiKey ${agentAccessAPIKey}`) .send() .expect(404); }); @@ -32,10 +73,10 @@ export default function (providerContext: FtrProviderContext) { it('should download an artifact with correct hash', async () => { const { body } = await supertestWithoutAuth .get( - '/api/endpoint/allowlist/download/endpoint-allowlist-windows-1.0.0/1825fb19fcc6dc391cae0bc4a2e96dd7f728a0c3ae9e1469251ada67f9e1b975' + '/api/endpoint/allowlist/download/endpoint-allowlist-macos-1.0.0/1825fb19fcc6dc391cae0bc4a2e96dd7f728a0c3ae9e1469251ada67f9e1b975' ) .set('kbn-xsrf', 'xxx') - .set('authorization', `ApiKey ${authKey}`) + .set('authorization', `ApiKey ${agentAccessAPIKey}`) .send() .expect(200); }); From 5e0d62f9db37e6b6a856d53d6edb5328e5291b48 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 30 Jun 2020 12:34:23 -0400 Subject: [PATCH 097/106] Type errors, final cleanup --- .../common/endpoint/generate_data.ts | 8 +- .../common/endpoint/types.ts | 4 +- .../policy/store/policy_details/index.test.ts | 8 +- .../endpoint_app_context_services.test.ts | 9 +- .../endpoint/endpoint_app_context_services.ts | 28 +++--- .../server/endpoint/ingest_integration.ts | 2 +- .../server/endpoint/lib/artifacts/common.ts | 2 +- .../endpoint/lib/artifacts/lists.test.ts | 13 ++- .../server/endpoint/lib/artifacts/lists.ts | 6 +- .../endpoint/lib/artifacts/manifest.test.ts | 55 +++++------ .../server/endpoint/lib/artifacts/manifest.ts | 18 ++-- .../lib/artifacts/manifest_entry.test.ts | 8 +- .../endpoint/lib/artifacts/manifest_entry.ts | 2 +- .../endpoint/lib/artifacts/task.mock.ts | 8 +- .../endpoint/lib/artifacts/task.test.ts | 3 +- .../server/endpoint/lib/artifacts/task.ts | 10 +- .../server/endpoint/mocks.ts | 12 ++- .../artifacts/download_exception_list.test.ts | 22 +++-- .../artifacts/download_exception_list.ts | 15 +-- .../endpoint/schemas/artifacts/lists.ts | 1 - .../schemas/artifacts/saved_objects.mock.ts | 5 +- .../artifacts/artifact_client.mock.ts | 6 +- .../artifacts/artifact_client.test.ts | 2 +- .../services/artifacts/artifact_client.ts | 6 +- .../artifacts/manifest_client.mock.ts | 6 +- .../artifacts/manifest_client.test.ts | 5 +- .../services/artifacts/manifest_client.ts | 15 +-- .../manifest_manager/manifest_manager.mock.ts | 38 ++++--- .../manifest_manager/manifest_manager.test.ts | 34 +++---- .../manifest_manager/manifest_manager.ts | 98 ++++++++++--------- .../server/endpoint/types.ts | 2 +- .../apis/security_solution/exceptions.ts | 5 +- .../api_feature/exception_list/data.json | 10 +- 33 files changed, 244 insertions(+), 222 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts index 778edff507daa..e6c9334594bfb 100644 --- a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts @@ -976,9 +976,11 @@ export class EndpointDocGenerator { streams: [], config: { artifact_manifest: { - manifest_version: 'baseline', - schema_version: '1.0.0', - artifacts: {}, + value: { + manifest_version: 'v0', + schema_version: '1.0.0', + artifacts: {}, + }, }, policy: { value: policyFactory(), diff --git a/x-pack/plugins/security_solution/common/endpoint/types.ts b/x-pack/plugins/security_solution/common/endpoint/types.ts index 4735f784a53d5..5e181a17f49cd 100644 --- a/x-pack/plugins/security_solution/common/endpoint/types.ts +++ b/x-pack/plugins/security_solution/common/endpoint/types.ts @@ -676,7 +676,9 @@ export type NewPolicyData = NewDatasource & { enabled: boolean; streams: []; config: { - artifact_manifest: ManifestSchema; + artifact_manifest: { + value: ManifestSchema; + }; policy: { value: PolicyConfig; }; diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts index aa451c021c6e4..0bd623b27f4fb 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts +++ b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts @@ -42,9 +42,11 @@ describe('policy details: ', () => { streams: [], config: { artifact_manifest: { - manifest_version: 'baseline', - schema_version: '1.0.0', - artifacts: {}, + value: { + manifest_version: 'v0', + schema_version: '1.0.0', + artifacts: {}, + }, }, policy: { value: policyConfigFactory(), diff --git a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts index ed1ccff90586d..2daf259941cbf 100644 --- a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts @@ -3,8 +3,9 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { EndpointAppContextService } from './endpoint_app_context_services'; + import { httpServerMock } from '../../../../../src/core/server/mocks'; +import { EndpointAppContextService } from './endpoint_app_context_services'; describe('test endpoint app context services', () => { it('should throw error on getAgentService if start is not called', async () => { @@ -15,10 +16,10 @@ describe('test endpoint app context services', () => { const endpointAppContextService = new EndpointAppContextService(); expect(endpointAppContextService.getManifestManager()).toEqual(undefined); }); - it('should return undefined on getScopedSavedObjectsClient if start is not called', async () => { + it('should throw error on getScopedSavedObjectsClient if start is not called', async () => { const endpointAppContextService = new EndpointAppContextService(); - expect( + expect(() => endpointAppContextService.getScopedSavedObjectsClient(httpServerMock.createKibanaRequest()) - ).toEqual(undefined); + ).toThrow(Error); }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts index eff2a32738f23..7b1734999a4b8 100644 --- a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts +++ b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts @@ -3,7 +3,11 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { SavedObjectsServiceStart, KibanaRequest, SavedObjectsClient } from 'src/core/server'; +import { + SavedObjectsServiceStart, + KibanaRequest, + SavedObjectsClientContract, +} from 'src/core/server'; import { AgentService, IngestManagerStartContract } from '../../../ingest_manager/server'; import { getDatasourceCreateCallback } from './ingest_integration'; import { ManifestManager } from './services/artifacts'; @@ -12,7 +16,7 @@ export type EndpointAppContextServiceStartContract = Pick< IngestManagerStartContract, 'agentService' > & { - manifestManager: ManifestManager; + manifestManager?: ManifestManager | undefined; registerIngestCallback: IngestManagerStartContract['registerExternalCallback']; savedObjectsStart: SavedObjectsServiceStart; }; @@ -30,10 +34,13 @@ export class EndpointAppContextService { this.agentService = dependencies.agentService; this.manifestManager = dependencies.manifestManager; this.savedObjectsStart = dependencies.savedObjectsStart; - dependencies.registerIngestCallback( - 'datasourceCreate', - getDatasourceCreateCallback(this.manifestManager) - ); + + if (this.manifestManager !== undefined) { + dependencies.registerIngestCallback( + 'datasourceCreate', + getDatasourceCreateCallback(this.manifestManager) + ); + } } public stop() {} @@ -49,11 +56,10 @@ export class EndpointAppContextService { return this.manifestManager; } - public getScopedSavedObjectsClient(req: KibanaRequest): SavedObjectsClient | undefined { - let client: SavedObjectsClient; - if (this.savedObjectsStart !== undefined) { - client = this.savedObjectsStart.getScopedClient(req, { excludedWrappers: ['security'] }); + public getScopedSavedObjectsClient(req: KibanaRequest): SavedObjectsClientContract { + if (!this.savedObjectsStart) { + throw new Error(`must call start on ${EndpointAppContextService.name} to call getter`); } - return client; + return this.savedObjectsStart.getScopedClient(req, { excludedWrappers: ['security'] }); } } diff --git a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts index 3cc0471b2df5c..a0309c49192be 100644 --- a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts +++ b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts @@ -4,9 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ +import { NewDatasource } from '../../../ingest_manager/common/types/models'; import { factory as policyConfigFactory } from '../../common/endpoint/models/policy_config'; import { NewPolicyData } from '../../common/endpoint/types'; -import { NewDatasource } from '../../../ingest_manager/common/types/models'; import { ManifestManager } from './services/artifacts'; /** diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts index f0c5ac2bac36c..be255845fc5ca 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts @@ -5,7 +5,7 @@ */ export const ArtifactConstants = { - GLOBAL_ALLOWLIST_NAME: 'endpoint-allowlist', + GLOBAL_ALLOWLIST_NAME: 'endpoint-exceptionlist', SAVED_OBJECT_TYPE: 'securitySolution:endpoint:exceptions-artifact', SUPPORTED_OPERATING_SYSTEMS: ['linux', 'macos', 'windows'], SCHEMA_VERSION: '1.0.0', diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts index 8af4e6eb5abf5..ada2f15a79343 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts @@ -8,7 +8,7 @@ import { ExceptionListClient } from '../../../../../lists/server'; import { listMock } from '../../../../../lists/server/mocks'; import { getFoundExceptionListItemSchemaMock } from '../../../../../lists/common/schemas/response/found_exception_list_item_schema.mock'; import { getExceptionListItemSchemaMock } from '../../../../../lists/common/schemas/response/exception_list_item_schema.mock'; -import { EntriesArray } from '../../../../../lists/common/schemas/types/entries'; +import { EntriesArray, EntryList } from '../../../../../lists/common/schemas/types/entries'; import { getFullEndpointExceptionList } from './lists'; describe('buildEventTypeSignal', () => { @@ -25,13 +25,13 @@ describe('buildEventTypeSignal', () => { { entries: [ { - field: 'some.not.nested.field', + field: 'nested.field', operator: 'included', type: 'exact_cased', value: 'some value', }, ], - field: 'some.field', + field: 'some.parentField', type: 'nested', }, { @@ -138,8 +138,11 @@ describe('buildEventTypeSignal', () => { field: 'server.domain', operator: 'included', type: 'list', - value: ['lists', 'not', 'supported'], - }, + list: { + id: 'lists_not_supported', + type: 'keyword', + }, + } as EntryList, { field: 'server.ip', operator: 'included', type: 'exists' }, ]; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts index 417f2f41b4e04..e4aa7328af615 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts @@ -166,7 +166,9 @@ function translateEntry( export function compressExceptionList( exceptionList: WrappedTranslatedExceptionList ): Promise { - return lzma.compress(JSON.stringify(exceptionList), (res: Buffer) => { - return res; + return new Promise((resolve, reject) => { + lzma.compress(JSON.stringify(exceptionList), undefined, (res: Buffer) => { + resolve(res); + }); }); } diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts index f79e38b3c4d7d..f4227d3f8e3d5 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import { ManifestSchemaVersion } from '../../../../common/endpoint/schema/common'; import { InternalArtifactSchema } from '../../schemas'; import { getInternalArtifactMock, @@ -26,50 +27,50 @@ describe('manifest', () => { artifacts.push(artifactMacos); artifacts.push(artifactWindows); - manifest1 = new Manifest(now, '1.0.0'); + manifest1 = new Manifest(now, '1.0.0', 'v0'); manifest1.addEntry(artifactLinux); manifest1.addEntry(artifactMacos); manifest1.addEntry(artifactWindows); manifest1.setVersion('abcd'); const newArtifactLinux = await getInternalArtifactMockWithDiffs('linux', '1.0.0'); - manifest2 = new Manifest(new Date(), '1.0.0'); + manifest2 = new Manifest(new Date(), '1.0.0', 'v0'); manifest2.addEntry(newArtifactLinux); manifest2.addEntry(artifactMacos); manifest2.addEntry(artifactWindows); }); test('Can create manifest with valid schema version', () => { - const manifest = new Manifest(new Date(), '1.0.0'); + const manifest = new Manifest(new Date(), '1.0.0', 'v0'); expect(manifest).toBeInstanceOf(Manifest); }); test('Cannot create manifest with invalid schema version', () => { expect(() => { - new Manifest(new Date(), 'abcd'); + new Manifest(new Date(), 'abcd' as ManifestSchemaVersion, 'v0'); }).toThrow(); }); test('Manifest transforms correctly to expected endpoint format', async () => { expect(manifest1.toEndpointFormat()).toStrictEqual({ artifacts: { - 'endpoint-allowlist-linux-1.0.0': { + 'endpoint-exceptionlist-linux-1.0.0': { sha256: '222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', size: 268, url: - '/api/endpoint/allowlist/download/endpoint-allowlist-linux-1.0.0/222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', }, - 'endpoint-allowlist-macos-1.0.0': { + 'endpoint-exceptionlist-macos-1.0.0': { sha256: '222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', size: 268, url: - '/api/endpoint/allowlist/download/endpoint-allowlist-macos-1.0.0/222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', + '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-1.0.0/222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', }, - 'endpoint-allowlist-windows-1.0.0': { + 'endpoint-exceptionlist-windows-1.0.0': { sha256: '222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', size: 268, url: - '/api/endpoint/allowlist/download/endpoint-allowlist-windows-1.0.0/222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', }, }, manifest_version: 'abcd', @@ -77,21 +78,13 @@ describe('manifest', () => { }); }); - test('Manifest cannot be converted to endpoint format without a version', async () => { - const manifest = new Manifest(new Date(), '1.0.0'); - manifest.addEntry(await getInternalArtifactMock('linux', '1.0.0')); - manifest.addEntry(await getInternalArtifactMock('macos', '1.0.0')); - manifest.addEntry(await getInternalArtifactMock('windows', '1.0.0')); - expect(manifest.toEndpointFormat).toThrow(); - }); - test('Manifest transforms correctly to expected saved object format', async () => { expect(manifest1.toSavedObject()).toStrictEqual({ created: now.getTime(), ids: [ - 'endpoint-allowlist-linux-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', - 'endpoint-allowlist-macos-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', - 'endpoint-allowlist-windows-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', + 'endpoint-exceptionlist-linux-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', + 'endpoint-exceptionlist-macos-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', + 'endpoint-exceptionlist-windows-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', ], }); }); @@ -101,12 +94,12 @@ describe('manifest', () => { expect(diffs).toEqual([ { id: - 'endpoint-allowlist-linux-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', + 'endpoint-exceptionlist-linux-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', type: 'delete', }, { id: - 'endpoint-allowlist-linux-1.0.0-03114bf3dc2258f0def5beaf675242b68b428c96eefab5f6c5533f0d8e4deb0b', + 'endpoint-exceptionlist-linux-1.0.0-03114bf3dc2258f0def5beaf675242b68b428c96eefab5f6c5533f0d8e4deb0b', type: 'add', }, ]); @@ -122,34 +115,34 @@ describe('manifest', () => { const entries = manifest1.getEntries(); const keys = Object.keys(entries); expect(keys).toEqual([ - 'endpoint-allowlist-linux-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', - 'endpoint-allowlist-macos-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', - 'endpoint-allowlist-windows-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', + 'endpoint-exceptionlist-linux-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', + 'endpoint-exceptionlist-macos-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', + 'endpoint-exceptionlist-windows-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', ]); }); test('Manifest returns true if contains artifact', async () => { const found = manifest1.contains( - 'endpoint-allowlist-macos-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466' + 'endpoint-exceptionlist-macos-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466' ); expect(found).toEqual(true); }); test('Manifest can be created from list of artifacts', async () => { - const manifest = Manifest.fromArtifacts(artifacts, '1.0.0'); + const manifest = Manifest.fromArtifacts(artifacts, '1.0.0', 'v0'); expect( manifest.contains( - 'endpoint-allowlist-linux-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466' + 'endpoint-exceptionlist-linux-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466' ) ).toEqual(true); expect( manifest.contains( - 'endpoint-allowlist-macos-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466' + 'endpoint-exceptionlist-macos-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466' ) ).toEqual(true); expect( manifest.contains( - 'endpoint-allowlist-windows-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466' + 'endpoint-exceptionlist-windows-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466' ) ).toEqual(true); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts index 08cb4ecd1ed4c..c343568226e22 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts @@ -24,18 +24,19 @@ export class Manifest { private schemaVersion: ManifestSchemaVersion; // For concurrency control - private version: string | undefined; + private version: string; - constructor(created: Date, schemaVersion: string) { + constructor(created: Date, schemaVersion: string, version: string) { this.created = created; this.entries = {}; + this.version = version; const [validated, errors] = validate( (schemaVersion as unknown) as object, manifestSchemaVersion ); - if (errors != null) { + if (errors != null || validated === null) { throw new Error(`Invalid manifest version: ${schemaVersion}`); } @@ -44,9 +45,10 @@ export class Manifest { public static fromArtifacts( artifacts: InternalArtifactSchema[], - schemaVersion: ManifestSchemaVersion + schemaVersion: string, + version: string ): Manifest { - const manifest = new Manifest(new Date(), schemaVersion); + const manifest = new Manifest(new Date(), schemaVersion, version); artifacts.forEach((artifact) => { manifest.addEntry(artifact); }); @@ -57,11 +59,11 @@ export class Manifest { return this.schemaVersion; } - public getVersion(): string | undefined { + public getVersion(): string { return this.version; } - public setVersion(version: string | undefined) { + public setVersion(version: string) { this.version = version; } @@ -102,7 +104,7 @@ export class Manifest { public toEndpointFormat(): ManifestSchema { const manifestObj: ManifestSchema = { - manifest_version: this.version ?? 'baseline', + manifest_version: this.version ?? 'v0', schema_version: this.schemaVersion, artifacts: {}, }; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts index dda7e8e75443f..8c37d689a9d34 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts @@ -24,12 +24,12 @@ describe('manifest_entry', () => { test('Correct doc_id is returned', () => { expect(manifestEntry.getDocId()).toEqual( - 'endpoint-allowlist-windows-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466' + 'endpoint-exceptionlist-windows-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466' ); }); test('Correct identifier is returned', () => { - expect(manifestEntry.getIdentifier()).toEqual('endpoint-allowlist-windows-1.0.0'); + expect(manifestEntry.getIdentifier()).toEqual('endpoint-exceptionlist-windows-1.0.0'); }); test('Correct sha256 is returned', () => { @@ -44,7 +44,7 @@ describe('manifest_entry', () => { test('Correct url is returned', () => { expect(manifestEntry.getUrl()).toEqual( - '/api/endpoint/allowlist/download/endpoint-allowlist-windows-1.0.0/222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466' + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466' ); }); @@ -57,7 +57,7 @@ describe('manifest_entry', () => { sha256: '222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', size: 268, url: - '/api/endpoint/allowlist/download/endpoint-allowlist-windows-1.0.0/222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', }); }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts index f5fd23c978c32..00fd446bf14b5 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts @@ -31,7 +31,7 @@ export class ManifestEntry { } public getUrl(): string { - return `/api/endpoint/allowlist/download/${this.getIdentifier()}/${this.getSha256()}`; + return `/api/endpoint/artifacts/download/${this.getIdentifier()}/${this.getSha256()}`; } public getArtifact(): InternalArtifactSchema { diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.mock.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.mock.ts index 409d48c3dbfd2..4391d89f3b2b2 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.mock.ts @@ -4,14 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ -import { taskManagerStartContract } from '../../../../../task_manager/server'; - -import { taskManagerMock } from '../../../../../task_manager/server/mocks'; - -import { createMockEndpointAppContext } from '../../mocks'; - import { ManifestTask } from './task'; export class MockManifestTask extends ManifestTask { - private runTask = jest.fn(); + public runTask = jest.fn(); } diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.test.ts index 15200cacaacf7..daa8a7dd83ee0 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.test.ts @@ -8,7 +8,6 @@ import { taskManagerMock } from '../../../../../task_manager/server/mocks'; import { TaskStatus } from '../../../../../task_manager/server'; import { createMockEndpointAppContext } from '../../mocks'; -import { getManifestManagerMock } from '../../services/artifacts/manifest_manager/manifest_manager.mock'; import { ManifestTaskConstants, ManifestTask } from './task'; import { MockManifestTask } from './task.mock'; @@ -25,7 +24,7 @@ describe('task', () => { test('task should be registered', () => { const mockTaskManager = taskManagerMock.createSetup(); - const manifestTask = new ManifestTask({ + new ManifestTask({ endpointAppContext: createMockEndpointAppContext(), taskManager: mockTaskManager, }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts index 95c849ebe4e00..100e762ae80c7 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts @@ -4,12 +4,12 @@ * you may not use this file except in compliance with the Elastic License. */ -import { Logger } from '../../../../../../../src/kibana/server'; +import { Logger } from 'src/core/server'; import { ConcreteTaskInstance, TaskManagerSetupContract, TaskManagerStartContract, -} from '../../../../../../plugins/task_manager/server'; +} from '../../../../../task_manager/server'; import { EndpointAppContext } from '../../types'; export const ManifestTaskConstants = { @@ -74,7 +74,7 @@ export class ManifestTask { return `${ManifestTaskConstants.TYPE}:${ManifestTaskConstants.VERSION}`; }; - private runTask = async (taskId: string) => { + public runTask = async (taskId: string) => { // Check that this task is current if (taskId !== this.getTaskId()) { // old task, return @@ -92,12 +92,12 @@ export class ManifestTask { manifestManager .refresh() .then((wrappedManifest) => { - if (wrappedManifest !== null) { + if (wrappedManifest) { return manifestManager.dispatch(wrappedManifest); } }) .then((wrappedManifest) => { - if (wrappedManifest !== null) { + if (wrappedManifest) { return manifestManager.commit(wrappedManifest); } }) diff --git a/x-pack/plugins/security_solution/server/endpoint/mocks.ts b/x-pack/plugins/security_solution/server/endpoint/mocks.ts index b6892ee5fb1b1..d0af41d2b5c04 100644 --- a/x-pack/plugins/security_solution/server/endpoint/mocks.ts +++ b/x-pack/plugins/security_solution/server/endpoint/mocks.ts @@ -6,7 +6,6 @@ import { IScopedClusterClient, SavedObjectsClientContract } from 'kibana/server'; import { loggingSystemMock, savedObjectsServiceMock } from 'src/core/server/mocks'; - import { xpackMocks } from '../../../../mocks'; import { AgentService, @@ -14,6 +13,7 @@ import { ExternalCallback, } from '../../../ingest_manager/server'; import { createDatasourceServiceMock } from '../../../ingest_manager/server/mocks'; +import { ConfigType } from '../config'; import { createMockConfig } from '../lib/detection_engine/routes/__mocks__'; import { EndpointAppContextService, @@ -23,14 +23,18 @@ import { ManifestManagerMock, getManifestManagerMock, } from './services/artifacts/manifest_manager/manifest_manager.mock'; +import { EndpointAppContext } from './types'; /** * Creates a mocked EndpointAppContext. */ -export const createMockEndpointAppContext = (mockManifestManager?: ManifestManagerMock) => { +export const createMockEndpointAppContext = ( + mockManifestManager?: ManifestManagerMock +): EndpointAppContext => { return { logFactory: loggingSystemMock.create(), - config: createMockConfig(), + // @ts-ignore + config: createMockConfig() as ConfigType, service: createMockEndpointAppContextService(mockManifestManager), }; }; @@ -45,6 +49,7 @@ export const createMockEndpointAppContextService = ( start: jest.fn(), stop: jest.fn(), getAgentService: jest.fn(), + // @ts-ignore getManifestManager: mockManifestManager ?? jest.fn(), getScopedSavedObjectsClient: jest.fn(), }; @@ -59,6 +64,7 @@ export const createMockEndpointAppContextServiceStartContract = (): jest.Mocked< return { agentService: createMockAgentService(), savedObjectsStart: savedObjectsServiceMock.createStartContract(), + // @ts-ignore manifestManager: getManifestManagerMock(), registerIngestCallback: jest.fn< ReturnType, diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts index 9a8e8fe0dfcba..02b9c7c3d89c9 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts @@ -78,6 +78,7 @@ describe('test alerts route', () => { let mockScopedClient: jest.Mocked; let mockSavedObjectClient: jest.Mocked; let mockResponse: jest.Mocked; + // @ts-ignore let routeConfig: RouteConfig; let routeHandler: RequestHandler; let endpointAppContextService: EndpointAppContextService; @@ -98,6 +99,7 @@ describe('test alerts route', () => { // The authentication with the Fleet Plugin needs a separate scoped SO Client ingestSavedObjectClient = savedObjectsClientMock.create(); ingestSavedObjectClient.find.mockReturnValue(Promise.resolve(mockIngestSOResponse)); + // @ts-ignore startContract.savedObjectsStart.getScopedClient.mockReturnValue(ingestSavedObjectClient); endpointAppContextService.start(startContract); @@ -114,7 +116,7 @@ describe('test alerts route', () => { it('should serve the compressed artifact to download', async () => { const mockRequest = httpServerMock.createKibanaRequest({ - path: `/api/endpoint/allowlist/download/${mockArtifactName}/123456`, + path: `/api/endpoint/artifacts/download/${mockArtifactName}/123456`, method: 'get', params: { sha256: '123456' }, headers: { @@ -144,7 +146,7 @@ describe('test alerts route', () => { ingestSavedObjectClient.get.mockImplementationOnce(() => Promise.resolve(soFindResp)); [routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) => - path.startsWith('/api/endpoint/allowlist/download') + path.startsWith('/api/endpoint/artifacts/download') )!; await routeHandler( @@ -172,7 +174,7 @@ describe('test alerts route', () => { it('should handle fetching a non-existent artifact', async () => { const mockRequest = httpServerMock.createKibanaRequest({ - path: `/api/endpoint/allowlist/download/${mockArtifactName}/123456`, + path: `/api/endpoint/artifacts/download/${mockArtifactName}/123456`, method: 'get', params: { sha256: '789' }, headers: { @@ -186,7 +188,7 @@ describe('test alerts route', () => { ); [routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) => - path.startsWith('/api/endpoint/allowlist/download') + path.startsWith('/api/endpoint/artifacts/download') )!; await routeHandler( @@ -206,7 +208,7 @@ describe('test alerts route', () => { it('should utilize the cache', async () => { const mockSha = '123456789'; const mockRequest = httpServerMock.createKibanaRequest({ - path: `/api/endpoint/allowlist/download/${mockArtifactName}/${mockSha}`, + path: `/api/endpoint/artifacts/download/${mockArtifactName}/${mockSha}`, method: 'get', params: { sha256: mockSha, identifier: mockArtifactName }, headers: { @@ -220,7 +222,7 @@ describe('test alerts route', () => { cache.set(cacheKey, mockCompressedArtifact.toString('binary')); [routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) => - path.startsWith('/api/endpoint/allowlist/download') + path.startsWith('/api/endpoint/artifacts/download') )!; await routeHandler( @@ -242,13 +244,13 @@ describe('test alerts route', () => { it('should respond with a 401 if a valid API Token is not supplied', async () => { const mockSha = '123456789'; const mockRequest = httpServerMock.createKibanaRequest({ - path: `/api/endpoint/allowlist/download/${mockArtifactName}/${mockSha}`, + path: `/api/endpoint/artifacts/download/${mockArtifactName}/${mockSha}`, method: 'get', params: { sha256: mockSha, identifier: mockArtifactName }, }); [routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) => - path.startsWith('/api/endpoint/allowlist/download') + path.startsWith('/api/endpoint/artifacts/download') )!; await routeHandler( @@ -268,7 +270,7 @@ describe('test alerts route', () => { it('should respond with a 404 if an agent cannot be linked to the API token', async () => { const mockSha = '123456789'; const mockRequest = httpServerMock.createKibanaRequest({ - path: `/api/endpoint/allowlist/download/${mockArtifactName}/${mockSha}`, + path: `/api/endpoint/artifacts/download/${mockArtifactName}/${mockSha}`, method: 'get', params: { sha256: mockSha, identifier: mockArtifactName }, headers: { @@ -282,7 +284,7 @@ describe('test alerts route', () => { ingestSavedObjectClient.find.mockReturnValue(Promise.resolve(mockIngestSOResponse)); [routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) => - path.startsWith('/api/endpoint/allowlist/download') + path.startsWith('/api/endpoint/artifacts/download') )!; await routeHandler( diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts index 404f1e37d1d82..8bf8a98b8d8f4 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { IRouter } from 'src/core/server'; +import { IRouter, SavedObjectsClientContract, HttpResponseOptions } from 'src/core/server'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths import { authenticateAgentWithAccessToken } from '../../../../../ingest_manager/server/services/agents/authenticate'; import { validate } from '../../../../common/validate'; @@ -18,7 +18,7 @@ import { } from '../../schemas/artifacts'; import { EndpointAppContext } from '../../types'; -const allowlistBaseRoute: string = '/api/endpoint/allowlist'; +const allowlistBaseRoute: string = '/api/endpoint/artifacts'; /** * Registers the exception list route to enable sensors to download a compressed allowlist @@ -39,8 +39,9 @@ export function registerDownloadExceptionListRoute( }, options: { tags: [] }, }, + // @ts-ignore async (context, req, res) => { - let scopedSOClient; + let scopedSOClient: SavedObjectsClientContract; const logger = endpointContext.logFactory.get('download_exception_list'); // The ApiKey must be associated with an enrolled Fleet agent @@ -63,11 +64,13 @@ export function registerDownloadExceptionListRoute( 'content-disposition': `attachment; filename=${artName}.xz`, }, }; + const [validated, errors] = validate(artifact, downloadArtifactResponseSchema); - if (errors != null) { - return res.internalError({ body: errors }); + + if (errors !== null || validated === null) { + return res.internalError({ body: errors! }); } else { - return res.ok(validated); + return res.ok((validated as unknown) as HttpResponseOptions); } }; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts index 7a85f6a8a96ad..21d1105a313e7 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts @@ -5,7 +5,6 @@ */ import * as t from 'io-ts'; - import { operator } from '../../../../../lists/common/schemas'; export const translatedEntryMatchAny = t.exact( diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts index a1b768b2e69ff..1a9cc55ca5725 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts @@ -4,9 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ -import { InternalArtifactSchema, InternalManifestSchema } from './saved_objects'; -import { getTranslatedExceptionListMock } from './lists.mock'; import { ArtifactConstants, buildArtifact } from '../../lib/artifacts'; +import { getTranslatedExceptionListMock } from './lists.mock'; +import { InternalArtifactSchema, InternalManifestSchema } from './saved_objects'; export const getInternalArtifactMock = async ( os: string, @@ -28,6 +28,7 @@ export const getInternalArtifactsMock = async ( os: string, schemaVersion: string ): Promise => { + // @ts-ignore return ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS.map(async () => { await buildArtifact(getTranslatedExceptionListMock(), os, schemaVersion); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.mock.ts index 5e13ef54a478f..6392c59b2377c 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.mock.ts @@ -4,12 +4,12 @@ * you may not use this file except in compliance with the Elastic License. */ -import { savedObjectsClientMock } from '../../../../../../../src/core/server/mocks'; - +import { savedObjectsClientMock } from 'src/core/server/mocks'; +import { SavedObjectsClientContract } from 'src/core/server'; import { ArtifactClient } from './artifact_client'; export const getArtifactClientMock = ( - savedObjectsClient?: typeof savedObjectsClientMock.create + savedObjectsClient?: SavedObjectsClientContract ): ArtifactClient => { if (savedObjectsClient !== undefined) { return new ArtifactClient(savedObjectsClient); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts index f942485aec7d4..08e29b5c6b82b 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { savedObjectsClientMock } from '../../../../../../../src/core/server/mocks'; +import { savedObjectsClientMock } from 'src/core/server/mocks'; import { ArtifactConstants } from '../../lib/artifacts'; import { getInternalArtifactMock } from '../../schemas/artifacts/saved_objects.mock'; import { getArtifactClientMock } from './artifact_client.mock'; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts index 47c76d643b018..4a3dcaae1bd3d 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts @@ -4,14 +4,14 @@ * you may not use this file except in compliance with the Elastic License. */ -import { SavedObject, SavedObjectsClient } from '../../../../../../../src/core/server'; +import { SavedObject, SavedObjectsClientContract } from 'src/core/server'; import { ArtifactConstants } from '../../lib/artifacts'; import { InternalArtifactSchema } from '../../schemas/artifacts'; export class ArtifactClient { - private savedObjectsClient: SavedObjectsClient; + private savedObjectsClient: SavedObjectsClientContract; - constructor(savedObjectsClient: SavedObjectsClient) { + constructor(savedObjectsClient: SavedObjectsClientContract) { this.savedObjectsClient = savedObjectsClient; } diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.mock.ts index 8029d328c8229..bfeacbcedf2cb 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.mock.ts @@ -4,12 +4,12 @@ * you may not use this file except in compliance with the Elastic License. */ -import { savedObjectsClientMock } from '../../../../../../../src/core/server/mocks'; - +import { SavedObjectsClientContract } from 'src/core/server'; +import { savedObjectsClientMock } from 'src/core/server/mocks'; import { ManifestClient } from './manifest_client'; export const getManifestClientMock = ( - savedObjectsClient?: typeof savedObjectsClientMock.create + savedObjectsClient?: SavedObjectsClientContract ): ManifestClient => { if (savedObjectsClient !== undefined) { return new ManifestClient(savedObjectsClient, '1.0.0'); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.test.ts index 7bdf567f06e8e..5780c6279ee6a 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.test.ts @@ -4,7 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ -import { savedObjectsClientMock } from '../../../../../../../src/core/server/mocks'; +import { savedObjectsClientMock } from 'src/core/server/mocks'; +import { ManifestSchemaVersion } from '../../../../common/endpoint/schema/common'; import { ManifestConstants } from '../../lib/artifacts'; import { getInternalManifestMock } from '../../schemas/artifacts/saved_objects.mock'; import { getManifestClientMock } from './manifest_client.mock'; @@ -19,7 +20,7 @@ describe('manifest_client', () => { test('cannot create ManifestClient with invalid schema version', () => { expect(() => { - new ManifestClient(savedObjectsClientMock.create(), 'invalid'); + new ManifestClient(savedObjectsClientMock.create(), 'invalid' as ManifestSchemaVersion); }).toThrow(); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.ts index 766d710a687b1..45182841e56fc 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.ts @@ -6,14 +6,14 @@ import { SavedObject, - SavedObjectsClient, + SavedObjectsClientContract, SavedObjectsUpdateResponse, -} from '../../../../../../../src/core/server'; -import { validate } from '../../../../common/validate'; +} from 'src/core/server'; import { manifestSchemaVersion, ManifestSchemaVersion, } from '../../../../common/endpoint/schema/common'; +import { validate } from '../../../../common/validate'; import { ManifestConstants } from '../../lib/artifacts'; import { InternalManifestSchema } from '../../schemas/artifacts'; @@ -23,9 +23,12 @@ interface UpdateManifestOpts { export class ManifestClient { private schemaVersion: ManifestSchemaVersion; - private savedObjectsClient: SavedObjectsClient; + private savedObjectsClient: SavedObjectsClientContract; - constructor(savedObjectsClient: SavedObjectsClient, schemaVersion: ManifestSchemaVersion) { + constructor( + savedObjectsClient: SavedObjectsClientContract, + schemaVersion: ManifestSchemaVersion + ) { this.savedObjectsClient = savedObjectsClient; const [validated, errors] = validate( @@ -33,7 +36,7 @@ export class ManifestClient { manifestSchemaVersion ); - if (errors != null) { + if (errors != null || validated === null) { throw new Error(`Invalid manifest version: ${schemaVersion}`); } diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts index 804f2b4484718..87e12d49b8fcf 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts @@ -5,28 +5,19 @@ */ // eslint-disable-next-line max-classes-per-file -import { - loggingSystemMock, - savedObjectsClientMock, -} from '../../../../../../../../src/core/server/mocks'; -import { Logger, SavedObjectsClient } from '../../../../../../../../src/core/server'; - -import { DatasourceServiceInterface } from '../../../../../../ingest_manager/server'; +import { savedObjectsClientMock, loggingSystemMock } from 'src/core/server/mocks'; +import { Logger } from 'src/core/server'; import { getFoundExceptionListItemSchemaMock } from '../../../../../../lists/common/schemas/response/found_exception_list_item_schema.mock'; import { listMock } from '../../../../../../lists/server/mocks'; - import { ExceptionsCache, Manifest, buildArtifact, getFullEndpointExceptionList, } from '../../../lib/artifacts'; -import { getInternalArtifactMock, getInternalArtifactsMock } from '../../../schemas'; - +import { InternalArtifactSchema } from '../../../schemas/artifacts'; import { getArtifactClientMock } from '../artifact_client.mock'; import { getManifestClientMock } from '../manifest_client.mock'; -import { ManifestClient } from '../manifest_client'; - import { ManifestManager } from './manifest_manager'; function getMockDatasource() { @@ -66,7 +57,7 @@ export function getDatasourceServiceMock() { async function mockBuildExceptionListArtifacts( os: string, schemaVersion: string -): InternalArtifactSchema[] { +): Promise { const mockExceptionClient = listMock.getExceptionListClient(); const first = getFoundExceptionListItemSchemaMock(); mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); @@ -74,37 +65,42 @@ async function mockBuildExceptionListArtifacts( return [await buildArtifact(exceptions, os, schemaVersion)]; } +// @ts-ignore export class ManifestManagerMock extends ManifestManager { + // @ts-ignore private buildExceptionListArtifacts = async () => { return mockBuildExceptionListArtifacts('linux', '1.0.0'); }; + // @ts-ignore private getLastDispatchedManifest = jest .fn() - .mockResolvedValue(new Manifest(new Date(), '1.0.0')); + .mockResolvedValue(new Manifest(new Date(), '1.0.0', 'v0')); + // @ts-ignore private getManifestClient = jest .fn() .mockReturnValue(getManifestClientMock(this.savedObjectsClient)); } -export const getManifestManagerMock = (opts: { - datasourceServiceMock?: DatasourceServiceMock; - savedObjectsClientMock?: SavedObjectsClient; +export const getManifestManagerMock = (opts?: { + datasourceService?: DatasourceServiceMock; + savedObjectsClient?: ReturnType; }): ManifestManagerMock => { let datasourceService = getDatasourceServiceMock(); - if (opts?.datasourceServiceMock !== undefined) { - datasourceService = opts.datasourceServiceMock; + if (opts?.datasourceService !== undefined) { + datasourceService = opts.datasourceService; } let savedObjectsClient = savedObjectsClientMock.create(); - if (opts?.savedObjectsClientMock !== undefined) { - savedObjectsClient = opts.savedObjectsClientMock; + if (opts?.savedObjectsClient !== undefined) { + savedObjectsClient = opts.savedObjectsClient; } const manifestManager = new ManifestManagerMock({ artifactClient: getArtifactClientMock(savedObjectsClient), cache: new ExceptionsCache(5), + // @ts-ignore datasourceService, exceptionListClient: listMock.getExceptionListClient(), logger: loggingSystemMock.create().get() as jest.Mocked, diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts index 35b332e30cbca..3c1ff199b05ab 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts @@ -4,56 +4,52 @@ * you may not use this file except in compliance with the Elastic License. */ -import { savedObjectsClientMock } from '../../../../../../../../src/core/server/mocks'; +import { savedObjectsClientMock } from 'src/core/server/mocks'; import { ArtifactConstants, ManifestConstants, Manifest } from '../../../lib/artifacts'; -import { getArtifactClientMock } from '../artifact_client.mock'; -import { getManifestClientMock } from '../manifest_client.mock'; -import { ManifestManager } from './manifest_manager'; import { getDatasourceServiceMock, getManifestManagerMock } from './manifest_manager.mock'; describe('manifest_manager', () => { describe('ManifestManager sanity checks', () => { - beforeAll(async () => {}); - test('ManifestManager can refresh manifest', async () => { const manifestManager = getManifestManagerMock(); const manifestWrapper = await manifestManager.refresh(); - expect(manifestManager.getLastDispatchedManifest).toHaveBeenCalled(); - expect(manifestWrapper.diffs).toEqual([ + expect(manifestWrapper!.diffs).toEqual([ { id: - 'endpoint-allowlist-linux-1.0.0-a0b2886af05849e1e7e7b05bd6e38ea2e2de6566bfb5f4bdbdeda8236de0ff5c', + 'endpoint-exceptionlist-linux-1.0.0-a0b2886af05849e1e7e7b05bd6e38ea2e2de6566bfb5f4bdbdeda8236de0ff5c', type: 'add', }, ]); - expect(manifestWrapper.manifest).toBeInstanceOf(Manifest); + expect(manifestWrapper!.manifest).toBeInstanceOf(Manifest); }); test('ManifestManager can dispatch manifest', async () => { const datasourceService = getDatasourceServiceMock(); - const manifestManager = getManifestManagerMock({ datasourceServiceMock: datasourceService }); + const manifestManager = getManifestManagerMock({ datasourceService }); const manifestWrapperRefresh = await manifestManager.refresh(); const manifestWrapperDispatch = await manifestManager.dispatch(manifestWrapperRefresh); expect(manifestWrapperRefresh).toEqual(manifestWrapperDispatch); - const entries = manifestWrapperDispatch.manifest.entries; - const artifact = Object.values(entries)[0].artifact; - expect(datasourceService.update.mock.calls[0][2].inputs[0].config.artifact_manifest).toEqual({ - manifest_version: 'baseline', + const entries = manifestWrapperDispatch!.manifest.getEntries(); + const artifact = Object.values(entries)[0].getArtifact(); + expect( + datasourceService.update.mock.calls[0][2].inputs[0].config.artifact_manifest.value + ).toEqual({ + manifest_version: 'v0', schema_version: '1.0.0', artifacts: { [artifact.identifier]: { sha256: artifact.sha256, size: artifact.size, - url: `/api/endpoint/allowlist/download/${artifact.identifier}/${artifact.sha256}`, + url: `/api/endpoint/artifacts/download/${artifact.identifier}/${artifact.sha256}`, }, }, }); }); test('ManifestManager can commit manifest', async () => { - const savedObjectsClient = savedObjectsClientMock.create(); + const savedObjectsClient: ReturnType = savedObjectsClientMock.create(); const manifestManager = getManifestManagerMock({ - savedObjectsClientMock: savedObjectsClient, + savedObjectsClient, }); const manifestWrapperRefresh = await manifestManager.refresh(); @@ -62,7 +58,7 @@ describe('manifest_manager', () => { id: 'abcd', type: 'delete', }; - manifestWrapperDispatch.diffs.push(diff); + manifestWrapperDispatch!.diffs.push(diff); await manifestManager.commit(manifestWrapperDispatch); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts index 7672c2134dd35..3f5d0e92836d4 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts @@ -4,10 +4,10 @@ * you may not use this file except in compliance with the Elastic License. */ -import { Logger, SavedObjectsClient } from '../../../../../../../../src/core/server'; +import { Logger, SavedObjectsClientContract, SavedObject } from 'src/core/server'; import { DatasourceServiceInterface } from '../../../../../../ingest_manager/server'; import { ExceptionListClient } from '../../../../../../lists/server'; -import { NewPolicyData } from '../../../../../common/endpoint/types'; +import { ManifestSchemaVersion } from '../../../../../common/endpoint/schema/common'; import { ArtifactConstants, ManifestConstants, @@ -15,13 +15,14 @@ import { buildArtifact, getFullEndpointExceptionList, ExceptionsCache, + ManifestDiff, } from '../../../lib/artifacts'; import { InternalArtifactSchema, InternalManifestSchema } from '../../../schemas/artifacts'; import { ArtifactClient } from '../artifact_client'; import { ManifestClient } from '../manifest_client'; export interface ManifestManagerContext { - savedObjectsClient: SavedObjectsClient; + savedObjectsClient: SavedObjectsClientContract; artifactClient: ArtifactClient; exceptionListClient: ExceptionListClient; datasourceService: DatasourceServiceInterface; @@ -39,12 +40,12 @@ export interface WrappedManifest { } export class ManifestManager { - private artifactClient: ArtifactClient; - private exceptionListClient: ExceptionListClient; - private datasourceService: DatasourceServiceInterface; - private savedObjectsClient: SavedObjectsClient; - private logger: Logger; - private cache: ExceptionsCache; + protected artifactClient: ArtifactClient; + protected exceptionListClient: ExceptionListClient; + protected datasourceService: DatasourceServiceInterface; + protected savedObjectsClient: SavedObjectsClientContract; + protected logger: Logger; + protected cache: ExceptionsCache; constructor(context: ManifestManagerContext) { this.artifactClient = context.artifactClient; @@ -56,7 +57,7 @@ export class ManifestManager { } private getManifestClient(schemaVersion: string): ManifestClient { - return new ManifestClient(this.savedObjectsClient, schemaVersion); + return new ManifestClient(this.savedObjectsClient, schemaVersion as ManifestSchemaVersion); } private async buildExceptionListArtifacts( @@ -79,41 +80,41 @@ export class ManifestManager { } private async getLastDispatchedManifest(schemaVersion: string): Promise { - const manifestClient = this.getManifestClient(schemaVersion); - - let manifestSo: InternalManifestSchema; - try { - manifestSo = await manifestClient.getManifest(); - } catch (err) { - if (err.output.statusCode !== 404) { - throw err; - } - } - - if (manifestSo !== undefined) { - const manifest = new Manifest(manifestSo.attributes.created, schemaVersion); - manifest.setVersion(manifestSo.version); - - for (const id of manifestSo.attributes.ids) { - const artifactSo = await this.artifactClient.getArtifact(id); - manifest.addEntry(artifactSo.attributes); - } + return this.getManifestClient(schemaVersion) + .getManifest() + .then(async (manifestSo: SavedObject) => { + if (manifestSo.version === undefined) { + throw new Error('No version returned for manifest.'); + } + const manifest = new Manifest( + new Date(manifestSo.attributes.created), + schemaVersion, + manifestSo.version + ); + + for (const id of manifestSo.attributes.ids) { + const artifactSo = await this.artifactClient.getArtifact(id); + manifest.addEntry(artifactSo.attributes); + } - return manifest; - } else { - return null; - } + return manifest; + }) + .catch((err) => { + if (err.output.statusCode !== 404) { + throw err; + } + return null; + }); } public async refresh(opts?: ManifestRefreshOpts): Promise { - let oldManifest: Manifest; + let oldManifest: Manifest | null; // Get the last-dispatched manifest oldManifest = await this.getLastDispatchedManifest(ManifestConstants.SCHEMA_VERSION); - // console.log(oldManifest); if (oldManifest === null && opts !== undefined && opts.initialize) { - oldManifest = new Manifest(new Date(), ManifestConstants.SCHEMA_VERSION); // create empty manifest + oldManifest = new Manifest(new Date(), ManifestConstants.SCHEMA_VERSION, 'v0'); // create empty manifest } else if (oldManifest == null) { this.logger.debug('Manifest does not exist yet. Waiting...'); return null; @@ -123,8 +124,11 @@ export class ManifestManager { const artifacts = await this.buildExceptionListArtifacts(ArtifactConstants.SCHEMA_VERSION); // Build new manifest - const newManifest = Manifest.fromArtifacts(artifacts, ManifestConstants.SCHEMA_VERSION); - newManifest.setVersion(oldManifest.getVersion()); + const newManifest = Manifest.fromArtifacts( + artifacts, + ManifestConstants.SCHEMA_VERSION, + oldManifest.getVersion() + ); // Get diffs const diffs = newManifest.diff(oldManifest); @@ -159,7 +163,7 @@ export class ManifestManager { * * @return {WrappedManifest | null} WrappedManifest if all dispatched, else null */ - public async dispatch(wrappedManifest: WrappedManifest | null): WrappedManifest | null { + public async dispatch(wrappedManifest: WrappedManifest | null): Promise { if (wrappedManifest === null) { this.logger.debug('wrappedManifest was null, aborting dispatch'); return null; @@ -180,14 +184,20 @@ export class ManifestManager { while (paging) { const { items, total, page } = await this.datasourceService.list(this.savedObjectsClient, { + page: 1, + perPage: 20, kuery: 'ingest-datasources.package.name:endpoint', }); for (const datasource of items) { const { id, revision, updated_at, updated_by, ...newDatasource } = datasource; - if (newDatasource.inputs.length > 0) { - newDatasource.inputs[0].config.artifact_manifest = wrappedManifest.manifest.toEndpointFormat(); + if (newDatasource.inputs.length > 0 && newDatasource.inputs[0].config !== undefined) { + const artifactManifest = newDatasource.inputs[0].config.artifact_manifest ?? { + value: {}, + }; + artifactManifest.value = wrappedManifest.manifest.toEndpointFormat(); + newDatasource.inputs[0].config.artifact_manifest = artifactManifest; await this.datasourceService .update(this.savedObjectsClient, id, newDatasource) @@ -201,7 +211,7 @@ export class ManifestManager { }); } else { success = false; - this.logger.debug(`Datasource ${id} has no inputs.`); + this.logger.debug(`Datasource ${id} has no config.`); } } @@ -225,11 +235,11 @@ export class ManifestManager { const manifestClient = this.getManifestClient(wrappedManifest.manifest.getSchemaVersion()); // Commit the new manifest - if (wrappedManifest.manifest.getVersion() === undefined) { + if (wrappedManifest.manifest.getVersion() === 'v0') { await manifestClient.createManifest(wrappedManifest.manifest.toSavedObject()); } else { const version = wrappedManifest.manifest.getVersion(); - if (version === 'baseline') { + if (version === 'v0') { throw new Error('Updating existing manifest with baseline version. Bad state.'); } await manifestClient.updateManifest(wrappedManifest.manifest.toSavedObject(), { diff --git a/x-pack/plugins/security_solution/server/endpoint/types.ts b/x-pack/plugins/security_solution/server/endpoint/types.ts index fbcc5bc833d73..3c6630db8ebd8 100644 --- a/x-pack/plugins/security_solution/server/endpoint/types.ts +++ b/x-pack/plugins/security_solution/server/endpoint/types.ts @@ -4,8 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ import { LoggerFactory } from 'kibana/server'; -import { EndpointAppContextService } from './endpoint_app_context_services'; import { ConfigType } from '../config'; +import { EndpointAppContextService } from './endpoint_app_context_services'; /** * The context for Endpoint apps. diff --git a/x-pack/test/api_integration/apis/security_solution/exceptions.ts b/x-pack/test/api_integration/apis/security_solution/exceptions.ts index 9650318923d91..f4704f857824a 100644 --- a/x-pack/test/api_integration/apis/security_solution/exceptions.ts +++ b/x-pack/test/api_integration/apis/security_solution/exceptions.ts @@ -12,7 +12,6 @@ import { getSupertestWithoutAuth } from '../fleet/agents/services'; export default function (providerContext: FtrProviderContext) { const { getService } = providerContext; const esArchiver = getService('esArchiver'); - const supertest = getService('supertest'); const supertestWithoutAuth = getSupertestWithoutAuth(providerContext); const authKey = 'OFpuVDdISUJ3TEZ2a1VFUFFhVDM6TnU1U0JtbHJSeC12Rm9qQWpoSHlUZw=='; @@ -22,7 +21,7 @@ export default function (providerContext: FtrProviderContext) { it('should fail to find artifact with invalid hash', async () => { const { body } = await supertestWithoutAuth - .get('/api/endpoint/allowlist/download/endpoint-allowlist-windows-1.0.0/abcd') + .get('/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/abcd') .set('kbn-xsrf', 'xxx') .set('authorization', `ApiKey ${authKey}`) .send() @@ -32,7 +31,7 @@ export default function (providerContext: FtrProviderContext) { it('should download an artifact with correct hash', async () => { const { body } = await supertestWithoutAuth .get( - '/api/endpoint/allowlist/download/endpoint-allowlist-windows-1.0.0/1825fb19fcc6dc391cae0bc4a2e96dd7f728a0c3ae9e1469251ada67f9e1b975' + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/1825fb19fcc6dc391cae0bc4a2e96dd7f728a0c3ae9e1469251ada67f9e1b975' ) .set('kbn-xsrf', 'xxx') .set('authorization', `ApiKey ${authKey}`) diff --git a/x-pack/test/functional/es_archives/security_solution/exceptions/api_feature/exception_list/data.json b/x-pack/test/functional/es_archives/security_solution/exceptions/api_feature/exception_list/data.json index 7f51fbd0c3503..11656cb7b63e2 100644 --- a/x-pack/test/functional/es_archives/security_solution/exceptions/api_feature/exception_list/data.json +++ b/x-pack/test/functional/es_archives/security_solution/exceptions/api_feature/exception_list/data.json @@ -1,7 +1,7 @@ { "type": "doc", "value": { - "id": "securitySolution:endpoint:exceptions-artifact:endpoint-allowlist-macos-1.0.0-1825fb19fcc6dc391cae0bc4a2e96dd7f728a0c3ae9e1469251ada67f9e1b975", + "id": "securitySolution:endpoint:exceptions-artifact:endpoint-exceptionlist-macos-1.0.0-1825fb19fcc6dc391cae0bc4a2e96dd7f728a0c3ae9e1469251ada67f9e1b975", "index": ".kibana", "source": { "references": [ @@ -10,7 +10,7 @@ "body": "ý7zXZ\u0000\u0000\u0001i\"Þ6\u0002\u0000!\u0001\u0016\u0000\u0000\u0000t/å£\u0001\u0000\u0015{\"exceptions_list\":[]}\u0000\u0000\u000052¤\u0000\u0001*\u0016RÌ9»B™\r\u0001\u0000\u0000\u0000\u0000\u0001YZ", "created": 1593016187465, "encoding": "xz", - "identifier": "endpoint-allowlist-macos-1.0.0", + "identifier": "endpoint-exceptionlist-macos-1.0.0", "sha256": "1825fb19fcc6dc391cae0bc4a2e96dd7f728a0c3ae9e1469251ada67f9e1b975", "size": 22 }, @@ -31,9 +31,9 @@ "securitySolution:endpoint:exceptions-manifest": { "created": 1593183699663, "ids": [ - "endpoint-allowlist-linux-1.0.0-6b065fbf5d9ae82039ca20e2f3f1629d2cd9d8d3f3517d0f295f879bf939a269", - "endpoint-allowlist-macos-1.0.0-1825fb19fcc6dc391cae0bc4a2e96dd7f728a0c3ae9e1469251ada67f9e1b975", - "endpoint-allowlist-windows-1.0.0-dbba762118dadd31597eb2cf6a39ff5a45aa31b27c1671f8b4d1e403d2102a77" + "endpoint-exceptionlist-linux-1.0.0-6b065fbf5d9ae82039ca20e2f3f1629d2cd9d8d3f3517d0f295f879bf939a269", + "endpoint-exceptionlist-macos-1.0.0-1825fb19fcc6dc391cae0bc4a2e96dd7f728a0c3ae9e1469251ada67f9e1b975", + "endpoint-exceptionlist-windows-1.0.0-dbba762118dadd31597eb2cf6a39ff5a45aa31b27c1671f8b4d1e403d2102a77" ] }, "type": "securitySolution:endpoint:exceptions-manifest", From 67d756aa064d4b1105a0677e192093fa4e5da29e Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 30 Jun 2020 15:07:29 -0400 Subject: [PATCH 098/106] Fix integration test and add test for invalid api key --- .../server/endpoint/lib/artifacts/common.ts | 4 +-- .../lib/artifacts/saved_object_mappings.ts | 1 - .../artifacts/download_exception_list.test.ts | 8 +++--- .../artifacts/index.ts} | 28 ++++++++++++++----- .../artifacts/api_feature}/data.json | 12 ++++---- 5 files changed, 33 insertions(+), 20 deletions(-) rename x-pack/test/api_integration/apis/{security_solution/exceptions.ts => endpoint/artifacts/index.ts} (75%) rename x-pack/test/functional/es_archives/{security_solution/exceptions/api_feature/exception_list => endpoint/artifacts/api_feature}/data.json (90%) diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts index be255845fc5ca..4c3153ca0ef11 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts @@ -6,12 +6,12 @@ export const ArtifactConstants = { GLOBAL_ALLOWLIST_NAME: 'endpoint-exceptionlist', - SAVED_OBJECT_TYPE: 'securitySolution:endpoint:exceptions-artifact', + SAVED_OBJECT_TYPE: 'endpoint:exceptions-artifact', SUPPORTED_OPERATING_SYSTEMS: ['linux', 'macos', 'windows'], SCHEMA_VERSION: '1.0.0', }; export const ManifestConstants = { - SAVED_OBJECT_TYPE: 'securitySolution:endpoint:exceptions-manifest', + SAVED_OBJECT_TYPE: 'endpoint:exceptions-manifest', SCHEMA_VERSION: '1.0.0', }; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts index d8f7b2485e2a8..d38026fbcbbd9 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts @@ -13,7 +13,6 @@ export const manifestSavedObjectType = ManifestConstants.SAVED_OBJECT_TYPE; export const exceptionsArtifactSavedObjectMappings: SavedObjectsType['mappings'] = { properties: { - // e.g. 'global-whitelist-windows-1.0.0' identifier: { type: 'keyword', }, diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts index 02b9c7c3d89c9..4e73c4ddb8265 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts @@ -4,10 +4,10 @@ * you may not use this file except in compliance with the Elastic License. */ import { - IClusterClient, + ILegacyClusterClient, IRouter, SavedObjectsClientContract, - IScopedClusterClient, + ILegacyScopedClusterClient, RouteConfig, RequestHandler, KibanaResponseFactory, @@ -74,8 +74,8 @@ const AuthHeader = 'ApiKey cGVkVHVISUJURUR0OTN3VzBGaHI6TnU1U0JtbHJSeC12Rm9qQWpoS describe('test alerts route', () => { let routerMock: jest.Mocked; - let mockClusterClient: jest.Mocked; - let mockScopedClient: jest.Mocked; + let mockClusterClient: jest.Mocked; + let mockScopedClient: jest.Mocked; let mockSavedObjectClient: jest.Mocked; let mockResponse: jest.Mocked; // @ts-ignore diff --git a/x-pack/test/api_integration/apis/security_solution/exceptions.ts b/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts similarity index 75% rename from x-pack/test/api_integration/apis/security_solution/exceptions.ts rename to x-pack/test/api_integration/apis/endpoint/artifacts/index.ts index 70abbef1f662a..13bbf4f3c4223 100644 --- a/x-pack/test/api_integration/apis/security_solution/exceptions.ts +++ b/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts @@ -6,8 +6,8 @@ import expect from '@kbn/expect'; -import { FtrProviderContext } from '../../ftr_provider_context'; -import { getSupertestWithoutAuth, setupIngest } from '../fleet/agents/services'; +import { FtrProviderContext } from '../../../ftr_provider_context'; +import { getSupertestWithoutAuth, setupIngest } from '../../fleet/agents/services'; const exceptionListBody = { list_id: 'endpoint_list', @@ -21,6 +21,7 @@ const exceptionListBody = { export default function (providerContext: FtrProviderContext) { const { getService } = providerContext; + const supertest = getService('supertest'); const esArchiver = getService('esArchiver'); const supertestWithoutAuth = getSupertestWithoutAuth(providerContext); let agentAccessAPIKey: string; @@ -28,20 +29,21 @@ export default function (providerContext: FtrProviderContext) { describe('artifact download', () => { setupIngest(providerContext); before(async () => { - await esArchiver.load('security_solution/exceptions/api_feature/exception_list'); + await esArchiver.load('endpoint/artifacts/api_feature', { useCreate: true }); + const { body: enrollmentApiKeysResponse } = await supertest .get(`/api/ingest_manager/fleet/enrollment-api-keys`) .expect(200); + expect(enrollmentApiKeysResponse.list).length(2); - expect(enrollmentApiKeysResponse.list).length(1); const { body: enrollmentApiKeyResponse } = await supertest .get( `/api/ingest_manager/fleet/enrollment-api-keys/${enrollmentApiKeysResponse.list[0].id}` ) .expect(200); - expect(enrollmentApiKeyResponse.item).to.have.key('api_key'); const enrollmentAPIToken = enrollmentApiKeyResponse.item.api_key; + // 2. Enroll agent const { body: enrollmentResponse } = await supertestWithoutAuth .post(`/api/ingest_manager/fleet/agents/enroll`) @@ -56,9 +58,10 @@ export default function (providerContext: FtrProviderContext) { }) .expect(200); expect(enrollmentResponse.success).to.eql(true); + agentAccessAPIKey = enrollmentResponse.item.access_api_key; }); - after(() => esArchiver.unload('security_solution/exceptions/api_feature/exception_list')); + after(() => esArchiver.unload('endpoint/artifacts/api_feature')); it('should fail to find artifact with invalid hash', async () => { const { body } = await supertestWithoutAuth @@ -72,12 +75,23 @@ export default function (providerContext: FtrProviderContext) { it('should download an artifact with correct hash', async () => { const { body } = await supertestWithoutAuth .get( - '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/1825fb19fcc6dc391cae0bc4a2e96dd7f728a0c3ae9e1469251ada67f9e1b975' + '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-1.0.0/1825fb19fcc6dc391cae0bc4a2e96dd7f728a0c3ae9e1469251ada67f9e1b975' ) .set('kbn-xsrf', 'xxx') .set('authorization', `ApiKey ${agentAccessAPIKey}`) .send() .expect(200); }); + + it('should fail on invalid api key', async () => { + const { body } = await supertestWithoutAuth + .get( + '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-1.0.0/1825fb19fcc6dc391cae0bc4a2e96dd7f728a0c3ae9e1469251ada67f9e1b975' + ) + .set('kbn-xsrf', 'xxx') + .set('authorization', `ApiKey iNvAlId`) + .send() + .expect(401); + }); }); } diff --git a/x-pack/test/functional/es_archives/security_solution/exceptions/api_feature/exception_list/data.json b/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json similarity index 90% rename from x-pack/test/functional/es_archives/security_solution/exceptions/api_feature/exception_list/data.json rename to x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json index 11656cb7b63e2..ab2779dde6b25 100644 --- a/x-pack/test/functional/es_archives/security_solution/exceptions/api_feature/exception_list/data.json +++ b/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json @@ -1,12 +1,12 @@ { "type": "doc", "value": { - "id": "securitySolution:endpoint:exceptions-artifact:endpoint-exceptionlist-macos-1.0.0-1825fb19fcc6dc391cae0bc4a2e96dd7f728a0c3ae9e1469251ada67f9e1b975", + "id": "endpoint:exceptions-artifact:endpoint-exceptionlist-macos-1.0.0-1825fb19fcc6dc391cae0bc4a2e96dd7f728a0c3ae9e1469251ada67f9e1b975", "index": ".kibana", "source": { "references": [ ], - "securitySolution:endpoint:exceptions-artifact": { + "endpoint:exceptions-artifact": { "body": "ý7zXZ\u0000\u0000\u0001i\"Þ6\u0002\u0000!\u0001\u0016\u0000\u0000\u0000t/å£\u0001\u0000\u0015{\"exceptions_list\":[]}\u0000\u0000\u000052¤\u0000\u0001*\u0016RÌ9»B™\r\u0001\u0000\u0000\u0000\u0000\u0001YZ", "created": 1593016187465, "encoding": "xz", @@ -14,7 +14,7 @@ "sha256": "1825fb19fcc6dc391cae0bc4a2e96dd7f728a0c3ae9e1469251ada67f9e1b975", "size": 22 }, - "type": "securitySolution:endpoint:exceptions-artifact", + "type": "endpoint:exceptions-artifact", "updated_at": "2020-06-24T16:29:47.584Z" } } @@ -23,12 +23,12 @@ { "type": "doc", "value": { - "id": "securitySolution:endpoint:exceptions-manifest:endpoint-manifest-1.0.0", + "id": "endpoint:exceptions-manifest:endpoint-manifest-1.0.0", "index": ".kibana", "source": { "references": [ ], - "securitySolution:endpoint:exceptions-manifest": { + "endpoint:exceptions-manifest": { "created": 1593183699663, "ids": [ "endpoint-exceptionlist-linux-1.0.0-6b065fbf5d9ae82039ca20e2f3f1629d2cd9d8d3f3517d0f295f879bf939a269", @@ -36,7 +36,7 @@ "endpoint-exceptionlist-windows-1.0.0-dbba762118dadd31597eb2cf6a39ff5a45aa31b27c1671f8b4d1e403d2102a77" ] }, - "type": "securitySolution:endpoint:exceptions-manifest", + "type": "endpoint:exceptions-manifest", "updated_at": "2020-06-26T15:01:39.704Z" } } From eae1639fc03e4e031fc287fddb6efbcdd4817882 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 30 Jun 2020 15:13:02 -0400 Subject: [PATCH 099/106] minor fixup --- x-pack/plugins/lists/server/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/lists/server/index.ts b/x-pack/plugins/lists/server/index.ts index 8279095c6ffa8..31f22108028a6 100644 --- a/x-pack/plugins/lists/server/index.ts +++ b/x-pack/plugins/lists/server/index.ts @@ -10,8 +10,8 @@ import { ConfigSchema } from './config'; import { ListPlugin } from './plugin'; // exporting these since its required at top level in siem plugin -export { ExceptionListClient } from './services/exception_lists/exception_list_client'; export { ListClient } from './services/lists/list_client'; +export { ExceptionListClient } from './services/exception_lists/exception_list_client'; export { ListPluginSetup } from './types'; export const config = { schema: ConfigSchema }; From 77ffeb7bebf5412eb7327956bac58d3eea306cf8 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Wed, 1 Jul 2020 13:01:25 -0400 Subject: [PATCH 100/106] Remove compression --- package.json | 1 - renovate.json5 | 8 ----- .../server/endpoint/lib/artifacts/lists.ts | 27 +++------------- .../artifacts/download_exception_list.test.ts | 20 ++++++------ .../artifacts/download_exception_list.ts | 31 ++++++++++--------- .../endpoint/schemas/artifacts/common.ts | 2 +- .../response/download_artifact_schema.ts | 2 +- .../manifest_manager/manifest_manager.ts | 2 +- .../apis/endpoint/artifacts/index.ts | 26 +++++++--------- .../endpoint/artifacts/api_feature/data.json | 16 +++++----- yarn.lock | 17 ---------- 11 files changed, 53 insertions(+), 99 deletions(-) diff --git a/package.json b/package.json index 57af163440788..b520be4df6969 100644 --- a/package.json +++ b/package.json @@ -144,7 +144,6 @@ "@kbn/test-subj-selector": "0.2.1", "@kbn/ui-framework": "1.0.0", "@kbn/ui-shared-deps": "1.0.0", - "@types/lzma-native": "^4.0.0", "JSONStream": "1.3.5", "abortcontroller-polyfill": "^1.4.0", "accept": "3.0.2", diff --git a/renovate.json5 b/renovate.json5 index 1efb84447c6f4..49a255d60f29e 100644 --- a/renovate.json5 +++ b/renovate.json5 @@ -589,14 +589,6 @@ '@types/lru-cache', ], }, - { - groupSlug: 'lzma-native', - groupName: 'lzma-native related packages', - packageNames: [ - 'lzma-native', - '@types/lzma-native', - ], - }, { groupSlug: 'mapbox-gl', groupName: 'mapbox-gl related packages', diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts index e4aa7328af615..0b49bab7014de 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts @@ -5,7 +5,6 @@ */ import { createHash } from 'crypto'; -import lzma from 'lzma-native'; import { validate } from '../../../../common/validate'; import { @@ -32,19 +31,16 @@ export async function buildArtifact( os: string, schemaVersion: string ): Promise { - const compressedExceptions: Buffer = await compressExceptionList(exceptions); - - const sha256 = createHash('sha256') - .update(compressedExceptions.toString('utf8'), 'utf8') - .digest('hex'); + const exceptionsBuffer = Buffer.from(JSON.stringify(exceptions)); + const sha256 = createHash('sha256').update(exceptionsBuffer.toString()).digest('hex'); return { identifier: `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-${os}-${schemaVersion}`, sha256, - encoding: 'xz', + encoding: 'application/json', created: Date.now(), - body: compressedExceptions.toString('binary'), - size: Buffer.from(JSON.stringify(exceptions)).byteLength, + body: exceptionsBuffer.toString('base64'), + size: exceptionsBuffer.byteLength, }; } @@ -159,16 +155,3 @@ function translateEntry( } return translatedEntry || undefined; } - -/** - * Compresses the exception list - */ -export function compressExceptionList( - exceptionList: WrappedTranslatedExceptionList -): Promise { - return new Promise((resolve, reject) => { - lzma.compress(JSON.stringify(exceptionList), undefined, (res: Buffer) => { - resolve(res); - }); - }); -} diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts index 4e73c4ddb8265..540976134d8ae 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts @@ -22,7 +22,6 @@ import { loggingSystemMock, } from 'src/core/server/mocks'; import { ExceptionsCache } from '../../lib/artifacts/cache'; -import { compressExceptionList } from '../../lib/artifacts/lists'; import { ArtifactConstants } from '../../lib/artifacts'; import { registerDownloadExceptionListRoute } from './download_exception_list'; import { EndpointAppContextService } from '../../endpoint_app_context_services'; @@ -114,7 +113,7 @@ describe('test alerts route', () => { ); }); - it('should serve the compressed artifact to download', async () => { + it('should serve the artifact to download', async () => { const mockRequest = httpServerMock.createKibanaRequest({ path: `/api/endpoint/artifacts/download/${mockArtifactName}/123456`, method: 'get', @@ -125,7 +124,6 @@ describe('test alerts route', () => { }); // Mock the SavedObjectsClient get response for fetching the artifact - const mockCompressedArtifact = await compressExceptionList(expectedEndpointExceptions); const mockArtifact = { id: '2468', type: 'test', @@ -134,9 +132,9 @@ describe('test alerts route', () => { identifier: mockArtifactName, schemaVersion: '1.0.0', sha256: '123456', - encoding: 'xz', + encoding: 'application/json', created: Date.now(), - body: mockCompressedArtifact, + body: Buffer.from(JSON.stringify(expectedEndpointExceptions)).toString('base64'), size: 100, }, }; @@ -162,14 +160,14 @@ describe('test alerts route', () => { ); const expectedHeaders = { - 'content-encoding': 'xz', - 'content-disposition': `attachment; filename=${mockArtifactName}.xz`, + 'content-encoding': 'application/json', + 'content-disposition': `attachment; filename=${mockArtifactName}.json`, }; expect(mockResponse.ok).toBeCalled(); expect(mockResponse.ok.mock.calls[0][0]?.headers).toEqual(expectedHeaders); - const compressedArtifact = mockResponse.ok.mock.calls[0][0]?.body; - expect(compressedArtifact).toEqual(mockCompressedArtifact); + const artifact = mockResponse.ok.mock.calls[0][0]?.body; + expect(artifact).toEqual(Buffer.from(mockArtifact.attributes.body, 'base64').toString()); }); it('should handle fetching a non-existent artifact', async () => { @@ -217,9 +215,9 @@ describe('test alerts route', () => { }); // Add to the download cache - const mockCompressedArtifact = await compressExceptionList(expectedEndpointExceptions); + const mockArtifact = expectedEndpointExceptions; const cacheKey = `${mockArtifactName}-${mockSha}`; - cache.set(cacheKey, mockCompressedArtifact.toString('binary')); + cache.set(cacheKey, JSON.stringify(mockArtifact)); [routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) => path.startsWith('/api/endpoint/artifacts/download') diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts index 8bf8a98b8d8f4..337393e768a8f 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts @@ -4,7 +4,13 @@ * you may not use this file except in compliance with the Elastic License. */ -import { IRouter, SavedObjectsClientContract, HttpResponseOptions } from 'src/core/server'; +import { + IRouter, + SavedObjectsClientContract, + HttpResponseOptions, + IKibanaResponse, + SavedObject, +} from 'src/core/server'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths import { authenticateAgentWithAccessToken } from '../../../../../ingest_manager/server/services/agents/authenticate'; import { validate } from '../../../../common/validate'; @@ -21,7 +27,7 @@ import { EndpointAppContext } from '../../types'; const allowlistBaseRoute: string = '/api/endpoint/artifacts'; /** - * Registers the exception list route to enable sensors to download a compressed allowlist + * Registers the exception list route to enable sensors to download an allowlist artifact */ export function registerDownloadExceptionListRoute( router: IRouter, @@ -56,17 +62,16 @@ export function registerDownloadExceptionListRoute( } } - const buildAndValidateResponse = (artName: string, body: string): object => { - const artifact = { - body: Buffer.from(body, 'binary'), + const buildAndValidateResponse = (artName: string, body: string): IKibanaResponse => { + const artifact: HttpResponseOptions = { + body, headers: { - 'content-encoding': 'xz', - 'content-disposition': `attachment; filename=${artName}.xz`, + 'content-encoding': 'application/json', + 'content-disposition': `attachment; filename=${artName}.json`, }, }; const [validated, errors] = validate(artifact, downloadArtifactResponseSchema); - if (errors !== null || validated === null) { return res.internalError({ body: errors! }); } else { @@ -84,12 +89,10 @@ export function registerDownloadExceptionListRoute( logger.debug(`Cache MISS artifact ${id}`); return scopedSOClient .get(ArtifactConstants.SAVED_OBJECT_TYPE, id) - .then((artifact) => { - cache.set(id, artifact.attributes.body); - return buildAndValidateResponse( - artifact.attributes.identifier, - artifact.attributes.body - ); + .then((artifact: SavedObject) => { + const body = Buffer.from(artifact.attributes.body, 'base64').toString(); + cache.set(id, body); + return buildAndValidateResponse(artifact.attributes.identifier, body); }) .catch((err) => { if (err?.output?.statusCode === 404) { diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/common.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/common.ts index b6a6f7b4af9dc..3c066e150288a 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/common.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/common.ts @@ -11,7 +11,7 @@ export const body = t.string; export const created = t.number; // TODO: Make this into an ISO Date string check export const encoding = t.keyof({ - xz: null, + 'application/json': null, }); export const schemaVersion = t.keyof({ diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/response/download_artifact_schema.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/response/download_artifact_schema.ts index 1323c67d1d029..537f7707889e4 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/response/download_artifact_schema.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/response/download_artifact_schema.ts @@ -7,7 +7,7 @@ import * as t from 'io-ts'; import { encoding } from '../common'; -const body = t.unknown; // TODO: create Buffer type +const body = t.string; const headers = t.exact( t.type({ 'content-encoding': encoding, diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts index 3f5d0e92836d4..55edbd251f8c6 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts @@ -139,7 +139,7 @@ export class ManifestManager { const artifact = newManifest.getArtifact(diff.id); try { await this.artifactClient.createArtifact(artifact); - // Cache the compressed body of the artifact + // Cache the body of the artifact this.cache.set(diff.id, artifact.body); } catch (err) { if (err.status === 409) { diff --git a/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts b/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts index 13bbf4f3c4223..2721592ba3350 100644 --- a/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts +++ b/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts @@ -9,16 +9,6 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../../ftr_provider_context'; import { getSupertestWithoutAuth, setupIngest } from '../../fleet/agents/services'; -const exceptionListBody = { - list_id: 'endpoint_list', - _tags: ['endpoint', 'process', 'malware', 'os:linux'], - tags: ['user added string for a tag', 'malware'], - type: 'endpoint', - description: 'This is a sample agnostic endpoint type exception', - name: 'Sample Endpoint Exception List', - namespace_type: 'agnostic', -}; - export default function (providerContext: FtrProviderContext) { const { getService } = providerContext; const supertest = getService('supertest'); @@ -52,7 +42,13 @@ export default function (providerContext: FtrProviderContext) { .send({ type: 'PERMANENT', metadata: { - local: {}, + local: { + elastic: { + agent: { + version: '7.0.0', + }, + }, + }, user_provided: {}, }, }) @@ -64,7 +60,7 @@ export default function (providerContext: FtrProviderContext) { after(() => esArchiver.unload('endpoint/artifacts/api_feature')); it('should fail to find artifact with invalid hash', async () => { - const { body } = await supertestWithoutAuth + await supertestWithoutAuth .get('/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/abcd') .set('kbn-xsrf', 'xxx') .set('authorization', `ApiKey ${agentAccessAPIKey}`) @@ -73,9 +69,9 @@ export default function (providerContext: FtrProviderContext) { }); it('should download an artifact with correct hash', async () => { - const { body } = await supertestWithoutAuth + await supertestWithoutAuth .get( - '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-1.0.0/1825fb19fcc6dc391cae0bc4a2e96dd7f728a0c3ae9e1469251ada67f9e1b975' + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d' ) .set('kbn-xsrf', 'xxx') .set('authorization', `ApiKey ${agentAccessAPIKey}`) @@ -84,7 +80,7 @@ export default function (providerContext: FtrProviderContext) { }); it('should fail on invalid api key', async () => { - const { body } = await supertestWithoutAuth + await supertestWithoutAuth .get( '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-1.0.0/1825fb19fcc6dc391cae0bc4a2e96dd7f728a0c3ae9e1469251ada67f9e1b975' ) diff --git a/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json b/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json index ab2779dde6b25..a886b60e7e0dc 100644 --- a/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json +++ b/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json @@ -1,17 +1,17 @@ { "type": "doc", "value": { - "id": "endpoint:exceptions-artifact:endpoint-exceptionlist-macos-1.0.0-1825fb19fcc6dc391cae0bc4a2e96dd7f728a0c3ae9e1469251ada67f9e1b975", + "id": "endpoint:exceptions-artifact:endpoint-exceptionlist-linux-1.0.0-a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d", "index": ".kibana", "source": { "references": [ ], "endpoint:exceptions-artifact": { - "body": "ý7zXZ\u0000\u0000\u0001i\"Þ6\u0002\u0000!\u0001\u0016\u0000\u0000\u0000t/å£\u0001\u0000\u0015{\"exceptions_list\":[]}\u0000\u0000\u000052¤\u0000\u0001*\u0016RÌ9»B™\r\u0001\u0000\u0000\u0000\u0000\u0001YZ", + "body": "eyJleGNlcHRpb25zX2xpc3QiOltdfQ==", "created": 1593016187465, - "encoding": "xz", - "identifier": "endpoint-exceptionlist-macos-1.0.0", - "sha256": "1825fb19fcc6dc391cae0bc4a2e96dd7f728a0c3ae9e1469251ada67f9e1b975", + "encoding": "application/json", + "identifier": "endpoint-exceptionlist-linux-1.0.0", + "sha256": "a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d", "size": 22 }, "type": "endpoint:exceptions-artifact", @@ -31,9 +31,9 @@ "endpoint:exceptions-manifest": { "created": 1593183699663, "ids": [ - "endpoint-exceptionlist-linux-1.0.0-6b065fbf5d9ae82039ca20e2f3f1629d2cd9d8d3f3517d0f295f879bf939a269", - "endpoint-exceptionlist-macos-1.0.0-1825fb19fcc6dc391cae0bc4a2e96dd7f728a0c3ae9e1469251ada67f9e1b975", - "endpoint-exceptionlist-windows-1.0.0-dbba762118dadd31597eb2cf6a39ff5a45aa31b27c1671f8b4d1e403d2102a77" + "endpoint-exceptionlist-linux-1.0.0-a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d", + "endpoint-exceptionlist-macos-1.0.0-a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d", + "endpoint-exceptionlist-windows-1.0.0-a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d" ] }, "type": "endpoint:exceptions-manifest", diff --git a/yarn.lock b/yarn.lock index 825eee85b5f33..a4b47837539c6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5361,13 +5361,6 @@ resolved "https://registry.yarnpkg.com/@types/lru-cache/-/lru-cache-5.1.0.tgz#57f228f2b80c046b4a1bd5cac031f81f207f4f03" integrity sha512-RaE0B+14ToE4l6UqdarKPnXwVDuigfFv+5j9Dze/Nqr23yyuqdNvzcZi3xB+3Agvi5R4EOgAksfv3lXX4vBt9w== -"@types/lzma-native@4.0.0", "@types/lzma-native@^4.0.0": - version "4.0.0" - resolved "https://registry.yarnpkg.com/@types/lzma-native/-/lzma-native-4.0.0.tgz#86440e70b1431f2bc1a470eb0ea8bcfc5788fe14" - integrity sha512-9mBLyFWJe8/whIRjGwca5pbXv7ANzPj7KNgsgCU/l5z8xcyIe6LNae+iKdT5rjnXcXGOwyW/PKYt0N6+7N5QlA== - dependencies: - "@types/node" "*" - "@types/mapbox-gl@^1.9.1": version "1.9.1" resolved "https://registry.yarnpkg.com/@types/mapbox-gl/-/mapbox-gl-1.9.1.tgz#78b62f8a1ead78bc525a4c1db84bb71fa0fcc579" @@ -21156,16 +21149,6 @@ lz-string@^1.4.4: resolved "https://registry.yarnpkg.com/lz-string/-/lz-string-1.4.4.tgz#c0d8eaf36059f705796e1e344811cf4c498d3a26" integrity sha1-wNjq82BZ9wV5bh40SBHPTEmNOiY= -lzma-native@6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/lzma-native/-/lzma-native-6.0.0.tgz#e49d57023c81c4e2a1cbe5ebcf223618457ce31d" - integrity sha512-rf5f4opPymsPHotgY2d0cUP3kbVxERSxWDGEbi2gnbnxuWGokFrBaQ02Oe9pssIwsgp0r0PnbSNg7VPY3AYe7w== - dependencies: - node-addon-api "^1.6.0" - node-pre-gyp "^0.11.0" - readable-stream "^2.3.5" - rimraf "^2.7.1" - macos-release@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/macos-release/-/macos-release-1.1.0.tgz#831945e29365b470aa8724b0ab36c8f8959d10fb" From 975839ef358d39263288884cc3732be0cd22395b Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Wed, 1 Jul 2020 13:05:06 -0400 Subject: [PATCH 101/106] Update task interval --- .../security_solution/server/endpoint/lib/artifacts/task.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts index 100e762ae80c7..08d02e70dac16 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts @@ -59,8 +59,7 @@ export class ManifestTask { taskType: ManifestTaskConstants.TYPE, scope: ['securitySolution'], schedule: { - // TODO: change this to '60s' before merging - interval: '5s', + interval: '60s', }, state: {}, params: { version: ManifestTaskConstants.VERSION }, From ddafd303e53d1732cbfc8516981d20830ca13d9d Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Wed, 1 Jul 2020 13:15:35 -0400 Subject: [PATCH 102/106] Removing .text suffix from translated list --- .../server/endpoint/lib/artifacts/lists.test.ts | 4 ++-- .../security_solution/server/endpoint/lib/artifacts/lists.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts index ada2f15a79343..738890fb4038f 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts @@ -102,7 +102,7 @@ describe('buildEventTypeSignal', () => { const expectedEndpointExceptions = { exceptions_list: [ { - field: 'server.domain.text', + field: 'server.domain', operator: 'included', type: 'exact_caseless', value: 'DOMAIN', @@ -114,7 +114,7 @@ describe('buildEventTypeSignal', () => { value: '192.168.1.1', }, { - field: 'host.hostname.text', + field: 'host.hostname', operator: 'included', type: 'exact_caseless_any', value: ['estc', 'kibana'], diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts index 0b49bab7014de..7fd057afdbd55 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts @@ -134,7 +134,7 @@ function translateEntry( case 'match': { const e = (entry as unknown) as EntryMatch; translatedEntry = { - field: e.field, + field: e.field.endsWith('.text') ? e.field.substring(0, e.field.length - 5) : e.field, operator: e.operator, type: e.field.endsWith('.text') ? 'exact_caseless' : 'exact_cased', value: e.value, @@ -145,7 +145,7 @@ function translateEntry( { const e = (entry as unknown) as EntryMatchAny; translatedEntry = { - field: e.field, + field: e.field.endsWith('.text') ? e.field.substring(0, e.field.length - 5) : e.field, operator: e.operator, type: e.field.endsWith('.text') ? 'exact_caseless_any' : 'exact_cased_any', value: e.value, From 2f74e05319be24a1c82667b59975dbb70491d7b6 Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Wed, 1 Jul 2020 13:27:20 -0400 Subject: [PATCH 103/106] Fixes hashes for unit tests --- .../endpoint/lib/artifacts/manifest.test.ts | 36 +++++++++---------- .../lib/artifacts/manifest_entry.test.ts | 10 +++--- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts index f4227d3f8e3d5..0434e3d8ffcb2 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts @@ -55,22 +55,22 @@ describe('manifest', () => { expect(manifest1.toEndpointFormat()).toStrictEqual({ artifacts: { 'endpoint-exceptionlist-linux-1.0.0': { - sha256: '222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', + sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', size: 268, url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', }, 'endpoint-exceptionlist-macos-1.0.0': { - sha256: '222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', + sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', size: 268, url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-1.0.0/222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', + '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', }, 'endpoint-exceptionlist-windows-1.0.0': { - sha256: '222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', + sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', size: 268, url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', }, }, manifest_version: 'abcd', @@ -82,9 +82,9 @@ describe('manifest', () => { expect(manifest1.toSavedObject()).toStrictEqual({ created: now.getTime(), ids: [ - 'endpoint-exceptionlist-linux-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', - 'endpoint-exceptionlist-macos-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', - 'endpoint-exceptionlist-windows-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', + 'endpoint-exceptionlist-linux-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + 'endpoint-exceptionlist-macos-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + 'endpoint-exceptionlist-windows-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', ], }); }); @@ -94,12 +94,12 @@ describe('manifest', () => { expect(diffs).toEqual([ { id: - 'endpoint-exceptionlist-linux-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', + 'endpoint-exceptionlist-linux-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', type: 'delete', }, { id: - 'endpoint-exceptionlist-linux-1.0.0-03114bf3dc2258f0def5beaf675242b68b428c96eefab5f6c5533f0d8e4deb0b', + 'endpoint-exceptionlist-linux-1.0.0-69328f83418f4957470640ed6cc605be6abb5fe80e0e388fd74f9764ad7ed5d1', type: 'add', }, ]); @@ -115,15 +115,15 @@ describe('manifest', () => { const entries = manifest1.getEntries(); const keys = Object.keys(entries); expect(keys).toEqual([ - 'endpoint-exceptionlist-linux-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', - 'endpoint-exceptionlist-macos-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', - 'endpoint-exceptionlist-windows-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', + 'endpoint-exceptionlist-linux-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + 'endpoint-exceptionlist-macos-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + 'endpoint-exceptionlist-windows-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', ]); }); test('Manifest returns true if contains artifact', async () => { const found = manifest1.contains( - 'endpoint-exceptionlist-macos-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466' + 'endpoint-exceptionlist-macos-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' ); expect(found).toEqual(true); }); @@ -132,17 +132,17 @@ describe('manifest', () => { const manifest = Manifest.fromArtifacts(artifacts, '1.0.0', 'v0'); expect( manifest.contains( - 'endpoint-exceptionlist-linux-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466' + 'endpoint-exceptionlist-linux-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' ) ).toEqual(true); expect( manifest.contains( - 'endpoint-exceptionlist-macos-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466' + 'endpoint-exceptionlist-macos-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' ) ).toEqual(true); expect( manifest.contains( - 'endpoint-exceptionlist-windows-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466' + 'endpoint-exceptionlist-windows-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' ) ).toEqual(true); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts index 8c37d689a9d34..34bd2b0f388e1 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts @@ -24,7 +24,7 @@ describe('manifest_entry', () => { test('Correct doc_id is returned', () => { expect(manifestEntry.getDocId()).toEqual( - 'endpoint-exceptionlist-windows-1.0.0-222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466' + 'endpoint-exceptionlist-windows-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' ); }); @@ -34,7 +34,7 @@ describe('manifest_entry', () => { test('Correct sha256 is returned', () => { expect(manifestEntry.getSha256()).toEqual( - '222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466' + '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' ); }); @@ -44,7 +44,7 @@ describe('manifest_entry', () => { test('Correct url is returned', () => { expect(manifestEntry.getUrl()).toEqual( - '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466' + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' ); }); @@ -54,10 +54,10 @@ describe('manifest_entry', () => { test('Correct record is returned', () => { expect(manifestEntry.getRecord()).toEqual({ - sha256: '222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', + sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', size: 268, url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/222c07e7741e5d8371958fadc5636141bfa330926886b54b233e6a4ecac86466', + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', }); }); }); From d517feaa9362a046a56951b6807310e54e59abfa Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Wed, 1 Jul 2020 13:56:09 -0400 Subject: [PATCH 104/106] clean up yarn.lock --- yarn.lock | 5 ----- 1 file changed, 5 deletions(-) diff --git a/yarn.lock b/yarn.lock index a4b47837539c6..ee61303e85f4a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -22439,11 +22439,6 @@ nock@12.0.3: lodash "^4.17.13" propagate "^2.0.0" -node-addon-api@^1.6.0: - version "1.7.1" - resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-1.7.1.tgz#cf813cd69bb8d9100f6bdca6755fc268f54ac492" - integrity sha512-2+DuKodWvwRTrCfKOeR24KIc5unKjOh8mz17NCzVnHWfjAdDqbfbjqh7gUT+BkXBRQM52+xCHciKWonJ3CbJMQ== - node-dir@^0.1.10: version "0.1.17" resolved "https://registry.yarnpkg.com/node-dir/-/node-dir-0.1.17.tgz#5f5665d93351335caabef8f1c554516cf5f1e4e5" From 3ed5c48df3fe7c250acb1dc633888dddd08d1ba7 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Wed, 1 Jul 2020 14:20:47 -0400 Subject: [PATCH 105/106] Remove lzma-native from package.json --- x-pack/plugins/security_solution/package.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/package.json b/x-pack/plugins/security_solution/package.json index ac0b4645f8edc..1ce5243bf7950 100644 --- a/x-pack/plugins/security_solution/package.json +++ b/x-pack/plugins/security_solution/package.json @@ -20,10 +20,8 @@ "@types/rbush": "^3.0.0", "@types/seedrandom": ">=2.0.0 <4.0.0", "lodash": "^4.17.15", - "lzma-native": "6.0.0", - "@types/lzma-native": "4.0.0", "querystring": "^0.2.0", "rbush": "^3.0.1", "redux-devtools-extension": "^2.13.8" } -} \ No newline at end of file +} From 0ec1f2beb1f6d69bd03045ee01e908908e69c065 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Wed, 1 Jul 2020 15:29:52 -0400 Subject: [PATCH 106/106] missed updating one of the tests --- .../artifacts/manifest_manager/manifest_manager.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts index 3c1ff199b05ab..0b1ae98d808cd 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts @@ -16,7 +16,7 @@ describe('manifest_manager', () => { expect(manifestWrapper!.diffs).toEqual([ { id: - 'endpoint-exceptionlist-linux-1.0.0-a0b2886af05849e1e7e7b05bd6e38ea2e2de6566bfb5f4bdbdeda8236de0ff5c', + 'endpoint-exceptionlist-linux-1.0.0-d34a1f6659bd86fc2023d7477aa2e5d2055c9c0fb0a0f10fae76bf8b94bebe49', type: 'add', }, ]);