From fac80fd061c39c2e3624662f767a056bc8506664 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Wed, 29 Jul 2020 00:40:07 -0400 Subject: [PATCH] [Security Solution][Exceptions] Use semantic version for manifest version + Scaling Tweaks (#73388) (#73610) * Manifest version is semantic version * Configurable task interval * Use task interval over scheduled when provided * Fix crash on download of large artifact * Don't need to generate linux artifacts * Configurable artifact validation * Test fixes * Test fixes * Type/test fixes * Final tweaks * Remove linux endpoint exception generation from UI * Fix paging so that we stop before 10k * Fix pagination * Fix pagination test Co-authored-by: Elastic Machine Co-authored-by: Elastic Machine --- .../common/endpoint/generate_data.ts | 2 +- .../common/endpoint/schema/common.ts | 5 +- .../common/endpoint/schema/manifest.ts | 4 +- .../exceptions/add_exception_modal/index.tsx | 2 +- .../security_solution/server/config.ts | 6 ++ .../server/endpoint/config.ts | 6 ++ .../endpoint/ingest_integration.test.ts | 25 +---- .../server/endpoint/ingest_integration.ts | 10 +- .../server/endpoint/lib/artifacts/common.ts | 4 +- .../endpoint/lib/artifacts/lists.test.ts | 12 ++- .../server/endpoint/lib/artifacts/lists.ts | 9 +- .../endpoint/lib/artifacts/manifest.test.ts | 49 +++------ .../server/endpoint/lib/artifacts/manifest.ts | 100 +++++++----------- .../server/endpoint/lib/artifacts/mocks.ts | 32 ++---- .../lib/artifacts/saved_object_mappings.ts | 8 +- .../endpoint/lib/artifacts/task.test.ts | 2 +- .../server/endpoint/lib/artifacts/task.ts | 31 ++++-- .../artifacts/download_exception_list.ts | 10 +- .../endpoint/schemas/artifacts/manifest.ts | 23 ++++ .../schemas/artifacts/saved_objects.mock.ts | 2 + .../schemas/artifacts/saved_objects.ts | 4 + .../services/artifacts/manifest_client.ts | 1 - .../manifest_manager/manifest_manager.test.ts | 82 ++++---------- .../manifest_manager/manifest_manager.ts | 62 +++++------ .../routes/__mocks__/index.ts | 2 + .../endpoint/artifacts/api_feature/data.json | 28 +---- .../apps/endpoint/policy_details.ts | 12 --- .../apis/artifacts/index.ts | 78 ++++++++++++-- 28 files changed, 284 insertions(+), 327 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/manifest.ts diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts index 97ac5c9030a3d..9a92270fc9c14 100644 --- a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts @@ -1041,7 +1041,7 @@ export class EndpointDocGenerator { config: { artifact_manifest: { value: { - manifest_version: 'WzAsMF0=', + manifest_version: '1.0.0', schema_version: 'v1', artifacts: {}, }, diff --git a/x-pack/plugins/security_solution/common/endpoint/schema/common.ts b/x-pack/plugins/security_solution/common/endpoint/schema/common.ts index 8f2ea1f8a6452..1c910927a7afa 100644 --- a/x-pack/plugins/security_solution/common/endpoint/schema/common.ts +++ b/x-pack/plugins/security_solution/common/endpoint/schema/common.ts @@ -23,8 +23,6 @@ export const encryptionAlgorithm = t.keyof({ export const identifier = t.string; -export const manifestVersion = t.string; - export const manifestSchemaVersion = t.keyof({ v1: null, }); @@ -34,4 +32,7 @@ export const relativeUrl = t.string; export const sha256 = t.string; +export const semanticVersion = t.string; +export type SemanticVersion = t.TypeOf; + export const size = t.number; diff --git a/x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts b/x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts index f8bb8b70f2d5b..f03db881837d5 100644 --- a/x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts +++ b/x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts @@ -11,9 +11,9 @@ import { encryptionAlgorithm, identifier, manifestSchemaVersion, - manifestVersion, relativeUrl, sha256, + semanticVersion, size, } from './common'; @@ -50,7 +50,7 @@ export type ManifestEntryDispatchSchema = t.TypeOf { - const osDefaults = ['windows', 'macos', 'linux']; + const osDefaults = ['windows', 'macos']; if (alertData) { const osTypes = getMappedNonEcsValue({ data: alertData.nonEcsData, diff --git a/x-pack/plugins/security_solution/server/config.ts b/x-pack/plugins/security_solution/server/config.ts index f7d961ae3ec5c..e2c06ae9f931f 100644 --- a/x-pack/plugins/security_solution/server/config.ts +++ b/x-pack/plugins/security_solution/server/config.ts @@ -29,6 +29,12 @@ export const configSchema = schema.object({ from: schema.string({ defaultValue: 'now-15m' }), to: schema.string({ defaultValue: 'now' }), }), + + /** + * Artifacts Configuration + */ + packagerTaskInterval: schema.string({ defaultValue: '60s' }), + validateArtifactDownloads: schema.boolean({ defaultValue: true }), }); export const createConfig$ = (context: PluginInitializerContext) => diff --git a/x-pack/plugins/security_solution/server/endpoint/config.ts b/x-pack/plugins/security_solution/server/endpoint/config.ts index 908e14468c5c7..6a3644f7aaf71 100644 --- a/x-pack/plugins/security_solution/server/endpoint/config.ts +++ b/x-pack/plugins/security_solution/server/endpoint/config.ts @@ -27,6 +27,12 @@ export const EndpointConfigSchema = schema.object({ from: schema.string({ defaultValue: 'now-15m' }), to: schema.string({ defaultValue: 'now' }), }), + + /** + * Artifacts Configuration + */ + packagerTaskInterval: schema.string({ defaultValue: '60s' }), + validateArtifactDownloads: schema.boolean({ defaultValue: true }), }); export function createConfig$(context: PluginInitializerContext) { diff --git a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.test.ts b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.test.ts index be749b2ebd25a..03999715dfc71 100644 --- a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.test.ts @@ -12,7 +12,6 @@ import { ManifestManagerMockType, } from './services/artifacts/manifest_manager/manifest_manager.mock'; import { getPackageConfigCreateCallback } from './ingest_integration'; -import { ManifestConstants } from './lib/artifacts'; describe('ingest_integration tests ', () => { describe('ingest_integration sanity checks', () => { @@ -30,16 +29,6 @@ describe('ingest_integration tests ', () => { expect(newPolicyConfig.inputs[0]!.config!.policy.value).toEqual(policyConfigFactory()); expect(newPolicyConfig.inputs[0]!.config!.artifact_manifest.value).toEqual({ artifacts: { - 'endpoint-exceptionlist-linux-v1': { - compression_algorithm: 'zlib', - decoded_sha256: 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', - decoded_size: 14, - encoded_sha256: 'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda', - encoded_size: 22, - encryption_algorithm: 'none', - relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', - }, 'endpoint-exceptionlist-macos-v1': { compression_algorithm: 'zlib', decoded_sha256: 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', @@ -61,7 +50,7 @@ describe('ingest_integration tests ', () => { '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', }, }, - manifest_version: 'a9b7ef358a363f327f479e31efc4f228b2277a7fb4d1914ca9b4e7ca9ffcf537', + manifest_version: '1.0.0', schema_version: 'v1', }); }); @@ -70,9 +59,7 @@ describe('ingest_integration tests ', () => { const logger = loggingSystemMock.create().get('ingest_integration.test'); const manifestManager = getManifestManagerMock(); manifestManager.pushArtifacts = jest.fn().mockResolvedValue([new Error('error updating')]); - const lastComputed = await manifestManager.getLastComputedManifest( - ManifestConstants.SCHEMA_VERSION - ); + const lastComputed = await manifestManager.getLastComputedManifest(); const callback = getPackageConfigCreateCallback(logger, manifestManager); const policyConfig = createNewPackageConfigMock(); @@ -90,9 +77,7 @@ describe('ingest_integration tests ', () => { const manifestManager = getManifestManagerMock({ mockType: ManifestManagerMockType.InitialSystemState, }); - const lastComputed = await manifestManager.getLastComputedManifest( - ManifestConstants.SCHEMA_VERSION - ); + const lastComputed = await manifestManager.getLastComputedManifest(); expect(lastComputed).toEqual(null); manifestManager.buildNewManifest = jest.fn().mockRejectedValue(new Error('abcd')); @@ -107,9 +92,7 @@ describe('ingest_integration tests ', () => { test('subsequent policy creations succeed', async () => { const logger = loggingSystemMock.create().get('ingest_integration.test'); const manifestManager = getManifestManagerMock(); - const lastComputed = await manifestManager.getLastComputedManifest( - ManifestConstants.SCHEMA_VERSION - ); + const lastComputed = await manifestManager.getLastComputedManifest(); manifestManager.buildNewManifest = jest.fn().mockResolvedValue(lastComputed); // no diffs const callback = getPackageConfigCreateCallback(logger, manifestManager); diff --git a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts index 11d4b12d0b76a..695267f322857 100644 --- a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts +++ b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts @@ -10,7 +10,7 @@ import { factory as policyConfigFactory } from '../../common/endpoint/models/pol import { NewPolicyData } from '../../common/endpoint/types'; import { ManifestManager } from './services/artifacts'; import { Manifest } from './lib/artifacts'; -import { reportErrors, ManifestConstants } from './lib/artifacts/common'; +import { reportErrors } from './lib/artifacts/common'; import { InternalArtifactCompleteSchema } from './schemas/artifacts'; import { manifestDispatchSchema } from '../../common/endpoint/schema/manifest'; @@ -18,14 +18,14 @@ const getManifest = async (logger: Logger, manifestManager: ManifestManager): Pr let manifest: Manifest | null = null; try { - manifest = await manifestManager.getLastComputedManifest(ManifestConstants.SCHEMA_VERSION); + manifest = await manifestManager.getLastComputedManifest(); // If we have not yet computed a manifest, then we have to do so now. This should only happen // once. if (manifest == null) { // New computed manifest based on current state of exception list - const newManifest = await manifestManager.buildNewManifest(ManifestConstants.SCHEMA_VERSION); - const diffs = newManifest.diff(Manifest.getDefault(ManifestConstants.SCHEMA_VERSION)); + const newManifest = await manifestManager.buildNewManifest(); + const diffs = newManifest.diff(Manifest.getDefault()); // Compress new artifacts const adds = diffs.filter((diff) => diff.type === 'add').map((diff) => diff.id); @@ -63,7 +63,7 @@ const getManifest = async (logger: Logger, manifestManager: ManifestManager): Pr logger.error(err); } - return manifest ?? Manifest.getDefault(ManifestConstants.SCHEMA_VERSION); + return manifest ?? Manifest.getDefault(); }; /** diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts index 7298a9bfa72a6..7f90aa7b91063 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts @@ -13,13 +13,11 @@ import { export const ArtifactConstants = { GLOBAL_ALLOWLIST_NAME: 'endpoint-exceptionlist', SAVED_OBJECT_TYPE: 'endpoint:user-artifact', - SUPPORTED_OPERATING_SYSTEMS: ['linux', 'macos', 'windows'], - SCHEMA_VERSION: 'v1', + SUPPORTED_OPERATING_SYSTEMS: ['macos', 'windows'], }; export const ManifestConstants = { SAVED_OBJECT_TYPE: 'endpoint:user-artifact-manifest', - SCHEMA_VERSION: 'v1', }; export const getArtifactId = (artifact: InternalArtifactSchema) => { diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts index bb8b4fb3d5ce7..fea3b2b9a4526 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts @@ -314,21 +314,23 @@ describe('buildEventTypeSignal', () => { test('it should convert the exception lists response to the proper endpoint format while paging', async () => { // The first call returns two exceptions const first = getFoundExceptionListItemSchemaMock(); + first.per_page = 2; + first.total = 4; first.data.push(getExceptionListItemSchemaMock()); // The second call returns two exceptions const second = getFoundExceptionListItemSchemaMock(); + second.per_page = 2; + second.total = 4; second.data.push(getExceptionListItemSchemaMock()); - // The third call returns no exceptions, paging stops - const third = getFoundExceptionListItemSchemaMock(); - third.data = []; mockExceptionClient.findExceptionListItem = jest .fn() .mockReturnValueOnce(first) - .mockReturnValueOnce(second) - .mockReturnValueOnce(third); + .mockReturnValueOnce(second); + const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1'); + // Expect 2 exceptions, the first two calls returned the same exception list items expect(resp.entries.length).toEqual(2); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts index 5998a88527f2f..e41781dd605a0 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts @@ -79,10 +79,10 @@ export async function getFullEndpointExceptionList( schemaVersion: string ): Promise { const exceptions: WrappedTranslatedExceptionList = { entries: [] }; - let numResponses = 0; let page = 1; + let paging = true; - do { + while (paging) { const response = await eClient.findExceptionListItem({ listId: ENDPOINT_LIST_ID, namespaceType: 'agnostic', @@ -94,17 +94,16 @@ export async function getFullEndpointExceptionList( }); if (response?.data !== undefined) { - numResponses = response.data.length; - exceptions.entries = exceptions.entries.concat( translateToEndpointExceptions(response, schemaVersion) ); + paging = (page - 1) * 100 + response.data.length < response.total; page++; } else { break; } - } while (numResponses > 0); + } const [validated, errors] = validate(exceptions, wrappedTranslatedExceptionList); if (errors != null) { diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts index 95587c6fc105d..3d70f7266277f 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts @@ -6,7 +6,7 @@ import { ManifestSchemaVersion } from '../../../../common/endpoint/schema/common'; import { InternalArtifactCompleteSchema } from '../../schemas'; -import { ManifestConstants, getArtifactId } from './common'; +import { getArtifactId } from './common'; import { Manifest } from './manifest'; import { getMockArtifacts, @@ -30,29 +30,21 @@ describe('manifest', () => { }); test('Can create manifest with valid schema version', () => { - const manifest = new Manifest('v1'); + const manifest = new Manifest(); expect(manifest).toBeInstanceOf(Manifest); }); test('Cannot create manifest with invalid schema version', () => { expect(() => { - new Manifest('abcd' as ManifestSchemaVersion); + new Manifest({ + schemaVersion: 'abcd' as ManifestSchemaVersion, + }); }).toThrow(); }); test('Empty manifest transforms correctly to expected endpoint format', async () => { expect(emptyManifest.toEndpointFormat()).toStrictEqual({ artifacts: { - 'endpoint-exceptionlist-linux-v1': { - compression_algorithm: 'zlib', - encryption_algorithm: 'none', - decoded_sha256: 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', - encoded_sha256: 'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda', - decoded_size: 14, - encoded_size: 22, - relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', - }, 'endpoint-exceptionlist-macos-v1': { compression_algorithm: 'zlib', encryption_algorithm: 'none', @@ -74,7 +66,7 @@ describe('manifest', () => { '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', }, }, - manifest_version: 'a9b7ef358a363f327f479e31efc4f228b2277a7fb4d1914ca9b4e7ca9ffcf537', + manifest_version: '1.0.0', schema_version: 'v1', }); }); @@ -82,16 +74,6 @@ describe('manifest', () => { test('Manifest transforms correctly to expected endpoint format', async () => { expect(manifest1.toEndpointFormat()).toStrictEqual({ artifacts: { - 'endpoint-exceptionlist-linux-v1': { - compression_algorithm: 'zlib', - encryption_algorithm: 'none', - decoded_sha256: '96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - encoded_sha256: '975382ab55d019cbab0bbac207a54e2a7d489fad6e8f6de34fc6402e5ef37b1e', - decoded_size: 432, - encoded_size: 147, - relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - }, 'endpoint-exceptionlist-macos-v1': { compression_algorithm: 'zlib', encryption_algorithm: 'none', @@ -113,15 +95,16 @@ describe('manifest', () => { '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', }, }, - manifest_version: 'a7f4760bfa2662e85e30fe4fb8c01b4c4a20938c76ab21d3c5a3e781e547cce7', + manifest_version: '1.0.0', schema_version: 'v1', }); }); test('Manifest transforms correctly to expected saved object format', async () => { expect(manifest1.toSavedObject()).toStrictEqual({ + schemaVersion: 'v1', + semanticVersion: '1.0.0', ids: [ - 'endpoint-exceptionlist-linux-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', 'endpoint-exceptionlist-macos-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', 'endpoint-exceptionlist-windows-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', ], @@ -133,12 +116,12 @@ describe('manifest', () => { expect(diffs).toEqual([ { id: - 'endpoint-exceptionlist-linux-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', + 'endpoint-exceptionlist-macos-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', type: 'delete', }, { id: - 'endpoint-exceptionlist-linux-v1-0a5a2013a79f9e60682472284a1be45ab1ff68b9b43426d00d665016612c15c8', + 'endpoint-exceptionlist-macos-v1-0a5a2013a79f9e60682472284a1be45ab1ff68b9b43426d00d665016612c15c8', type: 'add', }, ]); @@ -154,7 +137,6 @@ describe('manifest', () => { const entries = manifest1.getEntries(); const keys = Object.keys(entries); expect(keys).toEqual([ - 'endpoint-exceptionlist-linux-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', 'endpoint-exceptionlist-macos-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', 'endpoint-exceptionlist-windows-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', ]); @@ -168,13 +150,8 @@ describe('manifest', () => { }); test('Manifest can be created from list of artifacts', async () => { - const oldManifest = new Manifest(ManifestConstants.SCHEMA_VERSION); - const manifest = Manifest.fromArtifacts(artifacts, 'v1', oldManifest); - expect( - manifest.contains( - 'endpoint-exceptionlist-linux-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3' - ) - ).toEqual(true); + const oldManifest = new Manifest(); + const manifest = Manifest.fromArtifacts(artifacts, oldManifest); expect( manifest.contains( 'endpoint-exceptionlist-macos-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3' diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts index 6ece2bf0f48e8..9e0e940ea9a1d 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts @@ -3,8 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ - -import { createHash } from 'crypto'; +import semver from 'semver'; import { validate } from '../../../../common/validate'; import { InternalArtifactSchema, @@ -13,13 +12,15 @@ import { InternalArtifactCompleteSchema, } from '../../schemas/artifacts'; import { - manifestSchemaVersion, ManifestSchemaVersion, + SemanticVersion, + semanticVersion, } from '../../../../common/endpoint/schema/common'; -import { ManifestSchema, manifestSchema } from '../../../../common/endpoint/schema/manifest'; +import { manifestSchema, ManifestSchema } from '../../../../common/endpoint/schema/manifest'; import { ManifestEntry } from './manifest_entry'; import { maybeCompressArtifact, isCompressed } from './lists'; import { getArtifactId } from './common'; +import { ManifestVersion, manifestVersion } from '../../schemas/artifacts/manifest'; export interface ManifestDiff { type: string; @@ -28,37 +29,39 @@ export interface ManifestDiff { export class Manifest { private entries: Record; - private schemaVersion: ManifestSchemaVersion; - - // For concurrency control - private version: string | undefined; + private version: ManifestVersion; - constructor(schemaVersion: string, version?: string) { + constructor(version?: Partial) { this.entries = {}; - this.version = version; - const [validated, errors] = validate( - (schemaVersion as unknown) as object, - manifestSchemaVersion - ); + const decodedVersion = { + schemaVersion: version?.schemaVersion ?? 'v1', + semanticVersion: version?.semanticVersion ?? '1.0.0', + soVersion: version?.soVersion, + }; + const [validated, errors] = validate(decodedVersion, manifestVersion); if (errors != null || validated === null) { - throw new Error(`Invalid manifest schema version: ${schemaVersion}`); + throw new Error(errors ?? 'Invalid version format.'); } - this.schemaVersion = validated; + this.version = validated; } - public static getDefault(schemaVersion: string) { - return new Manifest(schemaVersion); + public static getDefault(schemaVersion?: ManifestSchemaVersion) { + return new Manifest({ schemaVersion, semanticVersion: '1.0.0' }); } public static fromArtifacts( artifacts: InternalArtifactCompleteSchema[], - schemaVersion: string, - oldManifest: Manifest + oldManifest: Manifest, + schemaVersion?: ManifestSchemaVersion ): Manifest { - const manifest = new Manifest(schemaVersion, oldManifest.getSoVersion()); + const manifest = new Manifest({ + schemaVersion, + semanticVersion: oldManifest.getSemanticVersion(), + soVersion: oldManifest.getSavedObjectVersion(), + }); artifacts.forEach((artifact) => { const id = getArtifactId(artifact); const existingArtifact = oldManifest.getArtifact(id); @@ -71,25 +74,12 @@ export class Manifest { return manifest; } - public static fromPkgConfig(manifestPkgConfig: ManifestSchema): Manifest | null { - if (manifestSchema.is(manifestPkgConfig)) { - const manifest = new Manifest(manifestPkgConfig.schema_version); - for (const [identifier, artifactRecord] of Object.entries(manifestPkgConfig.artifacts)) { - const artifact = { - identifier, - compressionAlgorithm: artifactRecord.compression_algorithm, - encryptionAlgorithm: artifactRecord.encryption_algorithm, - decodedSha256: artifactRecord.decoded_sha256, - decodedSize: artifactRecord.decoded_size, - encodedSha256: artifactRecord.encoded_sha256, - encodedSize: artifactRecord.encoded_size, - }; - manifest.addEntry(artifact); - } - return manifest; - } else { - return null; + public bumpSemanticVersion() { + const newSemanticVersion = semver.inc(this.getSemanticVersion(), 'patch'); + if (!semanticVersion.is(newSemanticVersion)) { + throw new Error(`Invalid semver: ${newSemanticVersion}`); } + this.version.semanticVersion = newSemanticVersion; } public async compressArtifact(id: string): Promise { @@ -112,30 +102,16 @@ export class Manifest { return null; } - public equals(manifest: Manifest): boolean { - return this.getSha256() === manifest.getSha256(); - } - - public getSha256(): string { - let sha256 = createHash('sha256'); - Object.keys(this.entries) - .sort() - .forEach((docId) => { - sha256 = sha256.update(docId); - }); - return sha256.digest('hex'); - } - public getSchemaVersion(): ManifestSchemaVersion { - return this.schemaVersion; + return this.version.schemaVersion; } - public getSoVersion(): string | undefined { - return this.version; + public getSavedObjectVersion(): string | undefined { + return this.version.soVersion; } - public setSoVersion(version: string) { - this.version = version; + public getSemanticVersion(): SemanticVersion { + return this.version.semanticVersion; } public addEntry(artifact: InternalArtifactSchema) { @@ -179,8 +155,8 @@ export class Manifest { public toEndpointFormat(): ManifestSchema { const manifestObj: ManifestSchema = { - manifest_version: this.getSha256(), - schema_version: this.schemaVersion, + manifest_version: this.getSemanticVersion(), + schema_version: this.getSchemaVersion(), artifacts: {}, }; @@ -198,7 +174,9 @@ export class Manifest { public toSavedObject(): InternalManifestSchema { return { - ids: Object.keys(this.entries), + ids: Object.keys(this.getEntries()), + schemaVersion: this.getSchemaVersion(), + semanticVersion: this.getSemanticVersion(), }; } } diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/mocks.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/mocks.ts index 0ec6cb2bd61b3..62fff4715b562 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/mocks.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/mocks.ts @@ -29,7 +29,7 @@ export const getMockArtifactsWithDiff = async (opts?: { compress: boolean }) => return Promise.all( ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS.map>( async (os) => { - if (os === 'linux') { + if (os === 'macos') { return getInternalArtifactMockWithDiffs(os, 'v1'); } return getInternalArtifactMock(os, 'v1', opts); @@ -49,21 +49,21 @@ export const getEmptyMockArtifacts = async (opts?: { compress: boolean }) => { }; export const getMockManifest = async (opts?: { compress: boolean }) => { - const manifest = new Manifest('v1'); + const manifest = new Manifest(); const artifacts = await getMockArtifacts(opts); artifacts.forEach((artifact) => manifest.addEntry(artifact)); return manifest; }; export const getMockManifestWithDiffs = async (opts?: { compress: boolean }) => { - const manifest = new Manifest('v1'); + const manifest = new Manifest(); const artifacts = await getMockArtifactsWithDiff(opts); artifacts.forEach((artifact) => manifest.addEntry(artifact)); return manifest; }; export const getEmptyMockManifest = async (opts?: { compress: boolean }) => { - const manifest = new Manifest('v1'); + const manifest = new Manifest(); const artifacts = await getEmptyMockArtifacts(opts); artifacts.forEach((artifact) => manifest.addEntry(artifact)); return manifest; @@ -74,16 +74,6 @@ export const createPackageConfigWithInitialManifestMock = (): PackageConfig => { packageConfig.inputs[0].config!.artifact_manifest = { value: { artifacts: { - 'endpoint-exceptionlist-linux-v1': { - compression_algorithm: 'zlib', - encryption_algorithm: 'none', - decoded_sha256: 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', - encoded_sha256: 'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda', - decoded_size: 14, - encoded_size: 22, - relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', - }, 'endpoint-exceptionlist-macos-v1': { compression_algorithm: 'zlib', encryption_algorithm: 'none', @@ -105,7 +95,7 @@ export const createPackageConfigWithInitialManifestMock = (): PackageConfig => { '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', }, }, - manifest_version: 'a9b7ef358a363f327f479e31efc4f228b2277a7fb4d1914ca9b4e7ca9ffcf537', + manifest_version: '1.0.0', schema_version: 'v1', }, }; @@ -117,16 +107,6 @@ export const createPackageConfigWithManifestMock = (): PackageConfig => { packageConfig.inputs[0].config!.artifact_manifest = { value: { artifacts: { - 'endpoint-exceptionlist-linux-v1': { - compression_algorithm: 'zlib', - encryption_algorithm: 'none', - decoded_sha256: '0a5a2013a79f9e60682472284a1be45ab1ff68b9b43426d00d665016612c15c8', - encoded_sha256: '57941169bb2c5416f9bd7224776c8462cb9a2be0fe8b87e6213e77a1d29be824', - decoded_size: 292, - encoded_size: 131, - relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/0a5a2013a79f9e60682472284a1be45ab1ff68b9b43426d00d665016612c15c8', - }, 'endpoint-exceptionlist-macos-v1': { compression_algorithm: 'zlib', encryption_algorithm: 'none', @@ -148,7 +128,7 @@ export const createPackageConfigWithManifestMock = (): PackageConfig => { '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', }, }, - manifest_version: '520f6cf88b3f36a065c6ca81058d5f8690aadadf6fe857f8dec4cc37589e7283', + manifest_version: '1.0.1', schema_version: 'v1', }, }; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts index 0fb433df95de3..734304516e37e 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts @@ -55,7 +55,13 @@ export const manifestSavedObjectMappings: SavedObjectsType['mappings'] = { type: 'date', index: false, }, - // array of doc ids + schemaVersion: { + type: 'keyword', + }, + semanticVersion: { + type: 'keyword', + index: false, + }, ids: { type: 'keyword', index: false, diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.test.ts index daa8a7dd83ee0..32d58da5c3b78 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.test.ts @@ -38,7 +38,7 @@ describe('task', () => { taskManager: mockTaskManagerSetup, }); const mockTaskManagerStart = taskManagerMock.createStart(); - manifestTask.start({ taskManager: mockTaskManagerStart }); + await manifestTask.start({ taskManager: mockTaskManagerStart }); expect(mockTaskManagerStart.ensureScheduled).toHaveBeenCalled(); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts index ba164059866ea..4f2dbdf7644e2 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts @@ -3,7 +3,6 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ - import { Logger } from 'src/core/server'; import { ConcreteTaskInstance, @@ -11,7 +10,7 @@ import { TaskManagerStartContract, } from '../../../../../task_manager/server'; import { EndpointAppContext } from '../../types'; -import { reportErrors, ManifestConstants } from './common'; +import { reportErrors } from './common'; import { InternalArtifactCompleteSchema } from '../../schemas/artifacts'; export const ManifestTaskConstants = { @@ -45,7 +44,23 @@ export class ManifestTask { createTaskRunner: ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => { return { run: async () => { + const taskInterval = (await this.endpointAppContext.config()).packagerTaskInterval; await this.runTask(taskInstance.id); + const nextRun = new Date(); + if (taskInterval.endsWith('s')) { + const seconds = parseInt(taskInterval.slice(0, -1), 10); + nextRun.setSeconds(nextRun.getSeconds() + seconds); + } else if (taskInterval.endsWith('m')) { + const minutes = parseInt(taskInterval.slice(0, -1), 10); + nextRun.setMinutes(nextRun.getMinutes() + minutes); + } else { + this.logger.error(`Invalid task interval: ${taskInterval}`); + return; + } + return { + state: {}, + runAt: nextRun, + }; }, cancel: async () => {}, }; @@ -61,7 +76,7 @@ export class ManifestTask { taskType: ManifestTaskConstants.TYPE, scope: ['securitySolution'], schedule: { - interval: '60s', + interval: (await this.endpointAppContext.config()).packagerTaskInterval, }, state: {}, params: { version: ManifestTaskConstants.VERSION }, @@ -92,19 +107,14 @@ export class ManifestTask { try { // Last manifest we computed, which was saved to ES - const oldManifest = await manifestManager.getLastComputedManifest( - ManifestConstants.SCHEMA_VERSION - ); + const oldManifest = await manifestManager.getLastComputedManifest(); if (oldManifest == null) { this.logger.debug('User manifest not available yet.'); return; } // New computed manifest based on current state of exception list - const newManifest = await manifestManager.buildNewManifest( - ManifestConstants.SCHEMA_VERSION, - oldManifest - ); + const newManifest = await manifestManager.buildNewManifest(oldManifest); const diffs = newManifest.diff(oldManifest); // Compress new artifacts @@ -131,6 +141,7 @@ export class ManifestTask { // Commit latest manifest state, if different if (diffs.length) { + newManifest.bumpSemanticVersion(); const error = await manifestManager.commit(newManifest); if (error) { throw error; diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts index 38e900c4d5015..d825841f1e257 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts @@ -3,7 +3,6 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ - import { IRouter, SavedObjectsClientContract, @@ -14,7 +13,6 @@ import { import LRU from 'lru-cache'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths import { authenticateAgentWithAccessToken } from '../../../../../ingest_manager/server/services/agents/authenticate'; -import { validate } from '../../../../common/validate'; import { LIMITED_CONCURRENCY_ENDPOINT_ROUTE_TAG } from '../../../../common/endpoint/constants'; import { buildRouteValidation } from '../../../utils/build_validation/route_validation'; import { ArtifactConstants } from '../../lib/artifacts'; @@ -63,6 +61,7 @@ export function registerDownloadExceptionListRoute( } } + const validateDownload = (await endpointContext.config()).validateArtifactDownloads; const buildAndValidateResponse = (artName: string, body: Buffer): IKibanaResponse => { const artifact: HttpResponseOptions = { body, @@ -72,11 +71,10 @@ export function registerDownloadExceptionListRoute( }, }; - const [validated, errors] = validate(artifact, downloadArtifactResponseSchema); - if (errors !== null || validated === null) { - return res.internalError({ body: errors! }); + if (validateDownload && !downloadArtifactResponseSchema.is(artifact)) { + return res.internalError({ body: 'Artifact failed to validate.' }); } else { - return res.ok((validated as unknown) as HttpResponseOptions); + return res.ok(artifact); } }; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/manifest.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/manifest.ts new file mode 100644 index 0000000000000..707d4c1374fe2 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/manifest.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; +import { manifestSchemaVersion, semanticVersion } from '../../../../common/endpoint/schema/common'; + +const optionalVersions = t.partial({ + soVersion: t.string, +}); + +export const manifestVersion = t.intersection([ + optionalVersions, + t.exact( + t.type({ + schemaVersion: manifestSchemaVersion, + semanticVersion, + }) + ), +]); +export type ManifestVersion = t.TypeOf; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts index d95627601a183..ae565f785c399 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts @@ -53,4 +53,6 @@ export const getInternalArtifactMockWithDiffs = async ( export const getInternalManifestMock = (): InternalManifestSchema => ({ ids: [], + schemaVersion: 'v1', + semanticVersion: '1.0.0', }); diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts index 4dea916dcb436..56f247b65d802 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts @@ -9,8 +9,10 @@ import { compressionAlgorithm, encryptionAlgorithm, identifier, + semanticVersion, sha256, size, + manifestSchemaVersion, } from '../../../../common/endpoint/schema/common'; import { created } from './common'; @@ -58,6 +60,8 @@ export type InternalArtifactCreateSchema = t.TypeOf; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.ts index 385f115e6301a..e55243f0650a5 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.ts @@ -3,7 +3,6 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ - import { SavedObject, SavedObjectsClientContract, diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts index c838f772fb66b..d99d6a959d7aa 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts @@ -16,22 +16,17 @@ describe('manifest_manager', () => { describe('ManifestManager sanity checks', () => { test('ManifestManager can retrieve and diff manifests', async () => { const manifestManager = getManifestManagerMock(); - const oldManifest = await manifestManager.getLastComputedManifest( - ManifestConstants.SCHEMA_VERSION - ); - const newManifest = await manifestManager.buildNewManifest( - ManifestConstants.SCHEMA_VERSION, - oldManifest! - ); + const oldManifest = await manifestManager.getLastComputedManifest(); + const newManifest = await manifestManager.buildNewManifest(oldManifest!); expect(newManifest.diff(oldManifest!)).toEqual([ { id: - 'endpoint-exceptionlist-linux-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', + 'endpoint-exceptionlist-macos-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', type: 'delete', }, { id: - 'endpoint-exceptionlist-linux-v1-0a5a2013a79f9e60682472284a1be45ab1ff68b9b43426d00d665016612c15c8', + 'endpoint-exceptionlist-macos-v1-0a5a2013a79f9e60682472284a1be45ab1ff68b9b43426d00d665016612c15c8', type: 'add', }, ]); @@ -40,23 +35,18 @@ describe('manifest_manager', () => { test('ManifestManager populates cache properly', async () => { const cache = new LRU({ max: 10, maxAge: 1000 * 60 * 60 }); const manifestManager = getManifestManagerMock({ cache }); - const oldManifest = await manifestManager.getLastComputedManifest( - ManifestConstants.SCHEMA_VERSION - ); - const newManifest = await manifestManager.buildNewManifest( - ManifestConstants.SCHEMA_VERSION, - oldManifest! - ); + const oldManifest = await manifestManager.getLastComputedManifest(); + const newManifest = await manifestManager.buildNewManifest(oldManifest!); const diffs = newManifest.diff(oldManifest!); expect(diffs).toEqual([ { id: - 'endpoint-exceptionlist-linux-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', + 'endpoint-exceptionlist-macos-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', type: 'delete', }, { id: - 'endpoint-exceptionlist-linux-v1-0a5a2013a79f9e60682472284a1be45ab1ff68b9b43426d00d665016612c15c8', + 'endpoint-exceptionlist-macos-v1-0a5a2013a79f9e60682472284a1be45ab1ff68b9b43426d00d665016612c15c8', type: 'add', }, ]); @@ -104,13 +94,8 @@ describe('manifest_manager', () => { test('ManifestManager cannot dispatch incomplete (uncompressed) artifact', async () => { const packageConfigService = createPackageConfigServiceMock(); const manifestManager = getManifestManagerMock({ packageConfigService }); - const oldManifest = await manifestManager.getLastComputedManifest( - ManifestConstants.SCHEMA_VERSION - ); - const newManifest = await manifestManager.buildNewManifest( - ManifestConstants.SCHEMA_VERSION, - oldManifest! - ); + const oldManifest = await manifestManager.getLastComputedManifest(); + const newManifest = await manifestManager.buildNewManifest(oldManifest!); const dispatchErrors = await manifestManager.tryDispatch(newManifest); expect(dispatchErrors.length).toEqual(1); expect(dispatchErrors[0].message).toEqual('Invalid manifest'); @@ -119,17 +104,14 @@ describe('manifest_manager', () => { test('ManifestManager can dispatch manifest', async () => { const packageConfigService = createPackageConfigServiceMock(); const manifestManager = getManifestManagerMock({ packageConfigService }); - const oldManifest = await manifestManager.getLastComputedManifest( - ManifestConstants.SCHEMA_VERSION - ); - const newManifest = await manifestManager.buildNewManifest( - ManifestConstants.SCHEMA_VERSION, - oldManifest! - ); + const oldManifest = await manifestManager.getLastComputedManifest(); + const newManifest = await manifestManager.buildNewManifest(oldManifest!); const diffs = newManifest.diff(oldManifest!); const newArtifactId = diffs[1].id; await newManifest.compressArtifact(newArtifactId); + newManifest.bumpSemanticVersion(); + const dispatchErrors = await manifestManager.tryDispatch(newManifest); expect(dispatchErrors).toEqual([]); @@ -140,10 +122,10 @@ describe('manifest_manager', () => { expect( packageConfigService.update.mock.calls[0][2].inputs[0].config!.artifact_manifest.value ).toEqual({ - manifest_version: '520f6cf88b3f36a065c6ca81058d5f8690aadadf6fe857f8dec4cc37589e7283', + manifest_version: '1.0.1', schema_version: 'v1', artifacts: { - 'endpoint-exceptionlist-linux-v1': { + 'endpoint-exceptionlist-macos-v1': { compression_algorithm: 'zlib', encryption_algorithm: 'none', decoded_sha256: '0a5a2013a79f9e60682472284a1be45ab1ff68b9b43426d00d665016612c15c8', @@ -151,17 +133,7 @@ describe('manifest_manager', () => { decoded_size: 292, encoded_size: 131, relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/0a5a2013a79f9e60682472284a1be45ab1ff68b9b43426d00d665016612c15c8', - }, - 'endpoint-exceptionlist-macos-v1': { - compression_algorithm: 'zlib', - encryption_algorithm: 'none', - decoded_sha256: '96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - encoded_sha256: '975382ab55d019cbab0bbac207a54e2a7d489fad6e8f6de34fc6402e5ef37b1e', - decoded_size: 432, - encoded_size: 147, - relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-v1/96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', + '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-v1/0a5a2013a79f9e60682472284a1be45ab1ff68b9b43426d00d665016612c15c8', }, 'endpoint-exceptionlist-windows-v1': { compression_algorithm: 'zlib', @@ -180,17 +152,14 @@ describe('manifest_manager', () => { test('ManifestManager fails to dispatch on conflict', async () => { const packageConfigService = createPackageConfigServiceMock(); const manifestManager = getManifestManagerMock({ packageConfigService }); - const oldManifest = await manifestManager.getLastComputedManifest( - ManifestConstants.SCHEMA_VERSION - ); - const newManifest = await manifestManager.buildNewManifest( - ManifestConstants.SCHEMA_VERSION, - oldManifest! - ); + const oldManifest = await manifestManager.getLastComputedManifest(); + const newManifest = await manifestManager.buildNewManifest(oldManifest!); const diffs = newManifest.diff(oldManifest!); const newArtifactId = diffs[1].id; await newManifest.compressArtifact(newArtifactId); + newManifest.bumpSemanticVersion(); + packageConfigService.update.mockRejectedValueOnce({ status: 409 }); const dispatchErrors = await manifestManager.tryDispatch(newManifest); expect(dispatchErrors).toEqual([{ status: 409 }]); @@ -202,13 +171,8 @@ describe('manifest_manager', () => { savedObjectsClient, }); - const oldManifest = await manifestManager.getLastComputedManifest( - ManifestConstants.SCHEMA_VERSION - ); - const newManifest = await manifestManager.buildNewManifest( - ManifestConstants.SCHEMA_VERSION, - oldManifest! - ); + const oldManifest = await manifestManager.getLastComputedManifest(); + const newManifest = await manifestManager.buildNewManifest(oldManifest!); const diffs = newManifest.diff(oldManifest!); const oldArtifactId = diffs[0].id; const newArtifactId = diffs[1].id; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts index 13ca51e1f2b39..b5f7b1384750f 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ - +import semver from 'semver'; import { Logger, SavedObjectsClientContract } from 'src/core/server'; import LRU from 'lru-cache'; import { PackageConfigServiceInterface } from '../../../../../../ingest_manager/server'; @@ -13,7 +13,6 @@ import { manifestDispatchSchema } from '../../../../../common/endpoint/schema/ma import { ArtifactConstants, - ManifestConstants, Manifest, buildArtifact, getFullEndpointExceptionList, @@ -52,6 +51,7 @@ export class ManifestManager { protected savedObjectsClient: SavedObjectsClientContract; protected logger: Logger; protected cache: LRU; + protected schemaVersion: ManifestSchemaVersion; constructor(context: ManifestManagerContext) { this.artifactClient = context.artifactClient; @@ -60,28 +60,27 @@ export class ManifestManager { this.savedObjectsClient = context.savedObjectsClient; this.logger = context.logger; this.cache = context.cache; + this.schemaVersion = 'v1'; } /** - * Gets a ManifestClient for the provided schemaVersion. + * Gets a ManifestClient for this manager's schemaVersion. * - * @param schemaVersion The schema version of the manifest. - * @returns {ManifestClient} A ManifestClient scoped to the provided schemaVersion. + * @returns {ManifestClient} A ManifestClient scoped to the appropriate schemaVersion. */ - protected getManifestClient(schemaVersion: string): ManifestClient { - return new ManifestClient(this.savedObjectsClient, schemaVersion as ManifestSchemaVersion); + protected getManifestClient(): ManifestClient { + return new ManifestClient(this.savedObjectsClient, this.schemaVersion); } /** * Builds an array of artifacts (one per supported OS) based on the current * state of exception-list-agnostic SOs. * - * @param schemaVersion The schema version of the artifact * @returns {Promise} An array of uncompressed artifacts built from exception-list-agnostic SOs. * @throws Throws/rejects if there are errors building the list. */ protected async buildExceptionListArtifacts( - schemaVersion: string + artifactSchemaVersion?: string ): Promise { return ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS.reduce< Promise @@ -89,10 +88,10 @@ export class ManifestManager { const exceptionList = await getFullEndpointExceptionList( this.exceptionListClient, os, - schemaVersion + artifactSchemaVersion ?? 'v1' ); const artifacts = await acc; - const artifact = await buildArtifact(exceptionList, os, schemaVersion); + const artifact = await buildArtifact(exceptionList, os, artifactSchemaVersion ?? 'v1'); return Promise.resolve([...artifacts, artifact]); }, Promise.resolve([])); } @@ -168,20 +167,23 @@ export class ManifestManager { * Returns the last computed manifest based on the state of the * user-artifact-manifest SO. * - * @param schemaVersion The schema version of the manifest. * @returns {Promise} The last computed manifest, or null if does not exist. * @throws Throws/rejects if there is an unexpected error retrieving the manifest. */ - public async getLastComputedManifest(schemaVersion: string): Promise { + public async getLastComputedManifest(): Promise { try { - const manifestClient = this.getManifestClient(schemaVersion); + const manifestClient = this.getManifestClient(); const manifestSo = await manifestClient.getManifest(); if (manifestSo.version === undefined) { throw new Error('No version returned for manifest.'); } - const manifest = new Manifest(schemaVersion, manifestSo.version); + const manifest = new Manifest({ + schemaVersion: this.schemaVersion, + semanticVersion: manifestSo.attributes.semanticVersion, + soVersion: manifestSo.version, + }); for (const id of manifestSo.attributes.ids) { const artifactSo = await this.artifactClient.getArtifact(id); @@ -199,22 +201,17 @@ export class ManifestManager { /** * Builds a new manifest based on the current user exception list. * - * @param schemaVersion The schema version of the manifest. * @param baselineManifest A baseline manifest to use for initializing pre-existing artifacts. * @returns {Promise} A new Manifest object reprenting the current exception list. */ - public async buildNewManifest( - schemaVersion: string, - baselineManifest?: Manifest - ): Promise { + public async buildNewManifest(baselineManifest?: Manifest): Promise { // Build new exception list artifacts - const artifacts = await this.buildExceptionListArtifacts(ArtifactConstants.SCHEMA_VERSION); + const artifacts = await this.buildExceptionListArtifacts(); // Build new manifest const manifest = Manifest.fromArtifacts( artifacts, - ManifestConstants.SCHEMA_VERSION, - baselineManifest ?? Manifest.getDefault(schemaVersion) + baselineManifest ?? Manifest.getDefault(this.schemaVersion) ); return manifest; @@ -247,14 +244,12 @@ export class ManifestManager { for (const packageConfig of items) { const { id, revision, updated_at, updated_by, ...newPackageConfig } = packageConfig; if (newPackageConfig.inputs.length > 0 && newPackageConfig.inputs[0].config !== undefined) { - const artifactManifest = newPackageConfig.inputs[0].config.artifact_manifest ?? { + const oldManifest = newPackageConfig.inputs[0].config.artifact_manifest ?? { value: {}, }; - const oldManifest = - Manifest.fromPkgConfig(artifactManifest.value) ?? - Manifest.getDefault(ManifestConstants.SCHEMA_VERSION); - if (!manifest.equals(oldManifest)) { + const newManifestVersion = manifest.getSemanticVersion(); + if (semver.gt(newManifestVersion, oldManifest.value.manifest_version)) { newPackageConfig.inputs[0].config.artifact_manifest = { value: serializedManifest, }; @@ -262,7 +257,7 @@ export class ManifestManager { try { await this.packageConfigService.update(this.savedObjectsClient, id, newPackageConfig); this.logger.debug( - `Updated package config ${id} with manifest version ${manifest.getSha256()}` + `Updated package config ${id} with manifest version ${manifest.getSemanticVersion()}` ); } catch (err) { errors.push(err); @@ -274,8 +269,7 @@ export class ManifestManager { errors.push(new Error(`Package config ${id} has no config.`)); } } - - paging = page * items.length < total; + paging = (page - 1) * 20 + items.length < total; page++; } @@ -290,11 +284,11 @@ export class ManifestManager { */ public async commit(manifest: Manifest): Promise { try { - const manifestClient = this.getManifestClient(manifest.getSchemaVersion()); + const manifestClient = this.getManifestClient(); // Commit the new manifest const manifestSo = manifest.toSavedObject(); - const version = manifest.getSoVersion(); + const version = manifest.getSavedObjectVersion(); if (version == null) { await manifestClient.createManifest(manifestSo); @@ -304,7 +298,7 @@ export class ManifestManager { }); } - this.logger.info(`Committed manifest ${manifest.getSha256()}`); + this.logger.info(`Committed manifest ${manifest.getSemanticVersion()}`); } catch (err) { return err; } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/index.ts index bfaab096a5013..196d816b6b257 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/index.ts @@ -25,6 +25,8 @@ export const createMockConfig = () => ({ from: 'now-15m', to: 'now', }, + packagerTaskInterval: '60s', + validateArtifactDownloads: true, }); export const mockGetCurrentUser = { diff --git a/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json b/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json index 47390f0428742..3af1070597671 100644 --- a/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json +++ b/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json @@ -1,28 +1,3 @@ -{ - "type": "doc", - "value": { - "id": "endpoint:user-artifact:endpoint-exceptionlist-linux-v1-d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f", - "index": ".kibana", - "source": { - "references": [ - ], - "endpoint:user-artifact": { - "body": "eJylkM8KwjAMxl9Fci59gN29iicvMqR02QjUbiSpKGPvbiw6ETwpuX1/fh9kBszKhALNcQa9TQgNCJ2nhOA+vJ4wdWaGqJSHPY8RRXxPCb3QkJEtP07IQUe2GOWYSoedqU8qXq16ikGqeAmpPNRtCqIU3WbnDx4WN38d/WvhQqmCXzDlIlojP9CsjLC0bqWtHwhaGN/1jHVkae3u+6N6Sg==", - "created": 1593016187465, - "compressionAlgorithm": "zlib", - "encryptionAlgorithm": "none", - "identifier": "endpoint-exceptionlist-linux-v1", - "encodedSha256": "5caaeabcb7864d47157fc7c28d5a7398b4f6bbaaa565d789c02ee809253b7613", - "encodedSize": 160, - "decodedSha256": "d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f", - "decodedSize": 358 - }, - "type": "endpoint:user-artifact", - "updated_at": "2020-06-24T16:29:47.584Z" - } - } -} - { "type": "doc", "value": { @@ -83,8 +58,9 @@ ], "endpoint:user-artifact-manifest": { "created": 1593183699663, + "schemaVersion": "v1", + "semanticVersion": "1.0.1", "ids": [ - "endpoint-exceptionlist-linux-v1-d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f", "endpoint-exceptionlist-macos-v1-d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658", "endpoint-exceptionlist-windows-v1-8d2bcc37e82fad5d06e2c9e4bd96793ea8905ace1d528a57d0d0579ecc8c647e" ] diff --git a/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts b/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts index cf76f297d83be..ef6b629f83018 100644 --- a/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts +++ b/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts @@ -118,18 +118,6 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { }, artifact_manifest: { artifacts: { - 'endpoint-exceptionlist-linux-v1': { - compression_algorithm: 'zlib', - decoded_sha256: - 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', - decoded_size: 14, - encoded_sha256: - 'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda', - encoded_size: 22, - encryption_algorithm: 'none', - relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', - }, 'endpoint-exceptionlist-macos-v1': { compression_algorithm: 'zlib', decoded_sha256: diff --git a/x-pack/test/security_solution_endpoint_api_int/apis/artifacts/index.ts b/x-pack/test/security_solution_endpoint_api_int/apis/artifacts/index.ts index a4a8de418157f..d5106f5549924 100644 --- a/x-pack/test/security_solution_endpoint_api_int/apis/artifacts/index.ts +++ b/x-pack/test/security_solution_endpoint_api_int/apis/artifacts/index.ts @@ -83,24 +83,24 @@ export default function (providerContext: FtrProviderContext) { it('should download an artifact with list items', async () => { await supertestWithoutAuth .get( - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f' + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/8d2bcc37e82fad5d06e2c9e4bd96793ea8905ace1d528a57d0d0579ecc8c647e' ) .set('kbn-xsrf', 'xxx') .set('authorization', `ApiKey ${agentAccessAPIKey}`) .send() .expect(200) .expect((response) => { - expect(response.body.byteLength).to.equal(160); + expect(response.body.byteLength).to.equal(191); const encodedHash = createHash('sha256').update(response.body).digest('hex'); expect(encodedHash).to.equal( - '5caaeabcb7864d47157fc7c28d5a7398b4f6bbaaa565d789c02ee809253b7613' + '73015ee5131dabd1b48aa4776d3e766d836f8dd8c9fa8999c9b931f60027f07f' ); const decodedBody = inflateSync(response.body); const decodedHash = createHash('sha256').update(decodedBody).digest('hex'); expect(decodedHash).to.equal( - 'd2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f' + '8d2bcc37e82fad5d06e2c9e4bd96793ea8905ace1d528a57d0d0579ecc8c647e' ); - expect(decodedBody.byteLength).to.equal(358); + expect(decodedBody.byteLength).to.equal(704); const artifactJson = JSON.parse(decodedBody.toString()); expect(artifactJson).to.eql({ entries: [ @@ -113,6 +113,35 @@ export default function (providerContext: FtrProviderContext) { type: 'exact_cased', value: 'Elastic, N.V.', }, + { + entries: [ + { + field: 'signer', + operator: 'included', + type: 'exact_cased', + value: '😈', + }, + { + field: 'trusted', + operator: 'included', + type: 'exact_cased', + value: 'true', + }, + ], + field: 'file.signature', + type: 'nested', + }, + ], + }, + { + type: 'simple', + entries: [ + { + field: 'actingProcess.file.signer', + operator: 'included', + type: 'exact_cased', + value: 'Another signer', + }, { entries: [ { @@ -280,7 +309,7 @@ export default function (providerContext: FtrProviderContext) { it('should download an artifact from cache', async () => { await supertestWithoutAuth .get( - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f' + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/8d2bcc37e82fad5d06e2c9e4bd96793ea8905ace1d528a57d0d0579ecc8c647e' ) .set('kbn-xsrf', 'xxx') .set('authorization', `ApiKey ${agentAccessAPIKey}`) @@ -292,22 +321,24 @@ export default function (providerContext: FtrProviderContext) { .then(async () => { await supertestWithoutAuth .get( - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f' + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/8d2bcc37e82fad5d06e2c9e4bd96793ea8905ace1d528a57d0d0579ecc8c647e' ) .set('kbn-xsrf', 'xxx') .set('authorization', `ApiKey ${agentAccessAPIKey}`) .send() .expect(200) .expect((response) => { + expect(response.body.byteLength).to.equal(191); const encodedHash = createHash('sha256').update(response.body).digest('hex'); expect(encodedHash).to.equal( - '5caaeabcb7864d47157fc7c28d5a7398b4f6bbaaa565d789c02ee809253b7613' + '73015ee5131dabd1b48aa4776d3e766d836f8dd8c9fa8999c9b931f60027f07f' ); const decodedBody = inflateSync(response.body); const decodedHash = createHash('sha256').update(decodedBody).digest('hex'); expect(decodedHash).to.equal( - 'd2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f' + '8d2bcc37e82fad5d06e2c9e4bd96793ea8905ace1d528a57d0d0579ecc8c647e' ); + expect(decodedBody.byteLength).to.equal(704); const artifactJson = JSON.parse(decodedBody.toString()); expect(artifactJson).to.eql({ entries: [ @@ -320,6 +351,35 @@ export default function (providerContext: FtrProviderContext) { type: 'exact_cased', value: 'Elastic, N.V.', }, + { + entries: [ + { + field: 'signer', + operator: 'included', + type: 'exact_cased', + value: '😈', + }, + { + field: 'trusted', + operator: 'included', + type: 'exact_cased', + value: 'true', + }, + ], + field: 'file.signature', + type: 'nested', + }, + ], + }, + { + type: 'simple', + entries: [ + { + field: 'actingProcess.file.signer', + operator: 'included', + type: 'exact_cased', + value: 'Another signer', + }, { entries: [ {