Skip to content

Commit

Permalink
[Security Solution][Endpoint] Fix base64 download bug and adopt new u…
Browse files Browse the repository at this point in the history 
…ser artifact/manifest format (#70998)

* Fix base64 download bug

* Add test for artifact download

* Add more tests to ensure cached versions of artifacts are correct

* Convert to new format

* missed some refs

* partial fix to wrapper format

* update fixtures and integration test

* Fixing unit tests

Co-authored-by: Alex Kahan <[email protected]>
Co-authored-by: Elastic Machine <[email protected]>
  • Loading branch information
@madirey @elasticmachine
3 people authored Jul 8, 2020
1 parent 8facae7 commit f5b77e1
Show file tree
Hide file tree
Showing 19 changed files with 349 additions and 145 deletions.
8 changes: 4 additions & 4 deletions x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,10 @@ import {
export const manifestEntrySchema = t.exact(
t.type({
relative_url: relativeUrl,
precompress_sha256: sha256,
precompress_size: size,
postcompress_sha256: sha256,
postcompress_size: size,
decoded_sha256: sha256,
decoded_size: size,
encoded_sha256: sha256,
encoded_size: size,
compression_algorithm: compressionAlgorithm,
encryption_algorithm: encryptionAlgorithm,
})
Expand Down
4 changes: 2 additions & 2 deletions x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,12 @@

export const ArtifactConstants = {
GLOBAL_ALLOWLIST_NAME: 'endpoint-exceptionlist',
SAVED_OBJECT_TYPE: 'endpoint:user-artifact',
SAVED_OBJECT_TYPE: 'endpoint:user-artifact:v2',
SUPPORTED_OPERATING_SYSTEMS: ['linux', 'macos', 'windows'],
SCHEMA_VERSION: '1.0.0',
};

export const ManifestConstants = {
SAVED_OBJECT_TYPE: 'endpoint:user-artifact-manifest',
SAVED_OBJECT_TYPE: 'endpoint:user-artifact-manifest:v2',
SCHEMA_VERSION: '1.0.0',
};
32 changes: 22 additions & 10 deletions x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,8 @@ describe('buildEventTypeSignal', () => {

test('it should convert the exception lists response to the proper endpoint format', async () => {
const expectedEndpointExceptions = {
exceptions_list: [
type: 'simple',
entries: [
{
entries: [
{
Expand All @@ -46,7 +47,9 @@ describe('buildEventTypeSignal', () => {
const first = getFoundExceptionListItemSchemaMock();
mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first);
const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0');
expect(resp).toEqual(expectedEndpointExceptions);
expect(resp).toEqual({
entries: [expectedEndpointExceptions],
});
});

test('it should convert simple fields', async () => {
Expand All @@ -57,7 +60,8 @@ describe('buildEventTypeSignal', () => {
];

const expectedEndpointExceptions = {
exceptions_list: [
type: 'simple',
entries: [
{
field: 'server.domain',
operator: 'included',
Expand All @@ -84,7 +88,9 @@ describe('buildEventTypeSignal', () => {
mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first);

const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0');
expect(resp).toEqual(expectedEndpointExceptions);
expect(resp).toEqual({
entries: [expectedEndpointExceptions],
});
});

test('it should convert fields case sensitive', async () => {
Expand All @@ -100,7 +106,8 @@ describe('buildEventTypeSignal', () => {
];

const expectedEndpointExceptions = {
exceptions_list: [
type: 'simple',
entries: [
{
field: 'server.domain',
operator: 'included',
Expand All @@ -127,7 +134,9 @@ describe('buildEventTypeSignal', () => {
mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first);

const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0');
expect(resp).toEqual(expectedEndpointExceptions);
expect(resp).toEqual({
entries: [expectedEndpointExceptions],
});
});

test('it should ignore unsupported entries', async () => {
Expand All @@ -147,7 +156,8 @@ describe('buildEventTypeSignal', () => {
];

const expectedEndpointExceptions = {
exceptions_list: [
type: 'simple',
entries: [
{
field: 'server.domain',
operator: 'included',
Expand All @@ -162,7 +172,9 @@ describe('buildEventTypeSignal', () => {
mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first);

const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0');
expect(resp).toEqual(expectedEndpointExceptions);
expect(resp).toEqual({
entries: [expectedEndpointExceptions],
});
});

test('it should convert the exception lists response to the proper endpoint format while paging', async () => {
Expand All @@ -182,7 +194,7 @@ describe('buildEventTypeSignal', () => {
.mockReturnValueOnce(second)
.mockReturnValueOnce(third);
const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0');
expect(resp.exceptions_list.length).toEqual(6);
expect(resp.entries.length).toEqual(3);
});

test('it should handle no exceptions', async () => {
Expand All @@ -191,6 +203,6 @@ describe('buildEventTypeSignal', () => {
exceptionsResponse.total = 0;
mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(exceptionsResponse);
const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0');
expect(resp.exceptions_list.length).toEqual(0);
expect(resp.entries.length).toEqual(0);
});
});
50 changes: 30 additions & 20 deletions x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
*/

import { createHash } from 'crypto';
import { ExceptionListItemSchema } from '../../../../../lists/common/schemas';
import { validate } from '../../../../common/validate';

import { Entry, EntryNested } from '../../../../../lists/common/schemas/types/entries';
Expand All @@ -14,13 +15,14 @@ import {
InternalArtifactSchema,
TranslatedEntry,
WrappedTranslatedExceptionList,
wrappedExceptionList,
wrappedTranslatedExceptionList,
TranslatedEntryNestedEntry,
translatedEntryNestedEntry,
translatedEntry as translatedEntryType,
TranslatedEntryMatcher,
translatedEntryMatchMatcher,
translatedEntryMatchAnyMatcher,
TranslatedExceptionListItem,
} from '../../schemas';
import { ArtifactConstants } from './common';

Expand All @@ -36,10 +38,10 @@ export async function buildArtifact(
identifier: `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-${os}-${schemaVersion}`,
compressionAlgorithm: 'none',
encryptionAlgorithm: 'none',
decompressedSha256: sha256,
compressedSha256: sha256,
decompressedSize: exceptionsBuffer.byteLength,
compressedSize: exceptionsBuffer.byteLength,
decodedSha256: sha256,
encodedSha256: sha256,
decodedSize: exceptionsBuffer.byteLength,
encodedSize: exceptionsBuffer.byteLength,
created: Date.now(),
body: exceptionsBuffer.toString('base64'),
};
Expand All @@ -50,7 +52,7 @@ export async function getFullEndpointExceptionList(
os: string,
schemaVersion: string
): Promise<WrappedTranslatedExceptionList> {
const exceptions: WrappedTranslatedExceptionList = { exceptions_list: [] };
const exceptions: WrappedTranslatedExceptionList = { entries: [] };
let numResponses = 0;
let page = 1;

Expand All @@ -68,7 +70,7 @@ export async function getFullEndpointExceptionList(
if (response?.data !== undefined) {
numResponses = response.data.length;

exceptions.exceptions_list = exceptions.exceptions_list.concat(
exceptions.entries = exceptions.entries.concat(
translateToEndpointExceptions(response, schemaVersion)
);

Expand All @@ -78,7 +80,7 @@ export async function getFullEndpointExceptionList(
}
} while (numResponses > 0);

const [validated, errors] = validate(exceptions, wrappedExceptionList);
const [validated, errors] = validate(exceptions, wrappedTranslatedExceptionList);
if (errors != null) {
throw new Error(errors);
}
Expand All @@ -92,19 +94,11 @@ export async function getFullEndpointExceptionList(
export function translateToEndpointExceptions(
exc: FoundExceptionListItemSchema,
schemaVersion: string
): TranslatedEntry[] {
): TranslatedExceptionListItem[] {
if (schemaVersion === '1.0.0') {
return exc.data
.flatMap((list) => {
return list.entries;
})
.reduce((entries: TranslatedEntry[], entry) => {
const translatedEntry = translateEntry(schemaVersion, entry);
if (translatedEntry !== undefined && translatedEntryType.is(translatedEntry)) {
entries.push(translatedEntry);
}
return entries;
}, []);
return exc.data.map((item) => {
return translateItem(schemaVersion, item);
});
} else {
throw new Error('unsupported schemaVersion');
}
Expand All @@ -124,6 +118,22 @@ function normalizeFieldName(field: string): string {
return field.endsWith('.text') ? field.substring(0, field.length - 5) : field;
}

function translateItem(
schemaVersion: string,
item: ExceptionListItemSchema
): TranslatedExceptionListItem {
return {
type: item.type,
entries: item.entries.reduce((translatedEntries: TranslatedEntry[], entry) => {
const translatedEntry = translateEntry(schemaVersion, entry);
if (translatedEntry !== undefined && translatedEntryType.is(translatedEntry)) {
translatedEntries.push(translatedEntry);
}
return translatedEntries;
}, []),
};
}

function translateEntry(
schemaVersion: string,
entry: Entry | EntryNested
Expand Down
56 changes: 28 additions & 28 deletions x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,32 +57,32 @@ describe('manifest', () => {
'endpoint-exceptionlist-linux-1.0.0': {
compression_algorithm: 'none',
encryption_algorithm: 'none',
precompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
postcompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
precompress_size: 268,
postcompress_size: 268,
decoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735',
encoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735',
decoded_size: 430,
encoded_size: 430,
relative_url:
'/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
'/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735',
},
'endpoint-exceptionlist-macos-1.0.0': {
compression_algorithm: 'none',
encryption_algorithm: 'none',
precompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
postcompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
precompress_size: 268,
postcompress_size: 268,
decoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735',
encoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735',
decoded_size: 430,
encoded_size: 430,
relative_url:
'/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
'/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-1.0.0/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735',
},
'endpoint-exceptionlist-windows-1.0.0': {
compression_algorithm: 'none',
encryption_algorithm: 'none',
precompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
postcompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
precompress_size: 268,
postcompress_size: 268,
decoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735',
encoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735',
decoded_size: 430,
encoded_size: 430,
relative_url:
'/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
'/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735',
},
},
manifest_version: 'abcd',
Expand All @@ -94,9 +94,9 @@ describe('manifest', () => {
expect(manifest1.toSavedObject()).toStrictEqual({
created: now.getTime(),
ids: [
'endpoint-exceptionlist-linux-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
'endpoint-exceptionlist-macos-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
'endpoint-exceptionlist-windows-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
'endpoint-exceptionlist-linux-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735',
'endpoint-exceptionlist-macos-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735',
'endpoint-exceptionlist-windows-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735',
],
});
});
Expand All @@ -106,36 +106,36 @@ describe('manifest', () => {
expect(diffs).toEqual([
{
id:
'endpoint-exceptionlist-linux-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
'endpoint-exceptionlist-linux-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735',
type: 'delete',
},
{
id:
'endpoint-exceptionlist-linux-1.0.0-69328f83418f4957470640ed6cc605be6abb5fe80e0e388fd74f9764ad7ed5d1',
'endpoint-exceptionlist-linux-1.0.0-3d3546e94f70493021ee845be32c66e36ea7a720c64b4d608d8029fe949f7e51',
type: 'add',
},
]);
});

test('Manifest returns data for given artifact', async () => {
const artifact = artifacts[0];
const returned = manifest1.getArtifact(`${artifact.identifier}-${artifact.compressedSha256}`);
const returned = manifest1.getArtifact(`${artifact.identifier}-${artifact.encodedSha256}`);
expect(returned).toEqual(artifact);
});

test('Manifest returns entries map', async () => {
const entries = manifest1.getEntries();
const keys = Object.keys(entries);
expect(keys).toEqual([
'endpoint-exceptionlist-linux-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
'endpoint-exceptionlist-macos-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
'endpoint-exceptionlist-windows-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
'endpoint-exceptionlist-linux-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735',
'endpoint-exceptionlist-macos-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735',
'endpoint-exceptionlist-windows-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735',
]);
});

test('Manifest returns true if contains artifact', async () => {
const found = manifest1.contains(
'endpoint-exceptionlist-macos-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c'
'endpoint-exceptionlist-macos-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735'
);
expect(found).toEqual(true);
});
Expand All @@ -144,17 +144,17 @@ describe('manifest', () => {
const manifest = Manifest.fromArtifacts(artifacts, '1.0.0', 'v0');
expect(
manifest.contains(
'endpoint-exceptionlist-linux-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c'
'endpoint-exceptionlist-linux-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735'
)
).toEqual(true);
expect(
manifest.contains(
'endpoint-exceptionlist-macos-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c'
'endpoint-exceptionlist-macos-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735'
)
).toEqual(true);
expect(
manifest.contains(
'endpoint-exceptionlist-windows-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c'
'endpoint-exceptionlist-windows-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735'
)
).toEqual(true);
});
Expand Down
Loading

0 comments on commit f5b77e1

Please sign in to comment.