Skip to content

Commit

Permalink
[SIEM][Detection Engine][Lists] Adds version and immutability data st…
Browse files Browse the repository at this point in the history
…ructures (#72730)

###  Summary

The intent is to get the data structures in similar to rules so that we can have eventually immutable and versioned lists in later releases without too much hassle of upgrading the list and list item data structures.

* Adds version and immutability data structures to the exception lists and the value lists.
* Adds an optional version number to the update route of each so that you can modify the number either direction or you can omit it and it works like the detection rules where it will auto-increment the number.
* Does _not_ add a version and immutability to the exception list items and value list items.
* Does _not_ update the version number when you add a new exception list item or value list item. 

**Examples:**

❯ ./post_list.sh
```json
{
  "_version": "WzAsMV0=",
  "id": "ip_list",
  "created_at": "2020-07-21T20:31:11.679Z",
  "created_by": "yo",
  "description": "This list describes bad internet ip",
  "immutable": false,
  "name": "Simple list with an ip",
  "tie_breaker_id": "d6bd7552-84d1-4f95-88c4-cc504517b4e5",
  "type": "ip",
  "updated_at": "2020-07-21T20:31:11.679Z",
  "updated_by": "yo",
  "version": 1
}
```
❯ ./post_exception_list.sh
```json
{
  "_tags": [
    "endpoint",
    "process",
    "malware",
    "os:linux"
  ],
  "_version": "WzMzOTgsMV0=",
  "created_at": "2020-07-21T20:31:35.933Z",
  "created_by": "yo",
  "description": "This is a sample endpoint type exception",
  "id": "2c24b100-cb91-11ea-a872-adfddf68361e",
  "immutable": false,
  "list_id": "simple_list",
  "name": "Sample Endpoint Exception List",
  "namespace_type": "single",
  "tags": [
    "user added string for a tag",
    "malware"
  ],
  "tie_breaker_id": "c11c4d53-d0be-4904-870e-d33ec7ca387f",
  "type": "detection",
  "updated_at": "2020-07-21T20:31:35.952Z",
  "updated_by": "yo",
  "version": 1
}
```

```json
❯ ./update_list.sh
{
  "_version": "WzEsMV0=",
  "created_at": "2020-07-21T20:31:11.679Z",
  "created_by": "yo",
  "description": "Some other description here for you",
  "id": "ip_list",
  "immutable": false,
  "name": "Changed the name here to something else",
  "tie_breaker_id": "d6bd7552-84d1-4f95-88c4-cc504517b4e5",
  "type": "ip",
  "updated_at": "2020-07-21T20:31:47.089Z",
  "updated_by": "yo",
  "version": 2
}
```

```json
❯ ./update_exception_list.sh
{
  "_tags": [
    "endpoint",
    "process",
    "malware",
    "os:linux"
  ],
  "_version": "WzMzOTksMV0=",
  "created_at": "2020-07-21T20:31:35.933Z",
  "created_by": "yo",
  "description": "Different description",
  "id": "2c24b100-cb91-11ea-a872-adfddf68361e",
  "immutable": false,
  "list_id": "simple_list",
  "name": "Sample Endpoint Exception List",
  "namespace_type": "single",
  "tags": [
    "user added string for a tag",
    "malware"
  ],
  "tie_breaker_id": "c11c4d53-d0be-4904-870e-d33ec7ca387f",
  "type": "endpoint",
  "updated_at": "2020-07-21T20:31:56.628Z",
  "updated_by": "yo",
  "version": 2
}
```

### Checklist

- [x] [Unit or functional tests](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility) were updated or added to match the most common scenarios
  • Loading branch information
FrankHassanabad committed Jul 21, 2020
1 parent c8f7f3b commit 821053c
Show file tree
Hide file tree
Showing 47 changed files with 255 additions and 19 deletions.
2 changes: 2 additions & 0 deletions x-pack/plugins/lists/common/constants.mock.ts
Original file line number Diff line number Diff line change
Expand Up @@ -61,3 +61,5 @@ export const COMMENTS = [];
export const FILTER = 'name:Nicolas Bourbaki';
export const CURSOR = 'c29tZXN0cmluZ2ZvcnlvdQ==';
export const _VERSION = 'WzI5NywxXQ==';
export const VERSION = 1;
export const IMMUTABLE = false;
12 changes: 12 additions & 0 deletions x-pack/plugins/lists/common/schemas/common/schemas.ts
Original file line number Diff line number Diff line change
Expand Up @@ -311,3 +311,15 @@ export type DeserializerOrUndefined = t.TypeOf<typeof deserializerOrUndefined>;
export const _version = t.string;
export const _versionOrUndefined = t.union([_version, t.undefined]);
export type _VersionOrUndefined = t.TypeOf<typeof _versionOrUndefined>;

export const version = t.number;
export type Version = t.TypeOf<typeof version>;

export const versionOrUndefined = t.union([version, t.undefined]);
export type VersionOrUndefined = t.TypeOf<typeof versionOrUndefined>;

export const immutable = t.boolean;
export type Immutable = t.TypeOf<typeof immutable>;

export const immutableOrUndefined = t.union([immutable, t.undefined]);
export type ImmutableOrUndefined = t.TypeOf<typeof immutableOrUndefined>;
Original file line number Diff line number Diff line change
Expand Up @@ -8,23 +8,27 @@ import { IndexEsListSchema } from '../../../common/schemas';
import {
DATE_NOW,
DESCRIPTION,
IMMUTABLE,
META,
NAME,
TIE_BREAKER,
TYPE,
USER,
VERSION,
} from '../../../common/constants.mock';

export const getIndexESListMock = (): IndexEsListSchema => ({
created_at: DATE_NOW,
created_by: USER,
description: DESCRIPTION,
deserializer: undefined,
immutable: IMMUTABLE,
meta: META,
name: NAME,
serializer: undefined,
tie_breaker_id: TIE_BREAKER,
type: TYPE,
updated_at: DATE_NOW,
updated_by: USER,
version: VERSION,
});
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,15 @@ import {
created_by,
description,
deserializerOrUndefined,
immutable,
metaOrUndefined,
name,
serializerOrUndefined,
tie_breaker_id,
type,
updated_at,
updated_by,
version,
} from '../common/schemas';

export const indexEsListSchema = t.exact(
Expand All @@ -28,13 +30,15 @@ export const indexEsListSchema = t.exact(
created_by,
description,
deserializer: deserializerOrUndefined,
immutable,
meta: metaOrUndefined,
name,
serializer: serializerOrUndefined,
tie_breaker_id,
type,
updated_at,
updated_by,
version,
})
);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,15 @@ import { SearchEsListSchema } from '../../../common/schemas';
import {
DATE_NOW,
DESCRIPTION,
IMMUTABLE,
LIST_ID,
LIST_INDEX,
META,
NAME,
TIE_BREAKER,
TYPE,
USER,
VERSION,
} from '../../../common/constants.mock';
import { getShardMock } from '../../get_shard.mock';

Expand All @@ -25,13 +27,15 @@ export const getSearchEsListMock = (): SearchEsListSchema => ({
created_by: USER,
description: DESCRIPTION,
deserializer: undefined,
immutable: IMMUTABLE,
meta: META,
name: NAME,
serializer: undefined,
tie_breaker_id: TIE_BREAKER,
type: TYPE,
updated_at: DATE_NOW,
updated_by: USER,
version: VERSION,
});

export const getSearchListMock = (): SearchResponse<SearchEsListSchema> => ({
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,15 @@ import {
created_by,
description,
deserializerOrUndefined,
immutable,
metaOrUndefined,
name,
serializerOrUndefined,
tie_breaker_id,
type,
updated_at,
updated_by,
version,
} from '../common/schemas';

export const searchEsListSchema = t.exact(
Expand All @@ -28,13 +30,15 @@ export const searchEsListSchema = t.exact(
created_by,
description,
deserializer: deserializerOrUndefined,
immutable,
meta: metaOrUndefined,
name,
serializer: serializerOrUndefined,
tie_breaker_id,
type,
updated_at,
updated_by,
version,
})
);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,14 @@
* you may not use this file except in compliance with the Elastic License.
*/

import { DESCRIPTION, ENDPOINT_TYPE, META, NAME, NAMESPACE_TYPE } from '../../constants.mock';
import {
DESCRIPTION,
ENDPOINT_TYPE,
META,
NAME,
NAMESPACE_TYPE,
VERSION,
} from '../../constants.mock';

import { CreateExceptionListSchema } from './create_exception_list_schema';

Expand All @@ -17,4 +24,5 @@ export const getCreateExceptionListSchemaMock = (): CreateExceptionListSchema =>
namespace_type: NAMESPACE_TYPE,
tags: [],
type: ENDPOINT_TYPE,
version: VERSION,
});
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,11 @@ import {
tags,
} from '../common/schemas';
import { RequiredKeepUndefined } from '../../types';
import { DefaultUuid } from '../../siem_common_deps';
import {
DefaultUuid,
DefaultVersionNumber,
DefaultVersionNumberDecoded,
} from '../../siem_common_deps';
import { NamespaceType } from '../types';

export const createExceptionListSchema = t.intersection([
Expand All @@ -39,6 +43,7 @@ export const createExceptionListSchema = t.intersection([
meta, // defaults to undefined if not set during decode
namespace_type, // defaults to 'single' if not set during decode
tags, // defaults to empty array if not set during decode
version: DefaultVersionNumber, // defaults to numerical 1 if not set during decode
})
),
]);
Expand All @@ -54,4 +59,5 @@ export type CreateExceptionListSchemaDecoded = Omit<
tags: Tags;
list_id: ListId;
namespace_type: NamespaceType;
version: DefaultVersionNumberDecoded;
};
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
* you may not use this file except in compliance with the Elastic License.
*/

import { DESCRIPTION, LIST_ID, META, NAME, TYPE } from '../../constants.mock';
import { DESCRIPTION, LIST_ID, META, NAME, TYPE, VERSION } from '../../constants.mock';

import { CreateListSchema } from './create_list_schema';

Expand All @@ -16,4 +16,5 @@ export const getCreateListSchemaMock = (): CreateListSchema => ({
name: NAME,
serializer: undefined,
type: TYPE,
version: VERSION,
});
15 changes: 13 additions & 2 deletions x-pack/plugins/lists/common/schemas/request/create_list_schema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import * as t from 'io-ts';

import { description, deserializer, id, meta, name, serializer, type } from '../common/schemas';
import { RequiredKeepUndefined } from '../../types';
import { DefaultVersionNumber, DefaultVersionNumberDecoded } from '../../siem_common_deps';

export const createListSchema = t.intersection([
t.exact(
Expand All @@ -17,8 +18,18 @@ export const createListSchema = t.intersection([
type,
})
),
t.exact(t.partial({ deserializer, id, meta, serializer })),
t.exact(
t.partial({
deserializer, // defaults to undefined if not set during decode
id, // defaults to undefined if not set during decode
meta, // defaults to undefined if not set during decode
serializer, // defaults to undefined if not set during decode
version: DefaultVersionNumber, // defaults to a numerical 1 if not set during decode
})
),
]);

export type CreateListSchema = t.OutputOf<typeof createListSchema>;
export type CreateListSchemaDecoded = RequiredKeepUndefined<t.TypeOf<typeof createListSchema>>;
export type CreateListSchemaDecoded = RequiredKeepUndefined<
Omit<t.TypeOf<typeof createListSchema>, 'version'>
> & { version: DefaultVersionNumberDecoded };
12 changes: 10 additions & 2 deletions x-pack/plugins/lists/common/schemas/request/patch_list_schema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

import * as t from 'io-ts';

import { _version, description, id, meta, name } from '../common/schemas';
import { _version, description, id, meta, name, version } from '../common/schemas';
import { RequiredKeepUndefined } from '../../types';

export const patchListSchema = t.intersection([
Expand All @@ -17,7 +17,15 @@ export const patchListSchema = t.intersection([
id,
})
),
t.exact(t.partial({ _version, description, meta, name })),
t.exact(
t.partial({
_version, // is undefined if not set during decode
description, // is undefined if not set during decode
meta, // is undefined if not set during decode
name, // is undefined if not set during decode
version, // is undefined if not set during decode
})
),
]);

export type PatchListSchema = t.OutputOf<typeof patchListSchema>;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import {
name,
namespace_type,
tags,
version,
} from '../common/schemas';
import { RequiredKeepUndefined } from '../../types';
import { NamespaceType } from '../types';
Expand All @@ -42,6 +43,7 @@ export const updateExceptionListSchema = t.intersection([
meta, // defaults to undefined if not set during decode
namespace_type, // defaults to 'single' if not set during decode
tags, // defaults to empty array if not set during decode
version, // defaults to undefined if not set during decode
})
),
]);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

import * as t from 'io-ts';

import { _version, description, id, meta, name } from '../common/schemas';
import { _version, description, id, meta, name, version } from '../common/schemas';
import { RequiredKeepUndefined } from '../../types';

export const updateListSchema = t.intersection([
Expand All @@ -23,6 +23,7 @@ export const updateListSchema = t.intersection([
t.partial({
_version, // defaults to undefined if not set during decode
meta, // defaults to undefined if not set during decode
version, // defaults to undefined if not set during decode
})
),
]);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ describe('create_endpoint_list_schema', () => {
const message = pipe(checked, foldLeftRight);

expect(getPaths(left(message.errors))).toEqual([
'invalid keys "_tags,["endpoint","process","malware","os:linux"],_version,created_at,created_by,description,id,meta,{},name,namespace_type,tags,["user added string for a tag","malware"],tie_breaker_id,type,updated_at,updated_by"',
'invalid keys "_tags,["endpoint","process","malware","os:linux"],_version,created_at,created_by,description,id,immutable,meta,{},name,namespace_type,tags,["user added string for a tag","malware"],tie_breaker_id,type,updated_at,updated_by,version"',
]);
expect(message.schema).toEqual({});
});
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,11 @@ import {
DATE_NOW,
DESCRIPTION,
ENDPOINT_TYPE,
IMMUTABLE,
META,
TIE_BREAKER,
USER,
VERSION,
_VERSION,
} from '../../constants.mock';
import { ENDPOINT_LIST_ID } from '../..';
Expand All @@ -23,6 +25,7 @@ export const getExceptionListSchemaMock = (): ExceptionListSchema => ({
created_by: USER,
description: DESCRIPTION,
id: '1',
immutable: IMMUTABLE,
list_id: ENDPOINT_LIST_ID,
meta: META,
name: 'Sample Endpoint Exception List',
Expand All @@ -32,4 +35,5 @@ export const getExceptionListSchemaMock = (): ExceptionListSchema => ({
type: ENDPOINT_TYPE,
updated_at: DATE_NOW,
updated_by: 'user_name',
version: VERSION,
});
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import {
description,
exceptionListType,
id,
immutable,
list_id,
metaOrUndefined,
name,
Expand All @@ -24,6 +25,7 @@ import {
tie_breaker_id,
updated_at,
updated_by,
version,
} from '../common/schemas';

export const exceptionListSchema = t.exact(
Expand All @@ -34,6 +36,7 @@ export const exceptionListSchema = t.exact(
created_by,
description,
id,
immutable,
list_id,
meta: metaOrUndefined,
name,
Expand All @@ -43,6 +46,7 @@ export const exceptionListSchema = t.exact(
type: exceptionListType,
updated_at,
updated_by,
version,
})
);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,14 @@ import { ListSchema } from '../../../common/schemas';
import {
DATE_NOW,
DESCRIPTION,
IMMUTABLE,
LIST_ID,
META,
NAME,
TIE_BREAKER,
TYPE,
USER,
VERSION,
} from '../../../common/constants.mock';

export const getListResponseMock = (): ListSchema => ({
Expand All @@ -23,11 +25,13 @@ export const getListResponseMock = (): ListSchema => ({
description: DESCRIPTION,
deserializer: undefined,
id: LIST_ID,
immutable: IMMUTABLE,
meta: META,
name: NAME,
serializer: undefined,
tie_breaker_id: TIE_BREAKER,
type: TYPE,
updated_at: DATE_NOW,
updated_by: USER,
version: VERSION,
});
Loading

0 comments on commit 821053c

Please sign in to comment.