diff --git a/x-pack/plugins/lists/README.md b/x-pack/plugins/lists/README.md index 884324c68cc82..b6061368f6b13 100644 --- a/x-pack/plugins/lists/README.md +++ b/x-pack/plugins/lists/README.md @@ -69,7 +69,7 @@ You should see the new list created like so: ```sh { - "id": "list-ip", + "id": "list_ip", "created_at": "2020-05-28T19:15:22.344Z", "created_by": "yo", "description": "This list describes bad internet ip", @@ -96,7 +96,7 @@ You should see the new list item created and attached to the above list like so: "value": "127.0.0.1", "created_at": "2020-05-28T19:15:49.790Z", "created_by": "yo", - "list_id": "list-ip", + "list_id": "list_ip", "tie_breaker_id": "a881bf2e-1e17-4592-bba8-d567cb07d234", "updated_at": "2020-05-28T19:15:49.790Z", "updated_by": "yo" @@ -195,7 +195,7 @@ You can then do find for each one like so: "cursor": "WzIwLFsiYzU3ZWZiYzQtNDk3Ny00YTMyLTk5NWYtY2ZkMjk2YmVkNTIxIl1d", "data": [ { - "id": "list-ip", + "id": "list_ip", "created_at": "2020-05-28T19:15:22.344Z", "created_by": "yo", "description": "This list describes bad internet ip", diff --git a/x-pack/plugins/lists/common/get_call_cluster.mock.ts b/x-pack/plugins/lists/common/get_call_cluster.mock.ts index be6f1b3a58629..0c2e761badc7c 100644 --- a/x-pack/plugins/lists/common/get_call_cluster.mock.ts +++ b/x-pack/plugins/lists/common/get_call_cluster.mock.ts @@ -21,5 +21,15 @@ export const getEmptyCreateDocumentResponseMock = (): CreateDocumentResponse => }); export const getCallClusterMock = ( - callCluster: unknown = getEmptyCreateDocumentResponseMock() -): LegacyAPICaller => jest.fn().mockResolvedValue(callCluster); + response: unknown = getEmptyCreateDocumentResponseMock() +): LegacyAPICaller => jest.fn().mockResolvedValue(response); + +export const getCallClusterMockMultiTimes = ( + responses: unknown[] = [getEmptyCreateDocumentResponseMock()] +): LegacyAPICaller => { + const returnJest = jest.fn(); + responses.forEach((response) => { + returnJest.mockResolvedValueOnce(response); + }); + return returnJest; +}; diff --git a/x-pack/plugins/lists/common/schemas/common/schemas.test.ts b/x-pack/plugins/lists/common/schemas/common/schemas.test.ts index f08f98d56913e..d450debd56293 100644 --- a/x-pack/plugins/lists/common/schemas/common/schemas.test.ts +++ b/x-pack/plugins/lists/common/schemas/common/schemas.test.ts @@ -7,9 +7,27 @@ import { pipe } from 'fp-ts/lib/pipeable'; import { left } from 'fp-ts/lib/Either'; -import { foldLeftRight, getPaths } from '../../siem_common_deps'; +import { exactCheck, foldLeftRight, getPaths } from '../../siem_common_deps'; -import { exceptionListType, operator, operator_type as operatorType } from './schemas'; +import { + EsDataTypeGeoPoint, + EsDataTypeGeoPointRange, + EsDataTypeRange, + EsDataTypeRangeTerm, + EsDataTypeSingle, + EsDataTypeUnion, + Type, + esDataTypeGeoPoint, + esDataTypeGeoPointRange, + esDataTypeRange, + esDataTypeRangeTerm, + esDataTypeSingle, + esDataTypeUnion, + exceptionListType, + operator, + operator_type as operatorType, + type, +} from './schemas'; describe('Common schemas', () => { describe('operatorType', () => { @@ -122,4 +140,289 @@ describe('Common schemas', () => { expect(keys.length).toEqual(2); }); }); + + describe('type', () => { + test('it will work with a given expected type', () => { + const payload: Type = 'keyword'; + const decoded = type.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + test('it will give an error if given a type that does not exist', () => { + const payload: Type | 'madeup' = 'madeup'; + const decoded = type.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "madeup" supplied to ""binary" | "boolean" | "byte" | "date" | "date_nanos" | "date_range" | "double" | "double_range" | "float" | "float_range" | "geo_point" | "geo_shape" | "half_float" | "integer" | "integer_range" | "ip" | "ip_range" | "keyword" | "long" | "long_range" | "shape" | "short" | "text""', + ]); + expect(message.schema).toEqual({}); + }); + }); + + describe('esDataTypeRange', () => { + test('it will work with a given gte, lte range', () => { + const payload: EsDataTypeRange = { gte: '127.0.0.1', lte: '127.0.0.1' }; + const decoded = esDataTypeRange.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + test('it will give an error if given an extra madeup value', () => { + const payload: EsDataTypeRange & { madeupvalue: string } = { + gte: '127.0.0.1', + lte: '127.0.0.1', + madeupvalue: 'something', + }; + const decoded = esDataTypeRange.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual(['invalid keys "madeupvalue"']); + expect(message.schema).toEqual({}); + }); + }); + + describe('esDataTypeRangeTerm', () => { + test('it will work with a date_range', () => { + const payload: EsDataTypeRangeTerm = { date_range: { gte: '2015', lte: '2017' } }; + const decoded = esDataTypeRangeTerm.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + test('it will give an error if given an extra madeup value for date_range', () => { + const payload: EsDataTypeRangeTerm & { madeupvalue: string } = { + date_range: { gte: '2015', lte: '2017' }, + madeupvalue: 'something', + }; + const decoded = esDataTypeRangeTerm.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual(['invalid keys "madeupvalue"']); + expect(message.schema).toEqual({}); + }); + + test('it will work with a double_range', () => { + const payload: EsDataTypeRangeTerm = { double_range: { gte: '2015', lte: '2017' } }; + const decoded = esDataTypeRangeTerm.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + test('it will give an error if given an extra madeup value for double_range', () => { + const payload: EsDataTypeRangeTerm & { madeupvalue: string } = { + double_range: { gte: '2015', lte: '2017' }, + madeupvalue: 'something', + }; + const decoded = esDataTypeRangeTerm.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual(['invalid keys "madeupvalue"']); + expect(message.schema).toEqual({}); + }); + + test('it will work with a float_range', () => { + const payload: EsDataTypeRangeTerm = { float_range: { gte: '2015', lte: '2017' } }; + const decoded = esDataTypeRangeTerm.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + test('it will give an error if given an extra madeup value for float_range', () => { + const payload: EsDataTypeRangeTerm & { madeupvalue: string } = { + float_range: { gte: '2015', lte: '2017' }, + madeupvalue: 'something', + }; + const decoded = esDataTypeRangeTerm.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual(['invalid keys "madeupvalue"']); + expect(message.schema).toEqual({}); + }); + + test('it will work with a integer_range', () => { + const payload: EsDataTypeRangeTerm = { integer_range: { gte: '2015', lte: '2017' } }; + const decoded = esDataTypeRangeTerm.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + test('it will give an error if given an extra madeup value for integer_range', () => { + const payload: EsDataTypeRangeTerm & { madeupvalue: string } = { + integer_range: { gte: '2015', lte: '2017' }, + madeupvalue: 'something', + }; + const decoded = esDataTypeRangeTerm.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual(['invalid keys "madeupvalue"']); + expect(message.schema).toEqual({}); + }); + + test('it will work with a ip_range', () => { + const payload: EsDataTypeRangeTerm = { ip_range: { gte: '127.0.0.1', lte: '127.0.0.2' } }; + const decoded = esDataTypeRangeTerm.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + test('it will work with a ip_range as a CIDR', () => { + const payload: EsDataTypeRangeTerm = { ip_range: '127.0.0.1/16' }; + const decoded = esDataTypeRangeTerm.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + test('it will give an error if given an extra madeup value for ip_range', () => { + const payload: EsDataTypeRangeTerm & { madeupvalue: string } = { + ip_range: { gte: '127.0.0.1', lte: '127.0.0.2' }, + madeupvalue: 'something', + }; + const decoded = esDataTypeRangeTerm.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual(['invalid keys "madeupvalue"']); + expect(message.schema).toEqual({}); + }); + + test('it will work with a long_range', () => { + const payload: EsDataTypeRangeTerm = { long_range: { gte: '2015', lte: '2017' } }; + const decoded = esDataTypeRangeTerm.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + test('it will give an error if given an extra madeup value for long_range', () => { + const payload: EsDataTypeRangeTerm & { madeupvalue: string } = { + long_range: { gte: '2015', lte: '2017' }, + madeupvalue: 'something', + }; + const decoded = esDataTypeRangeTerm.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual(['invalid keys "madeupvalue"']); + expect(message.schema).toEqual({}); + }); + }); + + describe('esDataTypeGeoPointRange', () => { + test('it will work with a given lat, lon range', () => { + const payload: EsDataTypeGeoPointRange = { lat: '20', lon: '30' }; + const decoded = esDataTypeGeoPointRange.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + test('it will give an error if given an extra madeup value', () => { + const payload: EsDataTypeGeoPointRange & { madeupvalue: string } = { + lat: '20', + lon: '30', + madeupvalue: 'something', + }; + const decoded = esDataTypeGeoPointRange.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual(['invalid keys "madeupvalue"']); + expect(message.schema).toEqual({}); + }); + }); + + describe('esDataTypeGeoPoint', () => { + test('it will work with a given lat, lon range', () => { + const payload: EsDataTypeGeoPoint = { geo_point: { lat: '127.0.0.1', lon: '127.0.0.1' } }; + const decoded = esDataTypeGeoPoint.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + test('it will work with a WKT (Well known text)', () => { + const payload: EsDataTypeGeoPoint = { geo_point: 'POINT (30 10)' }; + const decoded = esDataTypeGeoPoint.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + test('it will give an error if given an extra madeup value', () => { + const payload: EsDataTypeGeoPoint & { madeupvalue: string } = { + geo_point: 'POINT (30 10)', + madeupvalue: 'something', + }; + const decoded = esDataTypeGeoPoint.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual(['invalid keys "madeupvalue"']); + expect(message.schema).toEqual({}); + }); + }); + + describe('esDataTypeSingle', () => { + test('it will work with single type', () => { + const payload: EsDataTypeSingle = { boolean: 'true' }; + const decoded = esDataTypeSingle.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + test('it will not work with a madeup value', () => { + const payload: EsDataTypeSingle & { madeupValue: 'madeup' } = { + boolean: 'true', + madeupValue: 'madeup', + }; + const decoded = esDataTypeSingle.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual(['invalid keys "madeupValue"']); + expect(message.schema).toEqual({}); + }); + }); + + describe('esDataTypeUnion', () => { + test('it will work with a regular union', () => { + const payload: EsDataTypeUnion = { boolean: 'true' }; + const decoded = esDataTypeUnion.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + test('it will not work with a madeup value', () => { + const payload: EsDataTypeUnion & { madeupValue: 'madeupValue' } = { + boolean: 'true', + madeupValue: 'madeupValue', + }; + const decoded = esDataTypeUnion.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual(['invalid keys "madeupValue"']); + expect(message.schema).toEqual({}); + }); + }); }); diff --git a/x-pack/plugins/lists/common/schemas/common/schemas.ts b/x-pack/plugins/lists/common/schemas/common/schemas.ts index 6a6a2dc74b283..6bb6ee05034cb 100644 --- a/x-pack/plugins/lists/common/schemas/common/schemas.ts +++ b/x-pack/plugins/lists/common/schemas/common/schemas.ts @@ -38,19 +38,85 @@ export type Id = t.TypeOf; export const idOrUndefined = t.union([id, t.undefined]); export type IdOrUndefined = t.TypeOf; +export const binary = t.string; +export const binaryOrUndefined = t.union([binary, t.undefined]); + +export const boolean = t.string; +export const booleanOrUndefined = t.union([boolean, t.undefined]); + +export const byte = t.string; +export const byteOrUndefined = t.union([byte, t.undefined]); + +export const date = t.string; +export const dateOrUndefined = t.union([date, t.undefined]); + +export const date_nanos = t.string; +export const dateNanosOrUndefined = t.union([date_nanos, t.undefined]); + +export const double = t.string; +export const doubleOrUndefined = t.union([double, t.undefined]); + +export const float = t.string; +export const floatOrUndefined = t.union([float, t.undefined]); + +export const geo_shape = t.string; +export const geoShapeOrUndefined = t.union([geo_shape, t.undefined]); + +export const half_float = t.string; +export const halfFloatOrUndefined = t.union([half_float, t.undefined]); + +export const integer = t.string; +export const integerOrUndefined = t.union([integer, t.undefined]); + export const ip = t.string; export const ipOrUndefined = t.union([ip, t.undefined]); export const keyword = t.string; export const keywordOrUndefined = t.union([keyword, t.undefined]); +export const text = t.string; +export const textOrUndefined = t.union([text, t.undefined]); + +export const long = t.string; +export const longOrUndefined = t.union([long, t.undefined]); + +export const shape = t.string; +export const shapeOrUndefined = t.union([shape, t.undefined]); + +export const short = t.string; +export const shortOrUndefined = t.union([short, t.undefined]); + export const value = t.string; export const valueOrUndefined = t.union([value, t.undefined]); export const tie_breaker_id = t.string; // TODO: Use UUID for this instead of a string for validation export const _index = t.string; -export const type = t.keyof({ ip: null, keyword: null }); // TODO: Add the other data types here +export const type = t.keyof({ + binary: null, + boolean: null, + byte: null, + date: null, + date_nanos: null, + date_range: null, + double: null, + double_range: null, + float: null, + float_range: null, + geo_point: null, + geo_shape: null, + half_float: null, + integer: null, + integer_range: null, + ip: null, + ip_range: null, + keyword: null, + long: null, + long_range: null, + shape: null, + short: null, + text: null, +}); export const typeOrUndefined = t.union([type, t.undefined]); export type Type = t.TypeOf; @@ -60,7 +126,84 @@ export type Meta = t.TypeOf; export const metaOrUndefined = t.union([meta, t.undefined]); export type MetaOrUndefined = t.TypeOf; -export const esDataTypeUnion = t.union([t.type({ ip }), t.type({ keyword })]); +export const esDataTypeRange = t.exact(t.type({ gte: t.string, lte: t.string })); + +export const date_range = esDataTypeRange; +export const dateRangeOrUndefined = t.union([date_range, t.undefined]); + +export const double_range = esDataTypeRange; +export const doubleRangeOrUndefined = t.union([double_range, t.undefined]); + +export const float_range = esDataTypeRange; +export const floatRangeOrUndefined = t.union([float_range, t.undefined]); + +export const integer_range = esDataTypeRange; +export const integerRangeOrUndefined = t.union([integer_range, t.undefined]); + +// ip_range can be just a CIDR value as a range +export const ip_range = t.union([esDataTypeRange, t.string]); +export const ipRangeOrUndefined = t.union([ip_range, t.undefined]); + +export const long_range = esDataTypeRange; +export const longRangeOrUndefined = t.union([long_range, t.undefined]); + +export type EsDataTypeRange = t.TypeOf; + +export const esDataTypeRangeTerm = t.union([ + t.exact(t.type({ date_range })), + t.exact(t.type({ double_range })), + t.exact(t.type({ float_range })), + t.exact(t.type({ integer_range })), + t.exact(t.type({ ip_range })), + t.exact(t.type({ long_range })), +]); + +export type EsDataTypeRangeTerm = t.TypeOf; + +export const esDataTypeGeoPointRange = t.exact(t.type({ lat: t.string, lon: t.string })); +export type EsDataTypeGeoPointRange = t.TypeOf; + +export const geo_point = t.union([esDataTypeGeoPointRange, t.string]); +export type GeoPoint = t.TypeOf; + +export const geoPointOrUndefined = t.union([geo_point, t.undefined]); + +export const esDataTypeGeoPoint = t.exact(t.type({ geo_point })); +export type EsDataTypeGeoPoint = t.TypeOf; + +export const esDataTypeGeoShape = t.union([ + t.exact(t.type({ geo_shape: t.string })), + t.exact(t.type({ shape: t.string })), +]); + +export type EsDataTypeGeoShape = t.TypeOf; + +export const esDataTypeSingle = t.union([ + t.exact(t.type({ binary })), + t.exact(t.type({ boolean })), + t.exact(t.type({ byte })), + t.exact(t.type({ date })), + t.exact(t.type({ date_nanos })), + t.exact(t.type({ double })), + t.exact(t.type({ float })), + t.exact(t.type({ half_float })), + t.exact(t.type({ integer })), + t.exact(t.type({ ip })), + t.exact(t.type({ keyword })), + t.exact(t.type({ long })), + t.exact(t.type({ short })), + t.exact(t.type({ text })), +]); + +export type EsDataTypeSingle = t.TypeOf; + +export const esDataTypeUnion = t.union([ + esDataTypeRangeTerm, + esDataTypeGeoPoint, + esDataTypeGeoShape, + esDataTypeSingle, +]); + export type EsDataTypeUnion = t.TypeOf; export const tags = DefaultStringArray; @@ -153,3 +296,15 @@ export enum OperatorTypeEnum { EXISTS = 'exists', LIST = 'list', } + +export const serializer = t.string; +export type Serializer = t.TypeOf; + +export const serializerOrUndefined = t.union([serializer, t.undefined]); +export type SerializerOrUndefined = t.TypeOf; + +export const deserializer = t.string; +export type Deserializer = t.TypeOf; + +export const deserializerOrUndefined = t.union([deserializer, t.undefined]); +export type DeserializerOrUndefined = t.TypeOf; diff --git a/x-pack/plugins/lists/common/schemas/elastic_query/index_es_list_item_schema.mock.ts b/x-pack/plugins/lists/common/schemas/elastic_query/index_es_list_item_schema.mock.ts index 1e27e48aac310..0f1724e09286c 100644 --- a/x-pack/plugins/lists/common/schemas/elastic_query/index_es_list_item_schema.mock.ts +++ b/x-pack/plugins/lists/common/schemas/elastic_query/index_es_list_item_schema.mock.ts @@ -10,9 +10,11 @@ import { DATE_NOW, LIST_ID, META, TIE_BREAKER, USER, VALUE } from '../../../comm export const getIndexESListItemMock = (ip = VALUE): IndexEsListItemSchema => ({ created_at: DATE_NOW, created_by: USER, + deserializer: undefined, ip, list_id: LIST_ID, meta: META, + serializer: undefined, tie_breaker_id: TIE_BREAKER, updated_at: DATE_NOW, updated_by: USER, diff --git a/x-pack/plugins/lists/common/schemas/elastic_query/index_es_list_item_schema.ts b/x-pack/plugins/lists/common/schemas/elastic_query/index_es_list_item_schema.ts index 596498b64b771..006600ee5b7fd 100644 --- a/x-pack/plugins/lists/common/schemas/elastic_query/index_es_list_item_schema.ts +++ b/x-pack/plugins/lists/common/schemas/elastic_query/index_es_list_item_schema.ts @@ -11,9 +11,11 @@ import * as t from 'io-ts'; import { created_at, created_by, + deserializerOrUndefined, esDataTypeUnion, list_id, metaOrUndefined, + serializerOrUndefined, tie_breaker_id, updated_at, updated_by, @@ -24,8 +26,10 @@ export const indexEsListItemSchema = t.intersection([ t.type({ created_at, created_by, + deserializer: deserializerOrUndefined, list_id, meta: metaOrUndefined, + serializer: serializerOrUndefined, tie_breaker_id, updated_at, updated_by, diff --git a/x-pack/plugins/lists/common/schemas/elastic_query/index_es_list_schema.mock.ts b/x-pack/plugins/lists/common/schemas/elastic_query/index_es_list_schema.mock.ts index a6411ebce84b6..85a6b1362a582 100644 --- a/x-pack/plugins/lists/common/schemas/elastic_query/index_es_list_schema.mock.ts +++ b/x-pack/plugins/lists/common/schemas/elastic_query/index_es_list_schema.mock.ts @@ -19,8 +19,10 @@ export const getIndexESListMock = (): IndexEsListSchema => ({ created_at: DATE_NOW, created_by: USER, description: DESCRIPTION, + deserializer: undefined, meta: META, name: NAME, + serializer: undefined, tie_breaker_id: TIE_BREAKER, type: TYPE, updated_at: DATE_NOW, diff --git a/x-pack/plugins/lists/common/schemas/elastic_query/index_es_list_schema.ts b/x-pack/plugins/lists/common/schemas/elastic_query/index_es_list_schema.ts index e0924392628a9..fd1018bc46a8d 100644 --- a/x-pack/plugins/lists/common/schemas/elastic_query/index_es_list_schema.ts +++ b/x-pack/plugins/lists/common/schemas/elastic_query/index_es_list_schema.ts @@ -12,8 +12,10 @@ import { created_at, created_by, description, + deserializerOrUndefined, metaOrUndefined, name, + serializerOrUndefined, tie_breaker_id, type, updated_at, @@ -25,8 +27,10 @@ export const indexEsListSchema = t.exact( created_at, created_by, description, + deserializer: deserializerOrUndefined, meta: metaOrUndefined, name, + serializer: serializerOrUndefined, tie_breaker_id, type, updated_at, diff --git a/x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_item_schema.mock.ts b/x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_item_schema.mock.ts index ba69bee9ccf77..c8017c9c1279a 100644 --- a/x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_item_schema.mock.ts +++ b/x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_item_schema.mock.ts @@ -19,18 +19,46 @@ import { } from '../../../common/constants.mock'; import { getShardMock } from '../../get_shard.mock'; -export const getSearchEsListItemMock = (): SearchEsListItemSchema => ({ +export const getSearchEsListItemsAsAllUndefinedMock = (): SearchEsListItemSchema => ({ + binary: undefined, + boolean: undefined, + byte: undefined, created_at: DATE_NOW, created_by: USER, - ip: VALUE, + date: undefined, + date_nanos: undefined, + date_range: undefined, + deserializer: undefined, + double: undefined, + double_range: undefined, + float: undefined, + float_range: undefined, + geo_point: undefined, + geo_shape: undefined, + half_float: undefined, + integer: undefined, + integer_range: undefined, + ip: undefined, + ip_range: undefined, keyword: undefined, list_id: LIST_ID, + long: undefined, + long_range: undefined, meta: META, + serializer: undefined, + shape: undefined, + short: undefined, + text: undefined, tie_breaker_id: TIE_BREAKER, updated_at: DATE_NOW, updated_by: USER, }); +export const getSearchEsListItemMock = (): SearchEsListItemSchema => ({ + ...getSearchEsListItemsAsAllUndefinedMock(), + ip: VALUE, +}); + export const getSearchListItemMock = (): SearchResponse => ({ _scroll_id: '123', _shards: getShardMock(), diff --git a/x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_item_schema.test.ts b/x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_item_schema.test.ts new file mode 100644 index 0000000000000..7ac75b077acb5 --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_item_schema.test.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { pipe } from 'fp-ts/lib/pipeable'; +import { left } from 'fp-ts/lib/Either'; + +import { exactCheck, foldLeftRight, getPaths } from '../../siem_common_deps'; + +import { SearchEsListItemSchema, searchEsListItemSchema } from './search_es_list_item_schema'; +import { getSearchEsListItemMock } from './search_es_list_item_schema.mock'; + +describe('search_es_list_item_schema', () => { + test('it should validate against the mock', () => { + const payload: SearchEsListItemSchema = getSearchEsListItemMock(); + const decoded = searchEsListItemSchema.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + test('it should not validate with a madeup value', () => { + const payload: SearchEsListItemSchema & { madeupValue: string } = { + ...getSearchEsListItemMock(), + madeupValue: 'madeupvalue', + }; + const decoded = searchEsListItemSchema.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual(['invalid keys "madeupValue"']); + expect(message.schema).toEqual({}); + }); +}); diff --git a/x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_item_schema.ts b/x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_item_schema.ts index 902d3e6a9896e..76419587c5925 100644 --- a/x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_item_schema.ts +++ b/x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_item_schema.ts @@ -9,12 +9,35 @@ import * as t from 'io-ts'; import { + binaryOrUndefined, + booleanOrUndefined, + byteOrUndefined, created_at, created_by, + dateNanosOrUndefined, + dateOrUndefined, + dateRangeOrUndefined, + deserializerOrUndefined, + doubleOrUndefined, + doubleRangeOrUndefined, + floatOrUndefined, + floatRangeOrUndefined, + geoPointOrUndefined, + geoShapeOrUndefined, + halfFloatOrUndefined, + integerOrUndefined, + integerRangeOrUndefined, ipOrUndefined, + ipRangeOrUndefined, keywordOrUndefined, list_id, + longOrUndefined, + longRangeOrUndefined, metaOrUndefined, + serializerOrUndefined, + shapeOrUndefined, + shortOrUndefined, + textOrUndefined, tie_breaker_id, updated_at, updated_by, @@ -22,12 +45,35 @@ import { export const searchEsListItemSchema = t.exact( t.type({ + binary: binaryOrUndefined, + boolean: booleanOrUndefined, + byte: byteOrUndefined, created_at, created_by, + date: dateOrUndefined, + date_nanos: dateNanosOrUndefined, + date_range: dateRangeOrUndefined, + deserializer: deserializerOrUndefined, + double: doubleOrUndefined, + double_range: doubleRangeOrUndefined, + float: floatOrUndefined, + float_range: floatRangeOrUndefined, + geo_point: geoPointOrUndefined, + geo_shape: geoShapeOrUndefined, + half_float: halfFloatOrUndefined, + integer: integerOrUndefined, + integer_range: integerRangeOrUndefined, ip: ipOrUndefined, + ip_range: ipRangeOrUndefined, keyword: keywordOrUndefined, list_id, + long: longOrUndefined, + long_range: longRangeOrUndefined, meta: metaOrUndefined, + serializer: serializerOrUndefined, + shape: shapeOrUndefined, + short: shortOrUndefined, + text: textOrUndefined, tie_breaker_id, updated_at, updated_by, diff --git a/x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_schema.mock.ts b/x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_schema.mock.ts index ca9c4e16c6939..703d0d0f654a8 100644 --- a/x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_schema.mock.ts +++ b/x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_schema.mock.ts @@ -24,8 +24,10 @@ export const getSearchEsListMock = (): SearchEsListSchema => ({ created_at: DATE_NOW, created_by: USER, description: DESCRIPTION, + deserializer: undefined, meta: META, name: NAME, + serializer: undefined, tie_breaker_id: TIE_BREAKER, type: TYPE, updated_at: DATE_NOW, @@ -51,3 +53,15 @@ export const getSearchListMock = (): SearchResponse => ({ timed_out: false, took: 10, }); + +export const getEmptySearchListMock = (): SearchResponse => ({ + _scroll_id: '123', + _shards: getShardMock(), + hits: { + hits: [], + max_score: 0, + total: 0, + }, + timed_out: false, + took: 10, +}); diff --git a/x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_schema.test.ts b/x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_schema.test.ts new file mode 100644 index 0000000000000..739f102e6a872 --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_schema.test.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { pipe } from 'fp-ts/lib/pipeable'; +import { left } from 'fp-ts/lib/Either'; + +import { exactCheck, foldLeftRight, getPaths } from '../../siem_common_deps'; + +import { SearchEsListSchema, searchEsListSchema } from './search_es_list_schema'; +import { getSearchEsListMock } from './search_es_list_schema.mock'; + +describe('search_es_list_schema', () => { + test('it should validate against the mock', () => { + const payload: SearchEsListSchema = getSearchEsListMock(); + const decoded = searchEsListSchema.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + test('it should not validate with a madeup value', () => { + const payload: SearchEsListSchema & { madeupValue: string } = { + ...getSearchEsListMock(), + madeupValue: 'madeupvalue', + }; + const decoded = searchEsListSchema.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual(['invalid keys "madeupValue"']); + expect(message.schema).toEqual({}); + }); +}); diff --git a/x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_schema.ts b/x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_schema.ts index 00a7c6f321d38..46005b81ef680 100644 --- a/x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_schema.ts +++ b/x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_schema.ts @@ -12,8 +12,10 @@ import { created_at, created_by, description, + deserializerOrUndefined, metaOrUndefined, name, + serializerOrUndefined, tie_breaker_id, type, updated_at, @@ -25,8 +27,10 @@ export const searchEsListSchema = t.exact( created_at, created_by, description, + deserializer: deserializerOrUndefined, meta: metaOrUndefined, name, + serializer: serializerOrUndefined, tie_breaker_id, type, updated_at, diff --git a/x-pack/plugins/lists/common/schemas/request/create_list_schema.mock.ts b/x-pack/plugins/lists/common/schemas/request/create_list_schema.mock.ts index 7e6d8bb5ad803..482fabb3b997f 100644 --- a/x-pack/plugins/lists/common/schemas/request/create_list_schema.mock.ts +++ b/x-pack/plugins/lists/common/schemas/request/create_list_schema.mock.ts @@ -10,8 +10,10 @@ import { CreateListSchema } from './create_list_schema'; export const getCreateListSchemaMock = (): CreateListSchema => ({ description: DESCRIPTION, + deserializer: undefined, id: LIST_ID, meta: META, name: NAME, + serializer: undefined, type: TYPE, }); diff --git a/x-pack/plugins/lists/common/schemas/request/create_list_schema.test.ts b/x-pack/plugins/lists/common/schemas/request/create_list_schema.test.ts index c4456bf97865a..9b496a01045de 100644 --- a/x-pack/plugins/lists/common/schemas/request/create_list_schema.test.ts +++ b/x-pack/plugins/lists/common/schemas/request/create_list_schema.test.ts @@ -43,6 +43,26 @@ describe('create_list_schema', () => { expect(message.schema).toEqual(payload); }); + test('it should accept an undefined for serializer', () => { + const payload = getCreateListSchemaMock(); + delete payload.serializer; + const decoded = createListSchema.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + test('it should accept an undefined for deserializer', () => { + const payload = getCreateListSchemaMock(); + delete payload.deserializer; + const decoded = createListSchema.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + test('it should not allow an extra key to be sent in', () => { const payload: CreateListSchema & { extraKey?: string } = getCreateListSchemaMock(); payload.extraKey = 'some new value'; diff --git a/x-pack/plugins/lists/common/schemas/request/create_list_schema.ts b/x-pack/plugins/lists/common/schemas/request/create_list_schema.ts index 68df80b2a42dd..fcf4465f88c8d 100644 --- a/x-pack/plugins/lists/common/schemas/request/create_list_schema.ts +++ b/x-pack/plugins/lists/common/schemas/request/create_list_schema.ts @@ -6,7 +6,7 @@ import * as t from 'io-ts'; -import { description, id, meta, name, type } from '../common/schemas'; +import { description, deserializer, id, meta, name, serializer, type } from '../common/schemas'; import { Identity, RequiredKeepUndefined } from '../../types'; export const createListSchema = t.intersection([ @@ -17,7 +17,7 @@ export const createListSchema = t.intersection([ type, }) ), - t.exact(t.partial({ id, meta })), + t.exact(t.partial({ deserializer, id, meta, serializer })), ]); export type CreateListSchemaPartial = Identity>; diff --git a/x-pack/plugins/lists/common/schemas/request/import_list_item_query_schema.mock.ts b/x-pack/plugins/lists/common/schemas/request/import_list_item_query_schema.mock.ts index 6713083e6a49b..ebec124a0861c 100644 --- a/x-pack/plugins/lists/common/schemas/request/import_list_item_query_schema.mock.ts +++ b/x-pack/plugins/lists/common/schemas/request/import_list_item_query_schema.mock.ts @@ -9,6 +9,8 @@ import { LIST_ID, TYPE } from '../../constants.mock'; import { ImportListItemQuerySchema } from './import_list_item_query_schema'; export const getImportListItemQuerySchemaMock = (): ImportListItemQuerySchema => ({ + deserializer: undefined, list_id: LIST_ID, + serializer: undefined, type: TYPE, }); diff --git a/x-pack/plugins/lists/common/schemas/request/import_list_item_query_schema.test.ts b/x-pack/plugins/lists/common/schemas/request/import_list_item_query_schema.test.ts index ac007a704b92d..9d03229b4d1d9 100644 --- a/x-pack/plugins/lists/common/schemas/request/import_list_item_query_schema.test.ts +++ b/x-pack/plugins/lists/common/schemas/request/import_list_item_query_schema.test.ts @@ -57,6 +57,26 @@ describe('import_list_item_schema', () => { expect(message.schema).toEqual(payload); }); + test('it should accept an undefined for "serializer"', () => { + const payload = getImportListItemQuerySchemaMock(); + delete payload.serializer; + const decoded = importListItemQuerySchema.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + test('it should accept an undefined for "deserializer"', () => { + const payload = getImportListItemQuerySchemaMock(); + delete payload.deserializer; + const decoded = importListItemQuerySchema.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + test('it should not allow an extra key to be sent in', () => { const payload: ImportListItemQuerySchema & { extraKey?: string; diff --git a/x-pack/plugins/lists/common/schemas/request/import_list_item_query_schema.ts b/x-pack/plugins/lists/common/schemas/request/import_list_item_query_schema.ts index b37de61d0c2c3..2c671466702e0 100644 --- a/x-pack/plugins/lists/common/schemas/request/import_list_item_query_schema.ts +++ b/x-pack/plugins/lists/common/schemas/request/import_list_item_query_schema.ts @@ -8,10 +8,12 @@ import * as t from 'io-ts'; -import { list_id, type } from '../common/schemas'; +import { deserializer, list_id, serializer, type } from '../common/schemas'; import { Identity } from '../../types'; -export const importListItemQuerySchema = t.exact(t.partial({ list_id, type })); +export const importListItemQuerySchema = t.exact( + t.partial({ deserializer, list_id, serializer, type }) +); export type ImportListItemQuerySchemaPartial = Identity>; diff --git a/x-pack/plugins/lists/common/schemas/response/found_list_item_schema.mock.ts b/x-pack/plugins/lists/common/schemas/response/found_list_item_schema.mock.ts index e96188c619d78..204586a418527 100644 --- a/x-pack/plugins/lists/common/schemas/response/found_list_item_schema.mock.ts +++ b/x-pack/plugins/lists/common/schemas/response/found_list_item_schema.mock.ts @@ -8,9 +8,9 @@ import { FoundListItemSchema } from './found_list_item_schema'; import { getListItemResponseMock } from './list_item_schema.mock'; export const getFoundListItemSchemaMock = (): FoundListItemSchema => ({ - cursor: '123', + cursor: 'WzI1LFsiNmE3NmI2OWQtODBkZi00YWIyLThjM2UtODVmNDY2YjA2YTBlIl1d', data: [getListItemResponseMock()], page: 1, - per_page: 1, + per_page: 25, total: 1, }); diff --git a/x-pack/plugins/lists/common/schemas/response/list_item_schema.mock.ts b/x-pack/plugins/lists/common/schemas/response/list_item_schema.mock.ts index 309aeaa477c66..16e8057974917 100644 --- a/x-pack/plugins/lists/common/schemas/response/list_item_schema.mock.ts +++ b/x-pack/plugins/lists/common/schemas/response/list_item_schema.mock.ts @@ -19,9 +19,11 @@ import { export const getListItemResponseMock = (): ListItemSchema => ({ created_at: DATE_NOW, created_by: USER, + deserializer: undefined, id: LIST_ITEM_ID, list_id: LIST_ID, meta: META, + serializer: undefined, tie_breaker_id: TIE_BREAKER, type: TYPE, updated_at: DATE_NOW, diff --git a/x-pack/plugins/lists/common/schemas/response/list_item_schema.test.ts b/x-pack/plugins/lists/common/schemas/response/list_item_schema.test.ts index fbffd1d3ef245..8b73506d13750 100644 --- a/x-pack/plugins/lists/common/schemas/response/list_item_schema.test.ts +++ b/x-pack/plugins/lists/common/schemas/response/list_item_schema.test.ts @@ -58,6 +58,28 @@ describe('list_item_schema', () => { expect(message.schema).toEqual(payload); }); + test('it should accept an undefined for "serializer"', () => { + const payload = getListItemResponseMock(); + delete payload.serializer; + const decoded = listItemSchema.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + test('it should accept an undefined for "deserializer"', () => { + const payload = getListItemResponseMock(); + delete payload.deserializer; + const decoded = listItemSchema.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + test('it should NOT accept an undefined for "created_at"', () => { const payload = getListItemResponseMock(); delete payload.created_at; diff --git a/x-pack/plugins/lists/common/schemas/response/list_item_schema.ts b/x-pack/plugins/lists/common/schemas/response/list_item_schema.ts index 6c2f2ed9a7095..c2104aaf18b53 100644 --- a/x-pack/plugins/lists/common/schemas/response/list_item_schema.ts +++ b/x-pack/plugins/lists/common/schemas/response/list_item_schema.ts @@ -11,9 +11,11 @@ import * as t from 'io-ts'; import { created_at, created_by, + deserializerOrUndefined, id, list_id, metaOrUndefined, + serializerOrUndefined, tie_breaker_id, type, updated_at, @@ -25,9 +27,11 @@ export const listItemSchema = t.exact( t.type({ created_at, created_by, + deserializer: deserializerOrUndefined, id, list_id, meta: metaOrUndefined, + serializer: serializerOrUndefined, tie_breaker_id, type, updated_at, diff --git a/x-pack/plugins/lists/common/schemas/response/list_schema.mock.ts b/x-pack/plugins/lists/common/schemas/response/list_schema.mock.ts index 5016252bc564a..c165c4ed8e745 100644 --- a/x-pack/plugins/lists/common/schemas/response/list_schema.mock.ts +++ b/x-pack/plugins/lists/common/schemas/response/list_schema.mock.ts @@ -20,9 +20,11 @@ export const getListResponseMock = (): ListSchema => ({ created_at: DATE_NOW, created_by: USER, description: DESCRIPTION, + deserializer: undefined, id: LIST_ID, meta: META, name: NAME, + serializer: undefined, tie_breaker_id: TIE_BREAKER, type: TYPE, updated_at: DATE_NOW, diff --git a/x-pack/plugins/lists/common/schemas/response/list_schema.test.ts b/x-pack/plugins/lists/common/schemas/response/list_schema.test.ts index a37207271c06e..e7ae9b45a5e15 100644 --- a/x-pack/plugins/lists/common/schemas/response/list_schema.test.ts +++ b/x-pack/plugins/lists/common/schemas/response/list_schema.test.ts @@ -45,6 +45,28 @@ describe('list_schema', () => { expect(message.schema).toEqual(payload); }); + test('it should accept an undefined for "serializer"', () => { + const payload = getListResponseMock(); + delete payload.serializer; + const decoded = listSchema.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + test('it should accept an undefined for "deserializer"', () => { + const payload = getListResponseMock(); + delete payload.deserializer; + const decoded = listSchema.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + test('it should NOT accept an undefined for "created_at"', () => { const payload = getListResponseMock(); delete payload.created_at; diff --git a/x-pack/plugins/lists/common/schemas/response/list_schema.ts b/x-pack/plugins/lists/common/schemas/response/list_schema.ts index 4e664685db9c7..1950831bee694 100644 --- a/x-pack/plugins/lists/common/schemas/response/list_schema.ts +++ b/x-pack/plugins/lists/common/schemas/response/list_schema.ts @@ -12,9 +12,11 @@ import { created_at, created_by, description, + deserializerOrUndefined, id, metaOrUndefined, name, + serializerOrUndefined, tie_breaker_id, type, updated_at, @@ -26,9 +28,11 @@ export const listSchema = t.exact( created_at, created_by, description, + deserializer: deserializerOrUndefined, id, meta: metaOrUndefined, name, + serializer: serializerOrUndefined, tie_breaker_id, type, updated_at, diff --git a/x-pack/plugins/lists/common/schemas/types/default_entries_array.test.ts b/x-pack/plugins/lists/common/schemas/types/default_entries_array.test.ts index e7910be6bf4b5..21115690c0a5f 100644 --- a/x-pack/plugins/lists/common/schemas/types/default_entries_array.test.ts +++ b/x-pack/plugins/lists/common/schemas/types/default_entries_array.test.ts @@ -18,7 +18,7 @@ import { getEntriesArrayMock, getEntryMatchMock, getEntryNestedMock } from './en // different entry types, it returns 5 of these. To make more readable, // extracted here. const returnedSchemaError = - '"Array<({| field: string, operator: "excluded" | "included", type: "match", value: string |} | {| field: string, operator: "excluded" | "included", type: "match_any", value: DefaultStringArray |} | {| field: string, list: {| id: string, type: "ip" | "keyword" |}, operator: "excluded" | "included", type: "list" |} | {| field: string, operator: "excluded" | "included", type: "exists" |} | {| entries: Array<{| field: string, operator: "excluded" | "included", type: "match", value: string |}>, field: string, type: "nested" |})>"'; + '"Array<({| field: string, operator: "excluded" | "included", type: "match", value: string |} | {| field: string, operator: "excluded" | "included", type: "match_any", value: DefaultStringArray |} | {| field: string, list: {| id: string, type: "binary" | "boolean" | "byte" | "date" | "date_nanos" | "date_range" | "double" | "double_range" | "float" | "float_range" | "geo_point" | "geo_shape" | "half_float" | "integer" | "integer_range" | "ip" | "ip_range" | "keyword" | "long" | "long_range" | "shape" | "short" | "text" |}, operator: "excluded" | "included", type: "list" |} | {| field: string, operator: "excluded" | "included", type: "exists" |} | {| entries: Array<{| field: string, operator: "excluded" | "included", type: "match", value: string |}>, field: string, type: "nested" |})>"'; describe('default_entries_array', () => { test('it should validate an empty array', () => { diff --git a/x-pack/plugins/lists/server/routes/create_list_item_route.ts b/x-pack/plugins/lists/server/routes/create_list_item_route.ts index 2e1b7fa07221f..8ac5db3c7fd1c 100644 --- a/x-pack/plugins/lists/server/routes/create_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/create_list_item_route.ts @@ -36,26 +36,27 @@ export const createListItemRoute = (router: IRouter): void => { statusCode: 404, }); } else { - const listItem = await lists.getListItemByValue({ listId, type: list.type, value }); - if (listItem.length !== 0) { - return siemResponse.error({ - body: `list_id: "${listId}" already contains the given value: ${value}`, - statusCode: 409, - }); - } else { - const createdListItem = await lists.createListItem({ - id, - listId, - meta, - type: list.type, - value, - }); + const createdListItem = await lists.createListItem({ + deserializer: list.deserializer, + id, + listId, + meta, + serializer: list.serializer, + type: list.type, + value, + }); + if (createdListItem != null) { const [validated, errors] = validate(createdListItem, listItemSchema); if (errors != null) { return siemResponse.error({ body: errors, statusCode: 500 }); } else { return response.ok({ body: validated ?? {} }); } + } else { + return siemResponse.error({ + body: 'list item invalid', + statusCode: 400, + }); } } } catch (err) { diff --git a/x-pack/plugins/lists/server/routes/create_list_route.ts b/x-pack/plugins/lists/server/routes/create_list_route.ts index 9872bbfa09e23..eee7517523b0f 100644 --- a/x-pack/plugins/lists/server/routes/create_list_route.ts +++ b/x-pack/plugins/lists/server/routes/create_list_route.ts @@ -27,7 +27,7 @@ export const createListRoute = (router: IRouter): void => { async (context, request, response) => { const siemResponse = buildSiemResponse(response); try { - const { name, description, id, type, meta } = request.body; + const { name, description, deserializer, id, serializer, type, meta } = request.body; const lists = getListClient(context); const listExists = await lists.getListIndexExists(); if (!listExists) { @@ -45,7 +45,15 @@ export const createListRoute = (router: IRouter): void => { }); } } - const list = await lists.createList({ description, id, meta, name, type }); + const list = await lists.createList({ + description, + deserializer, + id, + meta, + name, + serializer, + type, + }); const [validated, errors] = validate(list, listSchema); if (errors != null) { return siemResponse.error({ body: errors, statusCode: 500 }); diff --git a/x-pack/plugins/lists/server/routes/delete_list_item_route.ts b/x-pack/plugins/lists/server/routes/delete_list_item_route.ts index 510be764cefba..bb278ba436725 100644 --- a/x-pack/plugins/lists/server/routes/delete_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/delete_list_item_route.ts @@ -52,7 +52,11 @@ export const deleteListItemRoute = (router: IRouter): void => { statusCode: 404, }); } else { - const deleted = await lists.deleteListItemByValue({ listId, type: list.type, value }); + const deleted = await lists.deleteListItemByValue({ + listId, + type: list.type, + value, + }); if (deleted == null || deleted.length === 0) { return siemResponse.error({ body: `list_id: "${listId}" with ${value} was not found`, diff --git a/x-pack/plugins/lists/server/routes/import_list_item_route.ts b/x-pack/plugins/lists/server/routes/import_list_item_route.ts index 67f345c2c6c1d..d75199140ea8e 100644 --- a/x-pack/plugins/lists/server/routes/import_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/import_list_item_route.ts @@ -48,7 +48,7 @@ export const importListItemRoute = (router: IRouter): void => { async (context, request, response) => { const siemResponse = buildSiemResponse(response); try { - const { list_id: listId, type } = request.query; + const { deserializer, list_id: listId, serializer, type } = request.query; const lists = getListClient(context); if (listId != null) { const list = await lists.getList({ id: listId }); @@ -59,8 +59,10 @@ export const importListItemRoute = (router: IRouter): void => { }); } await lists.importListItemsToStream({ + deserializer: list.deserializer, listId, meta: undefined, + serializer: list.serializer, stream: request.body.file, type: list.type, }); @@ -76,14 +78,18 @@ export const importListItemRoute = (router: IRouter): void => { // TODO: Should we prevent the same file from being uploaded multiple times? const list = await lists.createListIfItDoesNotExist({ description: `File uploaded from file system of ${filename}`, + deserializer, id: filename, meta: undefined, name: filename, + serializer, type, }); await lists.importListItemsToStream({ + deserializer: list.deserializer, listId: list.id, meta: undefined, + serializer: list.serializer, stream: request.body.file, type: list.type, }); diff --git a/x-pack/plugins/lists/server/routes/patch_list_item_route.ts b/x-pack/plugins/lists/server/routes/patch_list_item_route.ts index f706559dffdbd..e21a54c09a873 100644 --- a/x-pack/plugins/lists/server/routes/patch_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/patch_list_item_route.ts @@ -29,6 +29,7 @@ export const patchListItemRoute = (router: IRouter): void => { try { const { value, id, meta } = request.body; const lists = getListClient(context); + // TODO: This looks like just a regular update, implement a patchListItem API and add plumbing for that. const listItem = await lists.updateListItem({ id, meta, diff --git a/x-pack/plugins/lists/server/routes/patch_list_route.ts b/x-pack/plugins/lists/server/routes/patch_list_route.ts index 3a0d8714a14cd..9443ac2ed2eea 100644 --- a/x-pack/plugins/lists/server/routes/patch_list_route.ts +++ b/x-pack/plugins/lists/server/routes/patch_list_route.ts @@ -29,6 +29,7 @@ export const patchListRoute = (router: IRouter): void => { try { const { name, description, id, meta } = request.body; const lists = getListClient(context); + // TODO: This looks like just a regular update, implement a patchListItem API and add plumbing for that. const list = await lists.updateList({ description, id, meta, name }); if (list == null) { return siemResponse.error({ diff --git a/x-pack/plugins/lists/server/scripts/download_load_lists_example.sh b/x-pack/plugins/lists/server/scripts/download_load_lists_example.sh new file mode 100755 index 0000000000000..831f9da1e886e --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/download_load_lists_example.sh @@ -0,0 +1,76 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +# Simple example of a set of commands to load online lists into elastic as well as how to query them + +set -e +./check_env_variables.sh + +# What to cap lists at during downloads +export TOTAL_LINES_PER_FILE=100 + +# Download a set of lists to the $TMPDIR folder +echo "Downloading lists..." +cd ${TMPDIR} +echo "Downloading mdl.txt" +curl -s -L https://panwdbl.appspot.com/lists/mdl.txt | head -${TOTAL_LINES_PER_FILE} > mdl.txt +echo "Downloading cybercrime.txt" +curl -s -L https://cybercrime-tracker.net/all.php | head -${TOTAL_LINES_PER_FILE} > cybercrime.txt +echo "Downloading online-valid.csv" +curl -s -L https://data.phishtank.com/data/online-valid.csv | head -${TOTAL_LINES_PER_FILE} > online-valid.csv +echo "Downloading dynamic_dns.txt" +curl -s -L http://dns-bh.sagadc.org/dynamic_dns.txt | head -${TOTAL_LINES_PER_FILE} > dynamic_dns.txt +echo "Downloading ipblocklist.csv" +curl -s -L https://feodotracker.abuse.ch/downloads/ipblocklist.csv | head -${TOTAL_LINES_PER_FILE} > ipblocklist.csv +echo "Done downloading lists" +cd - > /dev/null + +# Import the lists in various formats from $TMPDIR folder +echo "Importing mdl.txt as a ip_range format" +./import_list_items_by_filename.sh ip_range ${TMPDIR}/mdl.txt + +echo "Importing mdl.txt as a regular ip format using a custom serializer into the list ip_custom_format_list" +./post_list.sh ./lists/new/lists/ip_custom_format.json +./import_list_items.sh ip_custom_format_list ${TMPDIR}/mdl.txt + +echo "Importing mdl.txt as a keyword format using a custom serializer into the list keyword_custom_format_list" +./post_list.sh ./lists/new/lists/keyword_custom_format.json +./import_list_items.sh keyword_custom_format_list ${TMPDIR}/mdl.txt + +echo "Calling /_find to iterate ip_range" +./find_list_items.sh mdl.txt + +echo "Calling /_find to iterate ip_custom_format_list" +./find_list_items.sh ip_custom_format_list + +echo "Calling /_find to iterate keyword_custom_format_list" +./find_list_items.sh keyword_custom_format_list + +echo "Exporting to the terminal each format" +./export_list_items.sh mdl.txt +./export_list_items.sh ip_custom_format_list +./export_list_items.sh keyword_custom_format_list + +echo "Querying against an IP that might or might not be in each list" +./get_list_item_by_value.sh mdl.txt 46.254.17.30 +./get_list_item_by_value.sh ip_custom_format_list 46.254.17.0/16 +./get_list_item_by_value.sh keyword_custom_format_list 46.254.17.30 + +echo "Importing cybercrime.txt as a ip_custom_format_list format" +./import_list_items.sh ip_custom_format_list ${TMPDIR}/cybercrime.txt + +echo "Importing cybercrime.txt as a keyword_custom_format_list format" +./import_list_items.sh keyword_custom_format_list ${TMPDIR}/cybercrime.txt + +echo "Importing cybercrime.txt as a ip_custom_format_list format" +./import_list_items.sh ip_custom_format_list ${TMPDIR}/ipblocklist.csv + +echo "Importing cybercrime.txt as a keyword_custom_format_list format" +./import_list_items.sh keyword_custom_format_list ${TMPDIR}/ipblocklist.csv + + diff --git a/x-pack/plugins/lists/server/scripts/find_list_items.sh b/x-pack/plugins/lists/server/scripts/find_list_items.sh index c4a610e313fa8..9c8bfd2d5a490 100755 --- a/x-pack/plugins/lists/server/scripts/find_list_items.sh +++ b/x-pack/plugins/lists/server/scripts/find_list_items.sh @@ -9,11 +9,11 @@ set -e ./check_env_variables.sh -PAGE=${1-1} -PER_PAGE=${2-20} -LIST_ID=${3-list-ip} +LIST_ID=${1-list-ip} +PAGE=${2-1} +PER_PAGE=${3-20} -# Example: ./find_list_items.sh 1 20 list-ip +# Example: ./find_list_items.sh list-ip 1 20 curl -s -k \ -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ -X GET "${KIBANA_URL}${SPACE_URL}/api/lists/items/_find?list_id=${LIST_ID}&page=${PAGE}&per_page=${PER_PAGE}" | jq . diff --git a/x-pack/plugins/lists/server/scripts/find_list_items_with_cursor.sh b/x-pack/plugins/lists/server/scripts/find_list_items_with_cursor.sh index 3fd5178b2d9b1..8924012cf62cf 100755 --- a/x-pack/plugins/lists/server/scripts/find_list_items_with_cursor.sh +++ b/x-pack/plugins/lists/server/scripts/find_list_items_with_cursor.sh @@ -9,15 +9,15 @@ set -e ./check_env_variables.sh -PAGE=${1-1} -PER_PAGE=${2-20} -LIST_ID=${3-list-ip} +LIST_ID=${1-list-ip} +PAGE=${2-1} +PER_PAGE=${3-20} CURSOR=${4-invalid} # Example: # ./find_list_items.sh 1 20 | jq .cursor # Copy the cursor into the argument below like so -# ./find_list_items_with_cursor.sh 1 10 list-ip eyJwYWdlX2luZGV4IjoyMCwic2VhcmNoX2FmdGVyIjpbIjAyZDZlNGY3LWUzMzAtNGZkYi1iNTY0LTEzZjNiOTk1MjRiYSJdfQ== +# ./find_list_items_with_cursor.sh list-ip 1 10 eyJwYWdlX2luZGV4IjoyMCwic2VhcmNoX2FmdGVyIjpbIjAyZDZlNGY3LWUzMzAtNGZkYi1iNTY0LTEzZjNiOTk1MjRiYSJdfQ== curl -s -k \ -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ -X GET "${KIBANA_URL}${SPACE_URL}/api/lists/items/_find?list_id=${LIST_ID}&page=${PAGE}&per_page=${PER_PAGE}&cursor=${CURSOR}" | jq . diff --git a/x-pack/plugins/lists/server/scripts/find_list_items_with_sort.sh b/x-pack/plugins/lists/server/scripts/find_list_items_with_sort.sh index dcea698be231d..37d80c3dd3f28 100755 --- a/x-pack/plugins/lists/server/scripts/find_list_items_with_sort.sh +++ b/x-pack/plugins/lists/server/scripts/find_list_items_with_sort.sh @@ -9,13 +9,13 @@ set -e ./check_env_variables.sh -PAGE=${1-1} -PER_PAGE=${2-20} -SORT_FIELD=${3-value} +LIST_ID=${1-list-ip} +PAGE=${2-1} +PER_PAGE=${3-20} +SORT_FIELD=${4-value} SORT_ORDER=${4-asc} -LIST_ID=${5-list-ip} -# Example: ./find_list_items_with_sort.sh 1 20 value asc list-ip +# Example: ./find_list_items_with_sort.sh list-ip 1 20 value asc curl -s -k \ -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ -X GET "${KIBANA_URL}${SPACE_URL}/api/lists/items/_find?list_id=${LIST_ID}&page=${PAGE}&per_page=${PER_PAGE}&sort_field=${SORT_FIELD}&sort_order=${SORT_ORDER}" | jq . diff --git a/x-pack/plugins/lists/server/scripts/find_list_items_with_sort_cursor.sh b/x-pack/plugins/lists/server/scripts/find_list_items_with_sort_cursor.sh index 07b67a9bd1c5f..27d8deb2fc95a 100755 --- a/x-pack/plugins/lists/server/scripts/find_list_items_with_sort_cursor.sh +++ b/x-pack/plugins/lists/server/scripts/find_list_items_with_sort_cursor.sh @@ -9,14 +9,14 @@ set -e ./check_env_variables.sh -PAGE=${1-1} -PER_PAGE=${2-20} -SORT_FIELD=${3-value} -SORT_ORDER=${4-asc} -LIST_ID=${5-list-ip} +LIST_ID=${1-list-ip} +PAGE=${2-1} +PER_PAGE=${3-20} +SORT_FIELD=${4-value} +SORT_ORDER=${5-asc} CURSOR=${6-invalid} -# Example: ./find_list_items_with_sort_cursor.sh 1 20 value asc list-ip +# Example: ./find_list_items_with_sort_cursor.sh list-ip 1 20 value asc curl -s -k \ -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ -X GET "${KIBANA_URL}${SPACE_URL}/api/lists/items/_find?list_id=${LIST_ID}&page=${PAGE}&per_page=${PER_PAGE}&sort_field=${SORT_FIELD}&sort_order=${SORT_ORDER}&cursor=${CURSOR}" | jq . diff --git a/x-pack/plugins/lists/server/scripts/lists/files/binary.txt b/x-pack/plugins/lists/server/scripts/lists/files/binary.txt new file mode 100644 index 0000000000000..bb0fd8fd61f4d --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/files/binary.txt @@ -0,0 +1,3 @@ +U29tZSBiaW5hcnkgYmxvYg== +SGVsbG8gc29tZSB0ZXh0IGZvciB5b3U= +TW9yZSB0ZXh0IGZvciB5b3U= diff --git a/x-pack/plugins/lists/server/scripts/lists/files/boolean.txt b/x-pack/plugins/lists/server/scripts/lists/files/boolean.txt new file mode 100644 index 0000000000000..aee76b20147aa --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/files/boolean.txt @@ -0,0 +1,14 @@ +true +false +true +false +true +true +true +true +true +false +false +false +false +false diff --git a/x-pack/plugins/lists/server/scripts/lists/files/byte.txt b/x-pack/plugins/lists/server/scripts/lists/files/byte.txt new file mode 100644 index 0000000000000..d34af7b25a89f --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/files/byte.txt @@ -0,0 +1,7 @@ +5 +10 +3 +20 +200 +100 +30 diff --git a/x-pack/plugins/lists/server/scripts/lists/files/date.txt b/x-pack/plugins/lists/server/scripts/lists/files/date.txt new file mode 100644 index 0000000000000..e051a24aa1858 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/files/date.txt @@ -0,0 +1,7 @@ +2020-06-25T17:57:01.978Z +2020-06-25T17:57:01.978Z +2020-06-25T17:57:01.978Z +2020-07-25T17:57:01.978Z +2020-08-25T17:57:01.978Z +2020-09-25T17:57:01.978Z + diff --git a/x-pack/plugins/lists/server/scripts/lists/files/date_nanos.txt b/x-pack/plugins/lists/server/scripts/lists/files/date_nanos.txt new file mode 100644 index 0000000000000..d9c0bbf8f2bd1 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/files/date_nanos.txt @@ -0,0 +1,7 @@ +2020-06-25T17:57:01.123456789Z +2020-06-25T17:57:01.123456789Z +2020-06-25T17:57:01.123456789Z +2020-07-25T17:57:01.123456789Z +2020-08-25T17:57:01.123456789Z +2020-09-25T17:57:01.123456789Z + diff --git a/x-pack/plugins/lists/server/scripts/lists/files/date_range.txt b/x-pack/plugins/lists/server/scripts/lists/files/date_range.txt new file mode 100644 index 0000000000000..fac1cdddbb128 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/files/date_range.txt @@ -0,0 +1,5 @@ +2020-06-20T17:57:01.978Z,2020-06-21T17:57:01.978Z +2020-06-21T17:57:01.978Z,2020-06-22T17:57:01.978Z +2020-06-22T17:57:01.978Z,2020-06-23T17:57:01.978Z +2020-06-24T17:57:01.978Z,2020-06-25T17:57:01.978Z +2020-06-26T17:57:01.978Z,2020-06-27T17:57:01.978Z diff --git a/x-pack/plugins/lists/server/scripts/lists/files/date_range_custom_format.txt b/x-pack/plugins/lists/server/scripts/lists/files/date_range_custom_format.txt new file mode 100644 index 0000000000000..94aa9db1a5851 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/files/date_range_custom_format.txt @@ -0,0 +1,5 @@ +2020-06-20T17:57:01.978Z/2020-06-21T17:57:01.978Z +2020-06-21T17:57:01.978Z/2020-06-22T17:57:01.978Z +2020-06-22T17:57:01.978Z/2020-06-23T17:57:01.978Z +2020-06-24T17:57:01.978Z/2020-06-25T17:57:01.978Z +2020-06-26T17:57:01.978Z/2020-06-27T17:57:01.978Z diff --git a/x-pack/plugins/lists/server/scripts/lists/files/double.txt b/x-pack/plugins/lists/server/scripts/lists/files/double.txt new file mode 100644 index 0000000000000..77418e371d14d --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/files/double.txt @@ -0,0 +1,7 @@ +5.4 +10.6 +3.8 +20.21 +200.22 +100.13 +30.5 diff --git a/x-pack/plugins/lists/server/scripts/lists/files/double_range.txt b/x-pack/plugins/lists/server/scripts/lists/files/double_range.txt new file mode 100644 index 0000000000000..80242a3144363 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/files/double_range.txt @@ -0,0 +1,7 @@ +5.4,6.6 +10.6,13 +3.8,4.9 +20.21,33.33 +200.22,300.55 +100.13,200.55 +30.5,50.5 diff --git a/x-pack/plugins/lists/server/scripts/lists/files/float.txt b/x-pack/plugins/lists/server/scripts/lists/files/float.txt new file mode 100644 index 0000000000000..77418e371d14d --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/files/float.txt @@ -0,0 +1,7 @@ +5.4 +10.6 +3.8 +20.21 +200.22 +100.13 +30.5 diff --git a/x-pack/plugins/lists/server/scripts/lists/files/float_range.txt b/x-pack/plugins/lists/server/scripts/lists/files/float_range.txt new file mode 100644 index 0000000000000..80242a3144363 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/files/float_range.txt @@ -0,0 +1,7 @@ +5.4,6.6 +10.6,13 +3.8,4.9 +20.21,33.33 +200.22,300.55 +100.13,200.55 +30.5,50.5 diff --git a/x-pack/plugins/lists/server/scripts/lists/files/geo_hash.txt b/x-pack/plugins/lists/server/scripts/lists/files/geo_hash.txt new file mode 100644 index 0000000000000..e4381962ab76d --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/files/geo_hash.txt @@ -0,0 +1,5 @@ +gzuzvrzx +gbsuv +gbsuw +gbsvh +gbsvn diff --git a/x-pack/plugins/lists/server/scripts/lists/files/geo_point.txt b/x-pack/plugins/lists/server/scripts/lists/files/geo_point.txt new file mode 100644 index 0000000000000..803dadb55ed1e --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/files/geo_point.txt @@ -0,0 +1,4 @@ +34.50,131.60 +-23.30,-67.62 +14.50,-90.88 +38.57,34.52 diff --git a/x-pack/plugins/lists/server/scripts/lists/files/geo_point_wkt.txt b/x-pack/plugins/lists/server/scripts/lists/files/geo_point_wkt.txt new file mode 100644 index 0000000000000..0ccddceb6a33f --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/files/geo_point_wkt.txt @@ -0,0 +1,4 @@ +POINT (131.60 34.50) +POINT (-67.62 -23.30) +POINT (-90.88 14.50) +POINT (34.52 38.57) diff --git a/x-pack/plugins/lists/server/scripts/lists/files/half_float.txt b/x-pack/plugins/lists/server/scripts/lists/files/half_float.txt new file mode 100644 index 0000000000000..77418e371d14d --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/files/half_float.txt @@ -0,0 +1,7 @@ +5.4 +10.6 +3.8 +20.21 +200.22 +100.13 +30.5 diff --git a/x-pack/plugins/lists/server/scripts/lists/files/integer.txt b/x-pack/plugins/lists/server/scripts/lists/files/integer.txt new file mode 100644 index 0000000000000..d34af7b25a89f --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/files/integer.txt @@ -0,0 +1,7 @@ +5 +10 +3 +20 +200 +100 +30 diff --git a/x-pack/plugins/lists/server/scripts/lists/files/integer_range.txt b/x-pack/plugins/lists/server/scripts/lists/files/integer_range.txt new file mode 100644 index 0000000000000..1bc59898bbcdd --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/files/integer_range.txt @@ -0,0 +1,7 @@ +5,8 +10,12 +3,8 +20,55 +200,300 +100,400 +30,50 diff --git a/x-pack/plugins/lists/server/scripts/lists/files/ip_range.txt b/x-pack/plugins/lists/server/scripts/lists/files/ip_range.txt new file mode 100644 index 0000000000000..df9109d36e4f9 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/files/ip_range.txt @@ -0,0 +1,6 @@ +192.168.0.1-192.168.0.3 +192.168.0.5-192.168.0.8 +92.168.0.10-192.168.0.20 +192.168.1.0-192.168.1.3 +192.168.1.5-192.168.1.8 +192.168.0.10-192.168.1.20 diff --git a/x-pack/plugins/lists/server/scripts/lists/files/ip_range_cidr.txt b/x-pack/plugins/lists/server/scripts/lists/files/ip_range_cidr.txt new file mode 100644 index 0000000000000..146087c1c432f --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/files/ip_range_cidr.txt @@ -0,0 +1,3 @@ +192.168.0.1/16 +192.168.1.0/16 +192.168.2.0/16 diff --git a/x-pack/plugins/lists/server/scripts/lists/files/ip_range_mixed.txt b/x-pack/plugins/lists/server/scripts/lists/files/ip_range_mixed.txt new file mode 100644 index 0000000000000..5f6e89f9d438b --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/files/ip_range_mixed.txt @@ -0,0 +1,5 @@ +127.0.0.1 +192.168.0.1-192.168.0.3 +192.168.0.1/16 +192.168.1.0/16 +192.168.2.0/16 diff --git a/x-pack/plugins/lists/server/scripts/lists/files/long.txt b/x-pack/plugins/lists/server/scripts/lists/files/long.txt new file mode 100644 index 0000000000000..d34af7b25a89f --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/files/long.txt @@ -0,0 +1,7 @@ +5 +10 +3 +20 +200 +100 +30 diff --git a/x-pack/plugins/lists/server/scripts/lists/files/long_range.txt b/x-pack/plugins/lists/server/scripts/lists/files/long_range.txt new file mode 100644 index 0000000000000..1bc59898bbcdd --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/files/long_range.txt @@ -0,0 +1,7 @@ +5,8 +10,12 +3,8 +20,55 +200,300 +100,400 +30,50 diff --git a/x-pack/plugins/lists/server/scripts/lists/files/short.txt b/x-pack/plugins/lists/server/scripts/lists/files/short.txt new file mode 100644 index 0000000000000..d34af7b25a89f --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/files/short.txt @@ -0,0 +1,7 @@ +5 +10 +3 +20 +200 +100 +30 diff --git a/x-pack/plugins/lists/server/scripts/lists/files/text.txt b/x-pack/plugins/lists/server/scripts/lists/files/text.txt new file mode 100644 index 0000000000000..aee32e3a4bd92 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/files/text.txt @@ -0,0 +1,2 @@ +kibana +rock01 diff --git a/x-pack/plugins/lists/server/scripts/lists/new/items/binary_item.json b/x-pack/plugins/lists/server/scripts/lists/new/items/binary_item.json new file mode 100644 index 0000000000000..9e4e35d90f082 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/items/binary_item.json @@ -0,0 +1,5 @@ +{ + "id": "binary_item", + "list_id": "binary_list", + "value": "U29tZSBiaW5hcnkgYmxvYg==" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/items/boolean_item.json b/x-pack/plugins/lists/server/scripts/lists/new/items/boolean_item.json new file mode 100644 index 0000000000000..6f6c7450fb2a1 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/items/boolean_item.json @@ -0,0 +1,5 @@ +{ + "id": "boolean_item", + "list_id": "boolean_list", + "value": "true" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/items/byte_item.json b/x-pack/plugins/lists/server/scripts/lists/new/items/byte_item.json new file mode 100644 index 0000000000000..36d48ca588451 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/items/byte_item.json @@ -0,0 +1,5 @@ +{ + "id": "byte_item", + "list_id": "byte_list", + "value": "10" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/items/date_item.json b/x-pack/plugins/lists/server/scripts/lists/new/items/date_item.json new file mode 100644 index 0000000000000..410c27e07685a --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/items/date_item.json @@ -0,0 +1,5 @@ +{ + "id": "date_item", + "list_id": "date_list", + "value": "2015-01-01" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/items/date_nanos_item.json b/x-pack/plugins/lists/server/scripts/lists/new/items/date_nanos_item.json new file mode 100644 index 0000000000000..e3aafdcb2c9ee --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/items/date_nanos_item.json @@ -0,0 +1,5 @@ +{ + "id": "nanos_item", + "list_id": "date_nanos_list", + "value": "2015-01-01T12:10:30.123456789Z" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/items/date_range_item.json b/x-pack/plugins/lists/server/scripts/lists/new/items/date_range_item.json new file mode 100644 index 0000000000000..7ca0498fcc0a8 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/items/date_range_item.json @@ -0,0 +1,5 @@ +{ + "id": "date_range_item", + "list_id": "date_range_list", + "value": "2015-10-31 12:00:00/2015-11-01" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/items/double_item.json b/x-pack/plugins/lists/server/scripts/lists/new/items/double_item.json new file mode 100644 index 0000000000000..9a050d23382d5 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/items/double_item.json @@ -0,0 +1,5 @@ +{ + "id": "double_item", + "list_id": "double_list", + "value": "23.3" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/items/double_range_item.json b/x-pack/plugins/lists/server/scripts/lists/new/items/double_range_item.json new file mode 100644 index 0000000000000..a5077264b96ee --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/items/double_range_item.json @@ -0,0 +1,5 @@ +{ + "id": "double_range_item", + "list_id": "double_range_list", + "value": "10-100" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/items/float_item.json b/x-pack/plugins/lists/server/scripts/lists/new/items/float_item.json new file mode 100644 index 0000000000000..e249f6a35ae7f --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/items/float_item.json @@ -0,0 +1,5 @@ +{ + "id": "float_item", + "list_id": "float_list", + "value": "23.2" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/items/float_range_item.json b/x-pack/plugins/lists/server/scripts/lists/new/items/float_range_item.json new file mode 100644 index 0000000000000..538e1dcd01f3a --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/items/float_range_item.json @@ -0,0 +1,5 @@ +{ + "id": "float_range_item", + "list_id": "float_range_list", + "value": "10-12.3" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/items/geo_point_item.json b/x-pack/plugins/lists/server/scripts/lists/new/items/geo_point_item.json new file mode 100644 index 0000000000000..3cead302586b9 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/items/geo_point_item.json @@ -0,0 +1,5 @@ +{ + "id": "geo_point_item", + "list_id": "geo_point_list", + "value": "41.12,-71.34" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/items/geo_shape_item.json b/x-pack/plugins/lists/server/scripts/lists/new/items/geo_shape_item.json new file mode 100644 index 0000000000000..e14796dc69a97 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/items/geo_shape_item.json @@ -0,0 +1,5 @@ +{ + "id": "geo_shape_item", + "list_id": "geo_shape_list", + "value": "POINT (-77.03653 38.897676)" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/items/half_float_item.json b/x-pack/plugins/lists/server/scripts/lists/new/items/half_float_item.json new file mode 100644 index 0000000000000..78b0e022024a9 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/items/half_float_item.json @@ -0,0 +1,5 @@ +{ + "id": "half_float_item", + "list_id": "half_float_list", + "value": "12.2" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/items/integer_item.json b/x-pack/plugins/lists/server/scripts/lists/new/items/integer_item.json new file mode 100644 index 0000000000000..9c3c30c91077a --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/items/integer_item.json @@ -0,0 +1,5 @@ +{ + "id": "integer_item", + "list_id": "integer_list", + "value": "2" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/items/integer_range_item.json b/x-pack/plugins/lists/server/scripts/lists/new/items/integer_range_item.json new file mode 100644 index 0000000000000..bf82ffb4c8d34 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/items/integer_range_item.json @@ -0,0 +1,5 @@ +{ + "id": "integer_item", + "list_id": "integer_range_list", + "value": "3-10" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/items/ip_item.json b/x-pack/plugins/lists/server/scripts/lists/new/items/ip_item.json new file mode 100644 index 0000000000000..563139c40c0ca --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/items/ip_item.json @@ -0,0 +1,5 @@ +{ + "id": "ip_item", + "list_id": "ip_list", + "value": "10.4.2.140" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/list_ip_item_everything.json b/x-pack/plugins/lists/server/scripts/lists/new/items/ip_item_everything.json similarity index 67% rename from x-pack/plugins/lists/server/scripts/lists/new/list_ip_item_everything.json rename to x-pack/plugins/lists/server/scripts/lists/new/items/ip_item_everything.json index 9730c1b7523f1..dc4b092de9a0c 100644 --- a/x-pack/plugins/lists/server/scripts/lists/new/list_ip_item_everything.json +++ b/x-pack/plugins/lists/server/scripts/lists/new/items/ip_item_everything.json @@ -1,6 +1,6 @@ { - "id": "hand_inserted_item_id_everything", - "list_id": "list-ip", + "id": "item_id_everything", + "list_id": "ip_list", "value": "127.0.0.2", "meta": { "level_1_meta": { diff --git a/x-pack/plugins/lists/server/scripts/lists/new/items/ip_range_item.json b/x-pack/plugins/lists/server/scripts/lists/new/items/ip_range_item.json new file mode 100644 index 0000000000000..8a6d7b0284e3c --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/items/ip_range_item.json @@ -0,0 +1,5 @@ +{ + "id": "ip_range_item", + "list_id": "ip_range_list", + "value": "192.168.0.0/16" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/items/keyword_item.json b/x-pack/plugins/lists/server/scripts/lists/new/items/keyword_item.json new file mode 100644 index 0000000000000..96d925c157490 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/items/keyword_item.json @@ -0,0 +1,4 @@ +{ + "list_id": "keyword_list", + "value": "kibana" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/items/long_item.json b/x-pack/plugins/lists/server/scripts/lists/new/items/long_item.json new file mode 100644 index 0000000000000..bc3f88c6dd06f --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/items/long_item.json @@ -0,0 +1,5 @@ +{ + "id": "long_item", + "list_id": "long_list", + "value": "34" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/items/long_range_item.json b/x-pack/plugins/lists/server/scripts/lists/new/items/long_range_item.json new file mode 100644 index 0000000000000..6f51b191e4a92 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/items/long_range_item.json @@ -0,0 +1,5 @@ +{ + "id": "long_range_item", + "list_id": "long_range_list", + "value": "12-23" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/items/shape_item.json b/x-pack/plugins/lists/server/scripts/lists/new/items/shape_item.json new file mode 100644 index 0000000000000..82d555a194637 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/items/shape_item.json @@ -0,0 +1,5 @@ +{ + "id": "shape_item", + "list_id": "shape_list", + "value": "POINT (-77.03653 38.897676)" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/items/short_item.json b/x-pack/plugins/lists/server/scripts/lists/new/items/short_item.json new file mode 100644 index 0000000000000..a0e28683cb1eb --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/items/short_item.json @@ -0,0 +1,5 @@ +{ + "id": "short_item", + "list_id": "short_list", + "value": "11" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/items/text_item.json b/x-pack/plugins/lists/server/scripts/lists/new/items/text_item.json new file mode 100644 index 0000000000000..14748d04bb6f4 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/items/text_item.json @@ -0,0 +1,5 @@ +{ + "id": "item_text", + "list_id": "text_list", + "value": "some text for you" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/list_keyword_item.json b/x-pack/plugins/lists/server/scripts/lists/new/list_keyword_item.json deleted file mode 100644 index b736e7b96ad98..0000000000000 --- a/x-pack/plugins/lists/server/scripts/lists/new/list_keyword_item.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "list_id": "list-keyword", - "value": "kibana" -} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/list_auto_id.json b/x-pack/plugins/lists/server/scripts/lists/new/lists/auto_id.json similarity index 100% rename from x-pack/plugins/lists/server/scripts/lists/new/list_auto_id.json rename to x-pack/plugins/lists/server/scripts/lists/new/lists/auto_id.json diff --git a/x-pack/plugins/lists/server/scripts/lists/new/lists/binary.json b/x-pack/plugins/lists/server/scripts/lists/new/lists/binary.json new file mode 100644 index 0000000000000..bb5de2e1cd7e9 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/lists/binary.json @@ -0,0 +1,6 @@ +{ + "id": "binary_list", + "name": "Simple list with binary", + "description": "This list has binary", + "type": "binary" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/lists/boolean.json b/x-pack/plugins/lists/server/scripts/lists/new/lists/boolean.json new file mode 100644 index 0000000000000..847ecac986e99 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/lists/boolean.json @@ -0,0 +1,6 @@ +{ + "id": "boolean_list", + "name": "Simple list with boolean values", + "description": "This list has booleans", + "type": "boolean" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/lists/byte.json b/x-pack/plugins/lists/server/scripts/lists/new/lists/byte.json new file mode 100644 index 0000000000000..c7042b032e447 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/lists/byte.json @@ -0,0 +1,6 @@ +{ + "id": "byte_list", + "name": "Simple list with bytes", + "description": "This list has bytes", + "type": "byte" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/lists/date.json b/x-pack/plugins/lists/server/scripts/lists/new/lists/date.json new file mode 100644 index 0000000000000..2ec6eef056d17 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/lists/date.json @@ -0,0 +1,6 @@ +{ + "id": "date_list", + "name": "Simple list with dates", + "description": "This list has dates", + "type": "date" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/lists/date_nanos.json b/x-pack/plugins/lists/server/scripts/lists/new/lists/date_nanos.json new file mode 100644 index 0000000000000..4fddde74609d4 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/lists/date_nanos.json @@ -0,0 +1,6 @@ +{ + "id": "date_nanos_list", + "name": "Simple list with date nanos", + "description": "This list has dates in the format of nanos", + "type": "date_nanos" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/lists/date_range.json b/x-pack/plugins/lists/server/scripts/lists/new/lists/date_range.json new file mode 100644 index 0000000000000..29729bad2b4d0 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/lists/date_range.json @@ -0,0 +1,6 @@ +{ + "id": "date_range_list", + "name": "Simple list with date ranges", + "description": "This list has date ranges", + "type": "date_range" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/lists/date_range_custom_format.json b/x-pack/plugins/lists/server/scripts/lists/new/lists/date_range_custom_format.json new file mode 100644 index 0000000000000..166dc2562dfac --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/lists/date_range_custom_format.json @@ -0,0 +1,8 @@ +{ + "id": "date_range_custom_format_list", + "name": "Simple list with date ranges", + "serializer": "(?.+)/(?.+)", + "deserializer": "{{gte}}/{{lte}}", + "description": "This list has date ranges", + "type": "date_range" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/lists/double.json b/x-pack/plugins/lists/server/scripts/lists/new/lists/double.json new file mode 100644 index 0000000000000..92350c36da68b --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/lists/double.json @@ -0,0 +1,6 @@ +{ + "id": "double_list", + "name": "Simple list with doubles", + "description": "This list has double values", + "type": "double" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/lists/double_range.json b/x-pack/plugins/lists/server/scripts/lists/new/lists/double_range.json new file mode 100644 index 0000000000000..d901c36a6ab01 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/lists/double_range.json @@ -0,0 +1,6 @@ +{ + "id": "double_range_list", + "name": "Simple list with double ranges", + "description": "This list has double ranges", + "type": "double_range" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/list_everything.json b/x-pack/plugins/lists/server/scripts/lists/new/lists/everything.json similarity index 88% rename from x-pack/plugins/lists/server/scripts/lists/new/list_everything.json rename to x-pack/plugins/lists/server/scripts/lists/new/lists/everything.json index 196b3b149ab82..65616dc163f62 100644 --- a/x-pack/plugins/lists/server/scripts/lists/new/list_everything.json +++ b/x-pack/plugins/lists/server/scripts/lists/new/lists/everything.json @@ -1,5 +1,5 @@ { - "id": "list-ip-everything", + "id": "ip_everything_list", "name": "Simple list with an ip", "description": "This list describes bad internet ip", "type": "ip", diff --git a/x-pack/plugins/lists/server/scripts/lists/new/lists/float.json b/x-pack/plugins/lists/server/scripts/lists/new/lists/float.json new file mode 100644 index 0000000000000..ac2cf994c2dac --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/lists/float.json @@ -0,0 +1,6 @@ +{ + "id": "float_list", + "name": "Simple list with floats", + "description": "This list has floats", + "type": "float" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/lists/float_range.json b/x-pack/plugins/lists/server/scripts/lists/new/lists/float_range.json new file mode 100644 index 0000000000000..34bf8d6d83e61 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/lists/float_range.json @@ -0,0 +1,6 @@ +{ + "id": "float_range_list", + "name": "Simple list with float ranges", + "description": "This list has float ranges", + "type": "float_range" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/lists/geo_point.json b/x-pack/plugins/lists/server/scripts/lists/new/lists/geo_point.json new file mode 100644 index 0000000000000..da5b7d6a3b120 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/lists/geo_point.json @@ -0,0 +1,6 @@ +{ + "id": "geo_point_list", + "name": "Simple list with geo points", + "description": "This list has geo points", + "type": "geo_point" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/lists/geo_shape.json b/x-pack/plugins/lists/server/scripts/lists/new/lists/geo_shape.json new file mode 100644 index 0000000000000..ccdaf43f9acc1 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/lists/geo_shape.json @@ -0,0 +1,6 @@ +{ + "id": "geo_shape_list", + "name": "Simple list with geo shapes", + "description": "This list has geo shapes", + "type": "geo_shape" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/lists/half_float.json b/x-pack/plugins/lists/server/scripts/lists/new/lists/half_float.json new file mode 100644 index 0000000000000..696c2b206816a --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/lists/half_float.json @@ -0,0 +1,6 @@ +{ + "id": "half_float_list", + "name": "Simple list with half float", + "description": "This list describes bad internet ip", + "type": "half_float" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/lists/integer.json b/x-pack/plugins/lists/server/scripts/lists/new/lists/integer.json new file mode 100644 index 0000000000000..0b09293434af0 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/lists/integer.json @@ -0,0 +1,6 @@ +{ + "id": "integer_list", + "name": "Simple list with an integer", + "description": "This list has integers", + "type": "integer" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/lists/integer_range.json b/x-pack/plugins/lists/server/scripts/lists/new/lists/integer_range.json new file mode 100644 index 0000000000000..9797b567a9e41 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/lists/integer_range.json @@ -0,0 +1,6 @@ +{ + "id": "integer_range_list", + "name": "Simple list with integer ranges", + "description": "This list has integer ranges", + "type": "integer_range" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/list_ip.json b/x-pack/plugins/lists/server/scripts/lists/new/lists/ip.json similarity index 85% rename from x-pack/plugins/lists/server/scripts/lists/new/list_ip.json rename to x-pack/plugins/lists/server/scripts/lists/new/lists/ip.json index 3e12ef1754f07..2d6531eb6db93 100644 --- a/x-pack/plugins/lists/server/scripts/lists/new/list_ip.json +++ b/x-pack/plugins/lists/server/scripts/lists/new/lists/ip.json @@ -1,5 +1,5 @@ { - "id": "list-ip", + "id": "ip_list", "name": "Simple list with an ip", "description": "This list describes bad internet ip", "type": "ip" diff --git a/x-pack/plugins/lists/server/scripts/lists/new/lists/ip_custom_format.json b/x-pack/plugins/lists/server/scripts/lists/new/lists/ip_custom_format.json new file mode 100644 index 0000000000000..2eab99ca3c5f5 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/lists/ip_custom_format.json @@ -0,0 +1,8 @@ +{ + "id": "ip_custom_format_list", + "name": "Simple list with an ip", + "description": "This list describes bad internet ip and matches the first found ip within a list", + "serializer": "(?((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?).){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))", + "deserializer": "{{value}}", + "type": "ip" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/list_ip_no_id.json b/x-pack/plugins/lists/server/scripts/lists/new/lists/ip_no_id.json similarity index 100% rename from x-pack/plugins/lists/server/scripts/lists/new/list_ip_no_id.json rename to x-pack/plugins/lists/server/scripts/lists/new/lists/ip_no_id.json diff --git a/x-pack/plugins/lists/server/scripts/lists/new/lists/ip_range.json b/x-pack/plugins/lists/server/scripts/lists/new/lists/ip_range.json new file mode 100644 index 0000000000000..a59f92595fafc --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/lists/ip_range.json @@ -0,0 +1,6 @@ +{ + "id": "ip_range_list", + "name": "Simple list with ip ranges", + "description": "This list has ip ranges", + "type": "ip_range" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/list_keyword.json b/x-pack/plugins/lists/server/scripts/lists/new/lists/keyword.json similarity index 83% rename from x-pack/plugins/lists/server/scripts/lists/new/list_keyword.json rename to x-pack/plugins/lists/server/scripts/lists/new/lists/keyword.json index e8f5fa7e38a06..0b49fa2f620a6 100644 --- a/x-pack/plugins/lists/server/scripts/lists/new/list_keyword.json +++ b/x-pack/plugins/lists/server/scripts/lists/new/lists/keyword.json @@ -1,5 +1,5 @@ { - "id": "list-keyword", + "id": "keyword_list", "name": "Simple list with a keyword", "description": "This list describes bad host names", "type": "keyword" diff --git a/x-pack/plugins/lists/server/scripts/lists/new/lists/keyword_custom_format.json b/x-pack/plugins/lists/server/scripts/lists/new/lists/keyword_custom_format.json new file mode 100644 index 0000000000000..bf83ad9ae8a79 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/lists/keyword_custom_format.json @@ -0,0 +1,8 @@ +{ + "id": "keyword_custom_format_list", + "name": "Simple list with a keyword using a custom format", + "description": "This parses the first found ipv4 only", + "serializer": "(?((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?).){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))", + "deserializer": "{{value}}", + "type": "keyword" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/lists/long.json b/x-pack/plugins/lists/server/scripts/lists/new/lists/long.json new file mode 100644 index 0000000000000..88fcee35a6a37 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/lists/long.json @@ -0,0 +1,6 @@ +{ + "id": "long_list", + "name": "Simple list with long values", + "description": "This list has long values", + "type": "long" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/lists/shape.json b/x-pack/plugins/lists/server/scripts/lists/new/lists/shape.json new file mode 100644 index 0000000000000..85991bbfa998a --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/lists/shape.json @@ -0,0 +1,6 @@ +{ + "id": "shape_list", + "name": "Simple list with shapes", + "description": "This list has shapes", + "type": "shape" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/lists/short.json b/x-pack/plugins/lists/server/scripts/lists/new/lists/short.json new file mode 100644 index 0000000000000..e3e3133ef2df6 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/lists/short.json @@ -0,0 +1,6 @@ +{ + "id": "short_list", + "name": "Simple list with short values", + "description": "This list has short value", + "type": "short" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/new/lists/text.json b/x-pack/plugins/lists/server/scripts/lists/new/lists/text.json new file mode 100644 index 0000000000000..2e0f989043038 --- /dev/null +++ b/x-pack/plugins/lists/server/scripts/lists/new/lists/text.json @@ -0,0 +1,6 @@ +{ + "id": "text_list", + "name": "Simple list with text", + "description": "This list has text", + "type": "text" +} diff --git a/x-pack/plugins/lists/server/scripts/lists/patches/list_ip_item.json b/x-pack/plugins/lists/server/scripts/lists/patches/ip_item.json similarity index 50% rename from x-pack/plugins/lists/server/scripts/lists/patches/list_ip_item.json rename to x-pack/plugins/lists/server/scripts/lists/patches/ip_item.json index 00c3496e71b35..3f535a0a51bff 100644 --- a/x-pack/plugins/lists/server/scripts/lists/patches/list_ip_item.json +++ b/x-pack/plugins/lists/server/scripts/lists/patches/ip_item.json @@ -1,4 +1,4 @@ { - "id": "hand_inserted_item_id", + "id": "ip_item", "value": "255.255.255.255" } diff --git a/x-pack/plugins/lists/server/scripts/lists/patches/simplest_updated_name.json b/x-pack/plugins/lists/server/scripts/lists/patches/simplest_updated_name.json index 1a57ab8b6a3b9..4610eff267645 100644 --- a/x-pack/plugins/lists/server/scripts/lists/patches/simplest_updated_name.json +++ b/x-pack/plugins/lists/server/scripts/lists/patches/simplest_updated_name.json @@ -1,4 +1,4 @@ { - "id": "list-ip", + "id": "ip_list", "name": "Changed the name here to something else" } diff --git a/x-pack/plugins/lists/server/scripts/lists/updates/list_ip_item.json b/x-pack/plugins/lists/server/scripts/lists/updates/ip_item.json similarity index 50% rename from x-pack/plugins/lists/server/scripts/lists/updates/list_ip_item.json rename to x-pack/plugins/lists/server/scripts/lists/updates/ip_item.json index 00c3496e71b35..3f535a0a51bff 100644 --- a/x-pack/plugins/lists/server/scripts/lists/updates/list_ip_item.json +++ b/x-pack/plugins/lists/server/scripts/lists/updates/ip_item.json @@ -1,4 +1,4 @@ { - "id": "hand_inserted_item_id", + "id": "ip_item", "value": "255.255.255.255" } diff --git a/x-pack/plugins/lists/server/scripts/lists/updates/simple_update.json b/x-pack/plugins/lists/server/scripts/lists/updates/simple_update.json index 936a070ede52c..3c766786be703 100644 --- a/x-pack/plugins/lists/server/scripts/lists/updates/simple_update.json +++ b/x-pack/plugins/lists/server/scripts/lists/updates/simple_update.json @@ -1,5 +1,5 @@ { - "id": "list-ip", + "id": "ip_list", "name": "Changed the name here to something else", "description": "Some other description here for you" } diff --git a/x-pack/plugins/lists/server/scripts/patch_list_item.sh b/x-pack/plugins/lists/server/scripts/patch_list_item.sh index 406b03dc6499c..82470f0aba533 100755 --- a/x-pack/plugins/lists/server/scripts/patch_list_item.sh +++ b/x-pack/plugins/lists/server/scripts/patch_list_item.sh @@ -10,10 +10,10 @@ set -e ./check_env_variables.sh # Uses a default if no argument is specified -LISTS=(${@:-./lists/patches/list_ip_item.json}) +LISTS=(${@:-./lists/patches/ip_item.json}) # Example: ./patch_list.sh -# Example: ./patch_list.sh ./lists/patches/list_ip_item.json +# Example: ./patch_list.sh ./lists/patches/ip_item.json for LIST in "${LISTS[@]}" do { [ -e "$LIST" ] || continue diff --git a/x-pack/plugins/lists/server/scripts/post_list.sh b/x-pack/plugins/lists/server/scripts/post_list.sh index 6aaffee0bc4b2..e0e442164535c 100755 --- a/x-pack/plugins/lists/server/scripts/post_list.sh +++ b/x-pack/plugins/lists/server/scripts/post_list.sh @@ -10,10 +10,10 @@ set -e ./check_env_variables.sh # Uses a default if no argument is specified -LISTS=(${@:-./lists/new/list_ip.json}) +LISTS=(${@:-./lists/new/lists/ip.json}) # Example: ./post_list.sh -# Example: ./post_list.sh ./lists/new/list_ip.json +# Example: ./post_list.sh ./lists/new/lists/ip.json for LIST in "${LISTS[@]}" do { [ -e "$LIST" ] || continue diff --git a/x-pack/plugins/lists/server/scripts/post_list_item.sh b/x-pack/plugins/lists/server/scripts/post_list_item.sh index b55a60420674f..49f9759c7879c 100755 --- a/x-pack/plugins/lists/server/scripts/post_list_item.sh +++ b/x-pack/plugins/lists/server/scripts/post_list_item.sh @@ -10,10 +10,10 @@ set -e ./check_env_variables.sh # Uses a default if no argument is specified -LISTS=(${@:-./lists/new/list_ip_item.json}) +LISTS=(${@:-./lists/new/items/ip_item.json}) # Example: ./post_list.sh -# Example: ./post_list.sh ./lists/new/list_ip_item.json +# Example: ./post_list.sh ./lists/new/items/ip_item.json for LIST in "${LISTS[@]}" do { [ -e "$LIST" ] || continue diff --git a/x-pack/plugins/lists/server/scripts/update_list_item.sh b/x-pack/plugins/lists/server/scripts/update_list_item.sh index e3153bfd25b19..e9915f905b971 100755 --- a/x-pack/plugins/lists/server/scripts/update_list_item.sh +++ b/x-pack/plugins/lists/server/scripts/update_list_item.sh @@ -10,10 +10,10 @@ set -e ./check_env_variables.sh # Uses a default if no argument is specified -LISTS=(${@:-./lists/updates/list_ip_item.json}) +LISTS=(${@:-./lists/updates/ip_item.json}) # Example: ./patch_list.sh -# Example: ./patch_list.sh ./lists/updates/list_ip_item.json +# Example: ./patch_list.sh ./lists/updates/ip_item.json for LIST in "${LISTS[@]}" do { [ -e "$LIST" ] || continue diff --git a/x-pack/plugins/lists/server/services/items/create_list_item.mock.ts b/x-pack/plugins/lists/server/services/items/create_list_item.mock.ts index 919aab5831440..96f29492482d5 100644 --- a/x-pack/plugins/lists/server/services/items/create_list_item.mock.ts +++ b/x-pack/plugins/lists/server/services/items/create_list_item.mock.ts @@ -20,10 +20,12 @@ import { export const getCreateListItemOptionsMock = (): CreateListItemOptions => ({ callCluster: getCallClusterMock(), dateNow: DATE_NOW, + deserializer: undefined, id: LIST_ITEM_ID, listId: LIST_ID, listItemIndex: LIST_ITEM_INDEX, meta: META, + serializer: undefined, tieBreaker: TIE_BREAKER, type: TYPE, user: USER, diff --git a/x-pack/plugins/lists/server/services/items/create_list_item.test.ts b/x-pack/plugins/lists/server/services/items/create_list_item.test.ts index 721d459bd7cc6..7fbdc900fe2a4 100644 --- a/x-pack/plugins/lists/server/services/items/create_list_item.test.ts +++ b/x-pack/plugins/lists/server/services/items/create_list_item.test.ts @@ -8,7 +8,7 @@ import { getListItemResponseMock } from '../../../common/schemas/response/list_i import { getIndexESListItemMock } from '../../../common/schemas/elastic_query/index_es_list_item_schema.mock'; import { LIST_ITEM_ID, LIST_ITEM_INDEX } from '../../../common/constants.mock'; -import { createListItem } from './create_list_item'; +import { CreateListItemOptions, createListItem } from './create_list_item'; import { getCreateListItemOptionsMock } from './create_list_item.mock'; describe('crete_list_item', () => { @@ -48,4 +48,15 @@ describe('crete_list_item', () => { expected.id = 'elastic-id-123'; expect(list).toEqual(expected); }); + + test('It returns null if an item does not match something such as an ip_range with an empty serializer', async () => { + const options: CreateListItemOptions = { + ...getCreateListItemOptionsMock(), + serializer: '', + type: 'ip_range', + value: '# some comment', + }; + const list = await createListItem(options); + expect(list).toEqual(null); + }); }); diff --git a/x-pack/plugins/lists/server/services/items/create_list_item.ts b/x-pack/plugins/lists/server/services/items/create_list_item.ts index 7c2f8ade6521e..333f34946828a 100644 --- a/x-pack/plugins/lists/server/services/items/create_list_item.ts +++ b/x-pack/plugins/lists/server/services/items/create_list_item.ts @@ -9,16 +9,20 @@ import { CreateDocumentResponse } from 'elasticsearch'; import { LegacyAPICaller } from 'kibana/server'; import { + DeserializerOrUndefined, IdOrUndefined, IndexEsListItemSchema, ListItemSchema, MetaOrUndefined, + SerializerOrUndefined, Type, } from '../../../common/schemas'; import { transformListItemToElasticQuery } from '../utils'; export interface CreateListItemOptions { + deserializer: DeserializerOrUndefined; id: IdOrUndefined; + serializer: SerializerOrUndefined; listId: string; type: Type; value: string; @@ -31,7 +35,9 @@ export interface CreateListItemOptions { } export const createListItem = async ({ + deserializer, id, + serializer, listId, type, value, @@ -41,33 +47,39 @@ export const createListItem = async ({ meta, dateNow, tieBreaker, -}: CreateListItemOptions): Promise => { +}: CreateListItemOptions): Promise => { const createdAt = dateNow ?? new Date().toISOString(); const tieBreakerId = tieBreaker ?? uuid.v4(); const baseBody = { created_at: createdAt, created_by: user, + deserializer, list_id: listId, meta, + serializer, tie_breaker_id: tieBreakerId, updated_at: createdAt, updated_by: user, }; - const body: IndexEsListItemSchema = { - ...baseBody, - ...transformListItemToElasticQuery({ type, value }), - }; - - const response = await callCluster('index', { - body, - id, - index: listItemIndex, - }); + const elasticQuery = transformListItemToElasticQuery({ serializer, type, value }); + if (elasticQuery != null) { + const body: IndexEsListItemSchema = { + ...baseBody, + ...elasticQuery, + }; + const response = await callCluster('index', { + body, + id, + index: listItemIndex, + }); - return { - id: response._id, - type, - value, - ...baseBody, - }; + return { + id: response._id, + type, + value, + ...baseBody, + }; + } else { + return null; + } }; diff --git a/x-pack/plugins/lists/server/services/items/create_list_items_bulk.mock.ts b/x-pack/plugins/lists/server/services/items/create_list_items_bulk.mock.ts index dd15d6f74a2ab..63f1b7dbf8d69 100644 --- a/x-pack/plugins/lists/server/services/items/create_list_items_bulk.mock.ts +++ b/x-pack/plugins/lists/server/services/items/create_list_items_bulk.mock.ts @@ -21,9 +21,11 @@ import { export const getCreateListItemBulkOptionsMock = (): CreateListItemsBulkOptions => ({ callCluster: getCallClusterMock(), dateNow: DATE_NOW, + deserializer: undefined, listId: LIST_ID, listItemIndex: LIST_ITEM_INDEX, meta: META, + serializer: undefined, tieBreaker: TIE_BREAKERS, type: TYPE, user: USER, diff --git a/x-pack/plugins/lists/server/services/items/create_list_items_bulk.test.ts b/x-pack/plugins/lists/server/services/items/create_list_items_bulk.test.ts index dbbb257f22d11..4ab1bfb856846 100644 --- a/x-pack/plugins/lists/server/services/items/create_list_items_bulk.test.ts +++ b/x-pack/plugins/lists/server/services/items/create_list_items_bulk.test.ts @@ -7,7 +7,7 @@ import { getIndexESListItemMock } from '../../../common/schemas/elastic_query/index_es_list_item_schema.mock'; import { LIST_ITEM_INDEX, TIE_BREAKERS, VALUE_2 } from '../../../common/constants.mock'; -import { createListItemsBulk } from './create_list_items_bulk'; +import { CreateListItemsBulkOptions, createListItemsBulk } from './create_list_items_bulk'; import { getCreateListItemBulkOptionsMock } from './create_list_items_bulk.mock'; describe('crete_list_item_bulk', () => { @@ -41,4 +41,35 @@ describe('crete_list_item_bulk', () => { options.value = []; expect(options.callCluster).not.toBeCalled(); }); + + test('It should skip over a value if it is not able to add that item because it is not parsable such as an ip_range with a serializer that only matches one ip', async () => { + const options: CreateListItemsBulkOptions = { + ...getCreateListItemBulkOptionsMock(), + serializer: '(?127.0.0.1)', // this will create a regular expression which will only match 127.0.0.1 and not 127.0.0.1 + type: 'ip_range', + value: ['127.0.0.1', '127.0.0.2'], + }; + await createListItemsBulk(options); + expect(options.callCluster).toBeCalledWith('bulk', { + body: [ + { create: { _index: LIST_ITEM_INDEX } }, + { + created_at: '2020-04-20T15:25:31.830Z', + created_by: 'some user', + deserializer: undefined, + ip_range: { + gte: '127.0.0.1', + lte: '127.0.0.1', + }, + list_id: 'some-list-id', + meta: {}, + serializer: '(?127.0.0.1)', + tie_breaker_id: TIE_BREAKERS[0], + updated_at: '2020-04-20T15:25:31.830Z', + updated_by: 'some user', + }, + ], + index: '.items', + }); + }); }); diff --git a/x-pack/plugins/lists/server/services/items/create_list_items_bulk.ts b/x-pack/plugins/lists/server/services/items/create_list_items_bulk.ts index ced1a6344f9b7..447c0f6bf95cc 100644 --- a/x-pack/plugins/lists/server/services/items/create_list_items_bulk.ts +++ b/x-pack/plugins/lists/server/services/items/create_list_items_bulk.ts @@ -10,12 +10,16 @@ import { LegacyAPICaller } from 'kibana/server'; import { transformListItemToElasticQuery } from '../utils'; import { CreateEsBulkTypeSchema, + DeserializerOrUndefined, IndexEsListItemSchema, MetaOrUndefined, + SerializerOrUndefined, Type, } from '../../../common/schemas'; export interface CreateListItemsBulkOptions { + deserializer: DeserializerOrUndefined; + serializer: SerializerOrUndefined; listId: string; type: Type; value: string[]; @@ -30,6 +34,8 @@ export interface CreateListItemsBulkOptions { export const createListItemsBulk = async ({ listId, type, + deserializer, + serializer, value, callCluster, listItemIndex, @@ -47,18 +53,30 @@ export const createListItemsBulk = async ({ const createdAt = dateNow ?? new Date().toISOString(); const tieBreakerId = tieBreaker != null && tieBreaker[index] != null ? tieBreaker[index] : uuid.v4(); - const elasticBody: IndexEsListItemSchema = { - created_at: createdAt, - created_by: user, - list_id: listId, - meta, - tie_breaker_id: tieBreakerId, - updated_at: createdAt, - updated_by: user, - ...transformListItemToElasticQuery({ type, value: singleValue }), - }; - const createBody: CreateEsBulkTypeSchema = { create: { _index: listItemIndex } }; - return [...accum, createBody, elasticBody]; + const elasticQuery = transformListItemToElasticQuery({ + serializer, + type, + value: singleValue, + }); + if (elasticQuery != null) { + const elasticBody: IndexEsListItemSchema = { + created_at: createdAt, + created_by: user, + deserializer, + list_id: listId, + meta, + serializer, + tie_breaker_id: tieBreakerId, + updated_at: createdAt, + updated_by: user, + ...elasticQuery, + }; + const createBody: CreateEsBulkTypeSchema = { create: { _index: listItemIndex } }; + return [...accum, createBody, elasticBody]; + } else { + // TODO: Report errors with return values from the bulk insert + return accum; + } }, [] ); diff --git a/x-pack/plugins/lists/server/services/items/find_list_item.mock.ts b/x-pack/plugins/lists/server/services/items/find_list_item.mock.ts new file mode 100644 index 0000000000000..702144cb58865 --- /dev/null +++ b/x-pack/plugins/lists/server/services/items/find_list_item.mock.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Client } from 'elasticsearch'; + +import { getSearchListMock } from '../../../common/schemas/elastic_response/search_es_list_schema.mock'; +import { getShardMock } from '../../../common/get_shard.mock'; +import { getSearchListItemMock } from '../../../common/schemas/elastic_response/search_es_list_item_schema.mock'; +import { getCallClusterMockMultiTimes } from '../../../common/get_call_cluster.mock'; +import { LIST_ID, LIST_INDEX, LIST_ITEM_INDEX } from '../../../common/constants.mock'; + +import { FindListItemOptions } from './find_list_item'; + +export const getFindCount = (): ReturnType => { + return Promise.resolve({ + _shards: getShardMock(), + count: 1, + }); +}; + +export const getFindListItemOptionsMock = (): FindListItemOptions => { + const callCluster = getCallClusterMockMultiTimes([ + getSearchListMock(), + getFindCount(), + getSearchListItemMock(), + ]); + return { + callCluster, + currentIndexPosition: 0, + filter: '', + listId: LIST_ID, + listIndex: LIST_INDEX, + listItemIndex: LIST_ITEM_INDEX, + page: 1, + perPage: 25, + searchAfter: undefined, + sortField: undefined, + sortOrder: undefined, + }; +}; diff --git a/x-pack/plugins/lists/server/services/items/find_list_item.test.ts b/x-pack/plugins/lists/server/services/items/find_list_item.test.ts new file mode 100644 index 0000000000000..023246437c46f --- /dev/null +++ b/x-pack/plugins/lists/server/services/items/find_list_item.test.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { getEmptySearchListMock } from '../../../common/schemas/elastic_response/search_es_list_schema.mock'; +import { getCallClusterMockMultiTimes } from '../../../common/get_call_cluster.mock'; +import { getFoundListItemSchemaMock } from '../../../common/schemas/response/found_list_item_schema.mock'; + +import { getFindListItemOptionsMock } from './find_list_item.mock'; +import { findListItem } from './find_list_item'; + +describe('find_list_item', () => { + test('should find a simple single list item', async () => { + const options = getFindListItemOptionsMock(); + const item = await findListItem(options); + const expected = getFoundListItemSchemaMock(); + expect(item).toEqual(expected); + }); + + test('should return null if the list is null', async () => { + const options = getFindListItemOptionsMock(); + options.callCluster = getCallClusterMockMultiTimes([getEmptySearchListMock()]); + const item = await findListItem(options); + expect(item).toEqual(null); + }); +}); diff --git a/x-pack/plugins/lists/server/services/items/find_list_item.ts b/x-pack/plugins/lists/server/services/items/find_list_item.ts index 0f120f73d195e..93fa682631b06 100644 --- a/x-pack/plugins/lists/server/services/items/find_list_item.ts +++ b/x-pack/plugins/lists/server/services/items/find_list_item.ts @@ -19,14 +19,14 @@ import { import { getList } from '../lists'; import { encodeCursor, - getQueryFilter, + getQueryFilterWithListId, getSearchAfterWithTieBreaker, getSortWithTieBreaker, scrollToStartPage, transformElasticToListItem, } from '../utils'; -interface FindListItemOptions { +export interface FindListItemOptions { listId: ListId; filter: Filter; currentIndexPosition: number; @@ -53,11 +53,11 @@ export const findListItem = async ({ listItemIndex, sortOrder, }: FindListItemOptions): Promise => { - const query = getQueryFilter({ filter }); const list = await getList({ callCluster, id: listId, listIndex }); if (list == null) { return null; } else { + const query = getQueryFilterWithListId({ filter, listId }); const sortField = sortFieldWithPossibleValue === 'value' ? list.type : sortFieldWithPossibleValue; const scroll = await scrollToStartPage({ diff --git a/x-pack/plugins/lists/server/services/items/get_list_item.test.ts b/x-pack/plugins/lists/server/services/items/get_list_item.test.ts index c39d6cdc00ee1..9f4cfca95a18c 100644 --- a/x-pack/plugins/lists/server/services/items/get_list_item.test.ts +++ b/x-pack/plugins/lists/server/services/items/get_list_item.test.ts @@ -7,7 +7,14 @@ import { getSearchListItemMock } from '../../../common/schemas/elastic_response/search_es_list_item_schema.mock'; import { getListItemResponseMock } from '../../../common/schemas/response/list_item_schema.mock'; import { getCallClusterMock } from '../../../common/get_call_cluster.mock'; -import { LIST_ID, LIST_INDEX } from '../../../common/constants.mock'; +import { + DATE_NOW, + LIST_ID, + LIST_INDEX, + META, + TIE_BREAKER, + USER, +} from '../../../common/constants.mock'; import { getListItem } from './get_list_item'; @@ -35,4 +42,45 @@ describe('get_list_item', () => { const list = await getListItem({ callCluster, id: LIST_ID, listItemIndex: LIST_INDEX }); expect(list).toEqual(null); }); + + test('it returns null if all the values underneath the source type is undefined', async () => { + const data = getSearchListItemMock(); + data.hits.hits[0]._source = { + binary: undefined, + boolean: undefined, + byte: undefined, + created_at: DATE_NOW, + created_by: USER, + date: undefined, + date_nanos: undefined, + date_range: undefined, + deserializer: undefined, + double: undefined, + double_range: undefined, + float: undefined, + float_range: undefined, + geo_point: undefined, + geo_shape: undefined, + half_float: undefined, + integer: undefined, + integer_range: undefined, + ip: undefined, + ip_range: undefined, + keyword: undefined, + list_id: LIST_ID, + long: undefined, + long_range: undefined, + meta: META, + serializer: undefined, + shape: undefined, + short: undefined, + text: undefined, + tie_breaker_id: TIE_BREAKER, + updated_at: DATE_NOW, + updated_by: USER, + }; + const callCluster = getCallClusterMock(data); + const list = await getListItem({ callCluster, id: LIST_ID, listItemIndex: LIST_INDEX }); + expect(list).toEqual(null); + }); }); diff --git a/x-pack/plugins/lists/server/services/items/get_list_item.ts b/x-pack/plugins/lists/server/services/items/get_list_item.ts index 0c84ba50525cf..6f2a7ad63a973 100644 --- a/x-pack/plugins/lists/server/services/items/get_list_item.ts +++ b/x-pack/plugins/lists/server/services/items/get_list_item.ts @@ -7,7 +7,8 @@ import { LegacyAPICaller } from 'kibana/server'; import { Id, ListItemSchema, SearchEsListItemSchema } from '../../../common/schemas'; -import { deriveTypeFromItem, transformElasticToListItem } from '../utils'; +import { transformElasticToListItem } from '../utils'; +import { findSourceType } from '../utils/find_source_type'; interface GetListItemOptions { id: Id; @@ -33,9 +34,13 @@ export const getListItem = async ({ }); if (listItemES.hits.hits.length) { - const type = deriveTypeFromItem({ item: listItemES.hits.hits[0]._source }); - const listItems = transformElasticToListItem({ response: listItemES, type }); - return listItems[0]; + const type = findSourceType(listItemES.hits.hits[0]._source); + if (type != null) { + const listItems = transformElasticToListItem({ response: listItemES, type }); + return listItems[0]; + } else { + return null; + } } else { return null; } diff --git a/x-pack/plugins/lists/server/services/items/get_list_item_by_values.ts b/x-pack/plugins/lists/server/services/items/get_list_item_by_values.ts index 5e232443625dd..0ce056c76535d 100644 --- a/x-pack/plugins/lists/server/services/items/get_list_item_by_values.ts +++ b/x-pack/plugins/lists/server/services/items/get_list_item_by_values.ts @@ -34,7 +34,7 @@ export const getListItemByValues = async ({ }, ignoreUnavailable: true, index: listItemIndex, - size: value.length, // This has a limit on the number which is 10k + size: 10000, // TODO: This has a limit on the number which is 10,000 the default of Elastic but we might want to provide a way to increase that number }); return transformElasticToListItem({ response, type }); }; diff --git a/x-pack/plugins/lists/server/services/items/list_item_mappings.json b/x-pack/plugins/lists/server/services/items/list_item_mappings.json index ca69c26df52b5..1381402c2f2f4 100644 --- a/x-pack/plugins/lists/server/services/items/list_item_mappings.json +++ b/x-pack/plugins/lists/server/services/items/list_item_mappings.json @@ -7,10 +7,10 @@ "list_id": { "type": "keyword" }, - "ip": { - "type": "ip" + "deserializer": { + "type": "keyword" }, - "keyword": { + "serializer": { "type": "keyword" }, "meta": { @@ -28,6 +28,75 @@ }, "updated_by": { "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "long": { + "type": "long" + }, + "integer": { + "type": "integer" + }, + "short": { + "type": "short" + }, + "byte": { + "type": "byte" + }, + "double": { + "type": "double" + }, + "float": { + "type": "float" + }, + "half_float": { + "type": "half_float" + }, + "date": { + "type": "date" + }, + "date_nanos": { + "type": "date_nanos" + }, + "boolean": { + "type": "boolean" + }, + "binary": { + "type": "binary" + }, + "integer_range": { + "type": "integer_range" + }, + "float_range": { + "type": "float_range" + }, + "long_range": { + "type": "long_range" + }, + "double_range": { + "type": "double_range" + }, + "date_range": { + "type": "date_range" + }, + "ip_range": { + "type": "ip_range" + }, + "geo_point": { + "type": "geo_point" + }, + "geo_shape": { + "type": "geo_shape" + }, + "keyword": { + "type": "keyword" + }, + "text": { + "type": "text" + }, + "shape": { + "type": "shape" } } } diff --git a/x-pack/plugins/lists/server/services/items/update_list_item.test.ts b/x-pack/plugins/lists/server/services/items/update_list_item.test.ts index 95b99dc87bab6..cf8bc80b67f6a 100644 --- a/x-pack/plugins/lists/server/services/items/update_list_item.test.ts +++ b/x-pack/plugins/lists/server/services/items/update_list_item.test.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import { ListItemSchema } from '../../../common/schemas'; import { getListItemResponseMock } from '../../../common/schemas/response/list_item_schema.mock'; import { updateListItem } from './update_list_item'; @@ -24,12 +25,11 @@ describe('update_list_item', () => { }); test('it returns a list item as expected with the id changed out for the elastic id when there is a list item to update', async () => { - const list = getListItemResponseMock(); - ((getListItem as unknown) as jest.Mock).mockResolvedValueOnce(list); + const listItem = getListItemResponseMock(); + ((getListItem as unknown) as jest.Mock).mockResolvedValueOnce(listItem); const options = getUpdateListItemOptionsMock(); const updatedList = await updateListItem(options); - const expected = getListItemResponseMock(); - expected.id = 'elastic-id-123'; + const expected: ListItemSchema = { ...getListItemResponseMock(), id: 'elastic-id-123' }; expect(updatedList).toEqual(expected); }); @@ -39,4 +39,17 @@ describe('update_list_item', () => { const updatedList = await updateListItem(options); expect(updatedList).toEqual(null); }); + + test('it returns null when the serializer and type such as ip_range returns nothing', async () => { + const listItem: ListItemSchema = { + ...getListItemResponseMock(), + serializer: '', + type: 'ip_range', + value: '127.0.0.1', + }; + ((getListItem as unknown) as jest.Mock).mockResolvedValueOnce(listItem); + const options = getUpdateListItemOptionsMock(); + const updatedList = await updateListItem(options); + expect(updatedList).toEqual(null); + }); }); diff --git a/x-pack/plugins/lists/server/services/items/update_list_item.ts b/x-pack/plugins/lists/server/services/items/update_list_item.ts index 4ddb6e4c4c742..24cd11cbb65e4 100644 --- a/x-pack/plugins/lists/server/services/items/update_list_item.ts +++ b/x-pack/plugins/lists/server/services/items/update_list_item.ts @@ -41,31 +41,42 @@ export const updateListItem = async ({ if (listItem == null) { return null; } else { - const doc: UpdateEsListItemSchema = { - meta, - updated_at: updatedAt, - updated_by: user, - ...transformListItemToElasticQuery({ type: listItem.type, value: value ?? listItem.value }), - }; - - const response = await callCluster('update', { - body: { - doc, - }, - id: listItem.id, - index: listItemIndex, - }); - return { - created_at: listItem.created_at, - created_by: listItem.created_by, - id: response._id, - list_id: listItem.list_id, - meta: meta ?? listItem.meta, - tie_breaker_id: listItem.tie_breaker_id, + const elasticQuery = transformListItemToElasticQuery({ + serializer: listItem.serializer, type: listItem.type, - updated_at: updatedAt, - updated_by: listItem.updated_by, value: value ?? listItem.value, - }; + }); + if (elasticQuery == null) { + return null; + } else { + const doc: UpdateEsListItemSchema = { + meta, + updated_at: updatedAt, + updated_by: user, + ...elasticQuery, + }; + + const response = await callCluster('update', { + body: { + doc, + }, + id: listItem.id, + index: listItemIndex, + }); + return { + created_at: listItem.created_at, + created_by: listItem.created_by, + deserializer: listItem.deserializer, + id: response._id, + list_id: listItem.list_id, + meta: meta ?? listItem.meta, + serializer: listItem.serializer, + tie_breaker_id: listItem.tie_breaker_id, + type: listItem.type, + updated_at: updatedAt, + updated_by: listItem.updated_by, + value: value ?? listItem.value, + }; + } } }; diff --git a/x-pack/plugins/lists/server/services/items/write_lines_to_bulk_list_items.mock.ts b/x-pack/plugins/lists/server/services/items/write_lines_to_bulk_list_items.mock.ts index ccacdfbb5ff65..b7e30e0a1c308 100644 --- a/x-pack/plugins/lists/server/services/items/write_lines_to_bulk_list_items.mock.ts +++ b/x-pack/plugins/lists/server/services/items/write_lines_to_bulk_list_items.mock.ts @@ -11,9 +11,11 @@ import { TestReadable } from './test_readable.mock'; export const getImportListItemsToStreamOptionsMock = (): ImportListItemsToStreamOptions => ({ callCluster: getCallClusterMock(), + deserializer: undefined, listId: LIST_ID, listItemIndex: LIST_ITEM_INDEX, meta: META, + serializer: undefined, stream: new TestReadable(), type: TYPE, user: USER, @@ -22,9 +24,11 @@ export const getImportListItemsToStreamOptionsMock = (): ImportListItemsToStream export const getWriteBufferToItemsOptionsMock = (): WriteBufferToItemsOptions => ({ buffer: [], callCluster: getCallClusterMock(), + deserializer: undefined, listId: LIST_ID, listItemIndex: LIST_ITEM_INDEX, meta: META, + serializer: undefined, type: TYPE, user: USER, }); diff --git a/x-pack/plugins/lists/server/services/items/write_lines_to_bulk_list_items.test.ts b/x-pack/plugins/lists/server/services/items/write_lines_to_bulk_list_items.test.ts index 71db6fa2cf62c..ceefcb1febae0 100644 --- a/x-pack/plugins/lists/server/services/items/write_lines_to_bulk_list_items.test.ts +++ b/x-pack/plugins/lists/server/services/items/write_lines_to_bulk_list_items.test.ts @@ -16,10 +16,10 @@ import { getWriteBufferToItemsOptionsMock, } from './write_lines_to_bulk_list_items.mock'; -import { getListItemByValues } from '.'; +import { createListItemsBulk } from '.'; -jest.mock('./get_list_item_by_values', () => ({ - getListItemByValues: jest.fn(), +jest.mock('./create_list_items_bulk', () => ({ + createListItemsBulk: jest.fn(), })); describe('write_lines_to_bulk_list_items', () => { @@ -33,33 +33,30 @@ describe('write_lines_to_bulk_list_items', () => { describe('importListItemsToStream', () => { test('It imports a set of items to a write buffer by calling "getListItemByValues" with an empty buffer', async () => { - ((getListItemByValues as unknown) as jest.Mock).mockResolvedValueOnce([]); const options = getImportListItemsToStreamOptionsMock(); const promise = importListItemsToStream(options); options.stream.push(null); await promise; - expect(getListItemByValues).toBeCalledWith(expect.objectContaining({ value: [] })); + expect(createListItemsBulk).toBeCalledWith(expect.objectContaining({ value: [] })); }); test('It imports a set of items to a write buffer by calling "getListItemByValues" with a single value given', async () => { - ((getListItemByValues as unknown) as jest.Mock).mockResolvedValueOnce([]); const options = getImportListItemsToStreamOptionsMock(); const promise = importListItemsToStream(options); options.stream.push('127.0.0.1\n'); options.stream.push(null); await promise; - expect(getListItemByValues).toBeCalledWith(expect.objectContaining({ value: ['127.0.0.1'] })); + expect(createListItemsBulk).toBeCalledWith(expect.objectContaining({ value: ['127.0.0.1'] })); }); test('It imports a set of items to a write buffer by calling "getListItemByValues" with two values given', async () => { - ((getListItemByValues as unknown) as jest.Mock).mockResolvedValueOnce([]); const options = getImportListItemsToStreamOptionsMock(); const promise = importListItemsToStream(options); options.stream.push('127.0.0.1\n'); options.stream.push('127.0.0.2\n'); options.stream.push(null); await promise; - expect(getListItemByValues).toBeCalledWith( + expect(createListItemsBulk).toBeCalledWith( expect.objectContaining({ value: ['127.0.0.1', '127.0.0.2'] }) ); }); @@ -67,92 +64,76 @@ describe('write_lines_to_bulk_list_items', () => { describe('writeBufferToItems', () => { test('It returns no duplicates and no lines processed when given empty items', async () => { - ((getListItemByValues as unknown) as jest.Mock).mockResolvedValueOnce([]); const options = getWriteBufferToItemsOptionsMock(); const linesResult = await writeBufferToItems(options); const expected: LinesResult = { - duplicatesFound: 0, linesProcessed: 0, }; expect(linesResult).toEqual(expected); }); test('It returns no lines processed when given items but no buffer', async () => { - const data = getListItemResponseMock(); - ((getListItemByValues as unknown) as jest.Mock).mockResolvedValueOnce([data]); const options = getWriteBufferToItemsOptionsMock(); const linesResult = await writeBufferToItems(options); const expected: LinesResult = { - duplicatesFound: 0, linesProcessed: 0, }; expect(linesResult).toEqual(expected); }); test('It returns 1 lines processed when given a buffer item that is not a duplicate', async () => { - const data = getListItemResponseMock(); - ((getListItemByValues as unknown) as jest.Mock).mockResolvedValueOnce([data]); const options = getWriteBufferToItemsOptionsMock(); options.buffer = ['255.255.255.255']; const linesResult = await writeBufferToItems(options); const expected: LinesResult = { - duplicatesFound: 0, linesProcessed: 1, }; expect(linesResult).toEqual(expected); }); - test('It filters a duplicate value out and reports it as a duplicate', async () => { + test('It does not filter duplicate values out', async () => { const data = getListItemResponseMock(); - ((getListItemByValues as unknown) as jest.Mock).mockResolvedValueOnce([data]); const options = getWriteBufferToItemsOptionsMock(); options.buffer = [data.value]; const linesResult = await writeBufferToItems(options); const expected: LinesResult = { - duplicatesFound: 1, - linesProcessed: 0, + linesProcessed: 1, }; expect(linesResult).toEqual(expected); }); - test('It filters a duplicate value out and reports it as a duplicate and processing a second value as not a duplicate', async () => { + test('It does not filter a duplicate value out and processes a second value normally', async () => { const data = getListItemResponseMock(); - ((getListItemByValues as unknown) as jest.Mock).mockResolvedValueOnce([data]); const options = getWriteBufferToItemsOptionsMock(); options.buffer = ['255.255.255.255', data.value]; const linesResult = await writeBufferToItems(options); const expected: LinesResult = { - duplicatesFound: 1, - linesProcessed: 1, + linesProcessed: 2, }; expect(linesResult).toEqual(expected); }); - test('It filters a duplicate value out and reports it as a duplicate and processing two other values', async () => { + test('It does not filter a duplicate value out and reports it as a duplicate and processes two other values', async () => { const data = getListItemResponseMock(); - ((getListItemByValues as unknown) as jest.Mock).mockResolvedValueOnce([data]); const options = getWriteBufferToItemsOptionsMock(); options.buffer = ['255.255.255.255', '192.168.0.1', data.value]; const linesResult = await writeBufferToItems(options); const expected: LinesResult = { - duplicatesFound: 1, - linesProcessed: 2, + linesProcessed: 3, }; expect(linesResult).toEqual(expected); }); - test('It filters two duplicate values out and reports processes a single value', async () => { + test('It does not filter two duplicate values out and reports the values normally', async () => { const dataItem1 = getListItemResponseMock(); dataItem1.value = '127.0.0.1'; const dataItem2 = getListItemResponseMock(); dataItem2.value = '127.0.0.2'; - ((getListItemByValues as unknown) as jest.Mock).mockResolvedValueOnce([dataItem1, dataItem2]); const options = getWriteBufferToItemsOptionsMock(); options.buffer = [dataItem1.value, dataItem2.value, '192.168.0.0.1']; const linesResult = await writeBufferToItems(options); const expected: LinesResult = { - duplicatesFound: 2, - linesProcessed: 1, + linesProcessed: 3, }; expect(linesResult).toEqual(expected); }); diff --git a/x-pack/plugins/lists/server/services/items/write_lines_to_bulk_list_items.ts b/x-pack/plugins/lists/server/services/items/write_lines_to_bulk_list_items.ts index de983158ac707..31b2b74c88431 100644 --- a/x-pack/plugins/lists/server/services/items/write_lines_to_bulk_list_items.ts +++ b/x-pack/plugins/lists/server/services/items/write_lines_to_bulk_list_items.ts @@ -8,13 +8,19 @@ import { Readable } from 'stream'; import { LegacyAPICaller } from 'kibana/server'; -import { MetaOrUndefined, Type } from '../../../common/schemas'; +import { + DeserializerOrUndefined, + MetaOrUndefined, + SerializerOrUndefined, + Type, +} from '../../../common/schemas'; import { BufferLines } from './buffer_lines'; -import { getListItemByValues } from './get_list_item_by_values'; import { createListItemsBulk } from './create_list_items_bulk'; export interface ImportListItemsToStreamOptions { + deserializer: DeserializerOrUndefined; + serializer: SerializerOrUndefined; listId: string; stream: Readable; callCluster: LegacyAPICaller; @@ -25,6 +31,8 @@ export interface ImportListItemsToStreamOptions { } export const importListItemsToStream = ({ + deserializer, + serializer, listId, stream, callCluster, @@ -39,9 +47,11 @@ export const importListItemsToStream = ({ await writeBufferToItems({ buffer: lines, callCluster, + deserializer, listId, listItemIndex, meta, + serializer, type, user, }); @@ -55,6 +65,8 @@ export const importListItemsToStream = ({ export interface WriteBufferToItemsOptions { listId: string; + deserializer: DeserializerOrUndefined; + serializer: SerializerOrUndefined; callCluster: LegacyAPICaller; listItemIndex: string; buffer: string[]; @@ -65,38 +77,29 @@ export interface WriteBufferToItemsOptions { export interface LinesResult { linesProcessed: number; - duplicatesFound: number; } export const writeBufferToItems = async ({ listId, callCluster, + deserializer, + serializer, listItemIndex, buffer, type, user, meta, }: WriteBufferToItemsOptions): Promise => { - const items = await getListItemByValues({ - callCluster, - listId, - listItemIndex, - type, - value: buffer, - }); - const duplicatesRemoved = buffer.filter( - (bufferedValue) => !items.some((item) => item.value === bufferedValue) - ); - const linesProcessed = duplicatesRemoved.length; - const duplicatesFound = buffer.length - duplicatesRemoved.length; await createListItemsBulk({ callCluster, + deserializer, listId, listItemIndex, meta, + serializer, type, user, - value: duplicatesRemoved, + value: buffer, }); - return { duplicatesFound, linesProcessed }; + return { linesProcessed: buffer.length }; }; diff --git a/x-pack/plugins/lists/server/services/items/write_list_items_to_stream.ts b/x-pack/plugins/lists/server/services/items/write_list_items_to_stream.ts index 91f4644d359e3..a500d2ba04da2 100644 --- a/x-pack/plugins/lists/server/services/items/write_list_items_to_stream.ts +++ b/x-pack/plugins/lists/server/services/items/write_list_items_to_stream.ts @@ -11,6 +11,7 @@ import { LegacyAPICaller } from 'kibana/server'; import { SearchEsListItemSchema } from '../../../common/schemas'; import { ErrorWithStatusCode } from '../../error_with_status_code'; +import { findSourceValue } from '../utils/find_source_value'; /** * How many results to page through from the network at a time @@ -144,10 +145,9 @@ export const writeResponseHitsToStream = ({ const stringToAppendOrEmpty = stringToAppend ?? ''; response.hits.hits.forEach((hit) => { - if (hit._source.ip != null) { - stream.push(`${hit._source.ip}${stringToAppendOrEmpty}`); - } else if (hit._source.keyword != null) { - stream.push(`${hit._source.keyword}${stringToAppendOrEmpty}`); + const value = findSourceValue(hit._source); + if (value != null) { + stream.push(`${value}${stringToAppendOrEmpty}`); } else { throw new ErrorWithStatusCode( `Encountered an error where hit._source was an unexpected type: ${JSON.stringify( diff --git a/x-pack/plugins/lists/server/services/lists/create_list.mock.ts b/x-pack/plugins/lists/server/services/lists/create_list.mock.ts index f0fd023d018ae..84273ff4cf814 100644 --- a/x-pack/plugins/lists/server/services/lists/create_list.mock.ts +++ b/x-pack/plugins/lists/server/services/lists/create_list.mock.ts @@ -22,10 +22,12 @@ export const getCreateListOptionsMock = (): CreateListOptions => ({ callCluster: getCallClusterMock(), dateNow: DATE_NOW, description: DESCRIPTION, + deserializer: undefined, id: LIST_ID, listIndex: LIST_INDEX, meta: META, name: NAME, + serializer: undefined, tieBreaker: TIE_BREAKER, type: TYPE, user: USER, diff --git a/x-pack/plugins/lists/server/services/lists/create_list.test.ts b/x-pack/plugins/lists/server/services/lists/create_list.test.ts index ef610ece1acc9..43af08bcaf7ff 100644 --- a/x-pack/plugins/lists/server/services/lists/create_list.test.ts +++ b/x-pack/plugins/lists/server/services/lists/create_list.test.ts @@ -4,11 +4,12 @@ * you may not use this file except in compliance with the Elastic License. */ +import { ListSchema } from '../../../common/schemas'; import { getListResponseMock } from '../../../common/schemas/response/list_schema.mock'; import { getIndexESListMock } from '../../../common/schemas/elastic_query/index_es_list_schema.mock'; import { LIST_ID, LIST_INDEX } from '../../../common/constants.mock'; -import { createList } from './create_list'; +import { CreateListOptions, createList } from './create_list'; import { getCreateListOptionsMock } from './create_list.mock'; describe('crete_list', () => { @@ -23,8 +24,23 @@ describe('crete_list', () => { test('it returns a list as expected with the id changed out for the elastic id', async () => { const options = getCreateListOptionsMock(); const list = await createList(options); - const expected = getListResponseMock(); - expected.id = 'elastic-id-123'; + const expected: ListSchema = { ...getListResponseMock(), id: 'elastic-id-123' }; + expect(list).toEqual(expected); + }); + + test('it returns a list as expected with the id changed out for the elastic id and seralizer and deseralizer set', async () => { + const options: CreateListOptions = { + ...getCreateListOptionsMock(), + deserializer: '{{value}}', + serializer: '(?)', + }; + const list = await createList(options); + const expected: ListSchema = { + ...getListResponseMock(), + deserializer: '{{value}}', + id: 'elastic-id-123', + serializer: '(?)', + }; expect(list).toEqual(expected); }); @@ -44,8 +60,7 @@ describe('crete_list', () => { const options = getCreateListOptionsMock(); options.id = undefined; const list = await createList(options); - const expected = getListResponseMock(); - expected.id = 'elastic-id-123'; + const expected: ListSchema = { ...getListResponseMock(), id: 'elastic-id-123' }; expect(list).toEqual(expected); }); }); diff --git a/x-pack/plugins/lists/server/services/lists/create_list.ts b/x-pack/plugins/lists/server/services/lists/create_list.ts index 29ac5ebd96710..3925fa5f0170c 100644 --- a/x-pack/plugins/lists/server/services/lists/create_list.ts +++ b/x-pack/plugins/lists/server/services/lists/create_list.ts @@ -10,16 +10,20 @@ import { LegacyAPICaller } from 'kibana/server'; import { Description, + DeserializerOrUndefined, IdOrUndefined, IndexEsListSchema, ListSchema, MetaOrUndefined, Name, + SerializerOrUndefined, Type, } from '../../../common/schemas'; export interface CreateListOptions { id: IdOrUndefined; + deserializer: DeserializerOrUndefined; + serializer: SerializerOrUndefined; type: Type; name: Name; description: Description; @@ -33,6 +37,8 @@ export interface CreateListOptions { export const createList = async ({ id, + deserializer, + serializer, name, type, description, @@ -48,8 +54,10 @@ export const createList = async ({ created_at: createdAt, created_by: user, description, + deserializer, meta, name, + serializer, tie_breaker_id: tieBreaker ?? uuid.v4(), type, updated_at: createdAt, diff --git a/x-pack/plugins/lists/server/services/lists/list_client.test.ts b/x-pack/plugins/lists/server/services/lists/list_client.test.ts index 0c3a58283ffe2..0e756758a0c01 100644 --- a/x-pack/plugins/lists/server/services/lists/list_client.test.ts +++ b/x-pack/plugins/lists/server/services/lists/list_client.test.ts @@ -10,7 +10,7 @@ import { LIST_INDEX, LIST_ITEM_INDEX } from '../../../common/constants.mock'; import { getListClientMock } from './list_client.mock'; describe('list_client', () => { - describe('Mock client sanity checks', () => { + describe('Mock client sanity checks (not exhaustive tests against it)', () => { test('it returns the get list index as expected', () => { const mock = getListClientMock(); expect(mock.getListIndex()).toEqual(LIST_INDEX); diff --git a/x-pack/plugins/lists/server/services/lists/list_client.ts b/x-pack/plugins/lists/server/services/lists/list_client.ts index 662d082a08e4b..be9da1a1c69f5 100644 --- a/x-pack/plugins/lists/server/services/lists/list_client.ts +++ b/x-pack/plugins/lists/server/services/lists/list_client.ts @@ -108,6 +108,8 @@ export class ListClient { public createList = async ({ id, + deserializer, + serializer, name, description, type, @@ -115,11 +117,24 @@ export class ListClient { }: CreateListOptions): Promise => { const { callCluster, user } = this; const listIndex = this.getListIndex(); - return createList({ callCluster, description, id, listIndex, meta, name, type, user }); + return createList({ + callCluster, + description, + deserializer, + id, + listIndex, + meta, + name, + serializer, + type, + user, + }); }; public createListIfItDoesNotExist = async ({ id, + deserializer, + serializer, name, description, type, @@ -127,7 +142,7 @@ export class ListClient { }: CreateListIfItDoesNotExistOptions): Promise => { const list = await this.getList({ id }); if (list == null) { - return this.createList({ description, id, meta, name, type }); + return this.createList({ description, deserializer, id, meta, name, serializer, type }); } else { return list; } @@ -304,6 +319,8 @@ export class ListClient { }; public importListItemsToStream = async ({ + deserializer, + serializer, type, listId, stream, @@ -313,9 +330,11 @@ export class ListClient { const listItemIndex = this.getListItemIndex(); return importListItemsToStream({ callCluster, + deserializer, listId, listItemIndex, meta, + serializer, stream, type, user, @@ -340,19 +359,23 @@ export class ListClient { public createListItem = async ({ id, + deserializer, + serializer, listId, value, type, meta, - }: CreateListItemOptions): Promise => { + }: CreateListItemOptions): Promise => { const { callCluster, user } = this; const listItemIndex = this.getListItemIndex(); return createListItem({ callCluster, + deserializer, id, listId, listItemIndex, meta, + serializer, type, user, value, diff --git a/x-pack/plugins/lists/server/services/lists/list_client_types.ts b/x-pack/plugins/lists/server/services/lists/list_client_types.ts index 0c7a6a6e99a8e..26e147a6fa130 100644 --- a/x-pack/plugins/lists/server/services/lists/list_client_types.ts +++ b/x-pack/plugins/lists/server/services/lists/list_client_types.ts @@ -11,6 +11,7 @@ import { LegacyAPICaller } from 'kibana/server'; import { Description, DescriptionOrUndefined, + DeserializerOrUndefined, Filter, Id, IdOrUndefined, @@ -20,6 +21,7 @@ import { NameOrUndefined, Page, PerPage, + SerializerOrUndefined, SortFieldOrUndefined, SortOrderOrUndefined, Type, @@ -47,6 +49,8 @@ export interface DeleteListItemOptions { export interface CreateListOptions { id: IdOrUndefined; + deserializer: DeserializerOrUndefined; + serializer: SerializerOrUndefined; name: Name; description: Description; type: Type; @@ -55,6 +59,8 @@ export interface CreateListOptions { export interface CreateListIfItDoesNotExistOptions { id: Id; + deserializer: DeserializerOrUndefined; + serializer: SerializerOrUndefined; name: Name; description: Description; type: Type; @@ -80,6 +86,8 @@ export interface ExportListItemsToStreamOptions { } export interface ImportListItemsToStreamOptions { + deserializer: DeserializerOrUndefined; + serializer: SerializerOrUndefined; listId: string; type: Type; stream: Readable; @@ -88,6 +96,8 @@ export interface ImportListItemsToStreamOptions { export interface CreateListItemOptions { id: IdOrUndefined; + deserializer: DeserializerOrUndefined; + serializer: SerializerOrUndefined; listId: string; type: Type; value: string; diff --git a/x-pack/plugins/lists/server/services/lists/list_mappings.json b/x-pack/plugins/lists/server/services/lists/list_mappings.json index 1136a53da787d..da9cfec18719a 100644 --- a/x-pack/plugins/lists/server/services/lists/list_mappings.json +++ b/x-pack/plugins/lists/server/services/lists/list_mappings.json @@ -4,6 +4,12 @@ "name": { "type": "keyword" }, + "deserializer": { + "type": "keyword" + }, + "serializer": { + "type": "keyword" + }, "description": { "type": "keyword" }, diff --git a/x-pack/plugins/lists/server/services/lists/update_list.test.ts b/x-pack/plugins/lists/server/services/lists/update_list.test.ts index 1c4fde40a777a..211d58f6050ca 100644 --- a/x-pack/plugins/lists/server/services/lists/update_list.test.ts +++ b/x-pack/plugins/lists/server/services/lists/update_list.test.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import { ListSchema } from '../../../common/schemas'; import { getListResponseMock } from '../../../common/schemas/response/list_schema.mock'; import { updateList } from './update_list'; @@ -28,8 +29,25 @@ describe('update_list', () => { ((getList as unknown) as jest.Mock).mockResolvedValueOnce(list); const options = getUpdateListOptionsMock(); const updatedList = await updateList(options); - const expected = getListResponseMock(); - expected.id = 'elastic-id-123'; + const expected: ListSchema = { ...getListResponseMock(), id: 'elastic-id-123' }; + expect(updatedList).toEqual(expected); + }); + + test('it returns a list with serializer and deserializer', async () => { + const list: ListSchema = { + ...getListResponseMock(), + deserializer: '{{value}}', + serializer: '(?)', + }; + ((getList as unknown) as jest.Mock).mockResolvedValueOnce(list); + const options = getUpdateListOptionsMock(); + const updatedList = await updateList(options); + const expected: ListSchema = { + ...getListResponseMock(), + deserializer: '{{value}}', + id: 'elastic-id-123', + serializer: '(?)', + }; expect(updatedList).toEqual(expected); }); diff --git a/x-pack/plugins/lists/server/services/lists/update_list.ts b/x-pack/plugins/lists/server/services/lists/update_list.ts index 7d763013fb10e..c7cc30aaae908 100644 --- a/x-pack/plugins/lists/server/services/lists/update_list.ts +++ b/x-pack/plugins/lists/server/services/lists/update_list.ts @@ -60,9 +60,11 @@ export const updateList = async ({ created_at: list.created_at, created_by: list.created_by, description: description ?? list.description, + deserializer: list.deserializer, id: response._id, meta, name: name ?? list.name, + serializer: list.serializer, tie_breaker_id: list.tie_breaker_id, type: list.type, updated_at: updatedAt, diff --git a/x-pack/plugins/lists/server/services/utils/derive_type_from_es_type.test.ts b/x-pack/plugins/lists/server/services/utils/derive_type_from_es_type.test.ts deleted file mode 100644 index 8240e2965755e..0000000000000 --- a/x-pack/plugins/lists/server/services/utils/derive_type_from_es_type.test.ts +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { getSearchEsListItemMock } from '../../../common/schemas/elastic_response/search_es_list_item_schema.mock'; -import { Type } from '../../../common/schemas'; - -import { deriveTypeFromItem } from './derive_type_from_es_type'; - -describe('derive_type_from_es_type', () => { - beforeEach(() => { - jest.clearAllMocks(); - }); - - afterEach(() => { - jest.clearAllMocks(); - }); - - test('it returns the item ip if it exists', () => { - const item = getSearchEsListItemMock(); - const derivedType = deriveTypeFromItem({ item }); - const expected: Type = 'ip'; - expect(derivedType).toEqual(expected); - }); - - test('it returns the item keyword if it exists', () => { - const item = getSearchEsListItemMock(); - item.ip = undefined; - item.keyword = 'some keyword'; - const derivedType = deriveTypeFromItem({ item }); - const expected: Type = 'keyword'; - expect(derivedType).toEqual(expected); - }); - - test('it throws an error with status code if neither one exists', () => { - const item = getSearchEsListItemMock(); - item.ip = undefined; - item.keyword = undefined; - const expected = `Was expecting a valid type from the Elastic Search List Item such as ip or keyword but did not found one here ${JSON.stringify( - item - )}`; - expect(() => deriveTypeFromItem({ item })).toThrowError(expected); - }); -}); diff --git a/x-pack/plugins/lists/server/services/utils/derive_type_from_es_type.ts b/x-pack/plugins/lists/server/services/utils/derive_type_from_es_type.ts deleted file mode 100644 index 7a65e74bf4947..0000000000000 --- a/x-pack/plugins/lists/server/services/utils/derive_type_from_es_type.ts +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { SearchEsListItemSchema, Type } from '../../../common/schemas'; -import { ErrorWithStatusCode } from '../../error_with_status_code'; - -interface DeriveTypeFromItemOptions { - item: SearchEsListItemSchema; -} - -export const deriveTypeFromItem = ({ item }: DeriveTypeFromItemOptions): Type => { - if (item.ip != null) { - return 'ip'; - } else if (item.keyword != null) { - return 'keyword'; - } else { - throw new ErrorWithStatusCode( - `Was expecting a valid type from the Elastic Search List Item such as ip or keyword but did not found one here ${JSON.stringify( - item - )}`, - 400 - ); - } -}; diff --git a/x-pack/plugins/lists/server/services/utils/find_source_type.test.ts b/x-pack/plugins/lists/server/services/utils/find_source_type.test.ts new file mode 100644 index 0000000000000..87417cb3c1913 --- /dev/null +++ b/x-pack/plugins/lists/server/services/utils/find_source_type.test.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { getSearchEsListItemMock } from '../../../common/schemas/elastic_response/search_es_list_item_schema.mock'; +import { SearchEsListItemSchema, Type } from '../../../common/schemas'; + +import { findSourceType } from './find_source_type'; + +describe('find_source_type', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + test('it returns the item ip if it exists', () => { + const listItem = getSearchEsListItemMock(); + const derivedType = findSourceType(listItem); + const expected: Type = 'ip'; + expect(derivedType).toEqual(expected); + }); + + test('it returns the item keyword if it exists', () => { + const listItem: SearchEsListItemSchema = { + ...getSearchEsListItemMock(), + ip: undefined, + keyword: 'some keyword', + }; + const derivedType = findSourceType(listItem); + const expected: Type = 'keyword'; + expect(derivedType).toEqual(expected); + }); + + test('it returns a null if all the attached types are undefined', () => { + const item: SearchEsListItemSchema = { + ...getSearchEsListItemMock(), + ip: undefined, + keyword: undefined, + }; + expect(findSourceType(item)).toEqual(null); + }); +}); diff --git a/x-pack/plugins/lists/server/services/utils/find_source_type.ts b/x-pack/plugins/lists/server/services/utils/find_source_type.ts new file mode 100644 index 0000000000000..3d16ae30ca552 --- /dev/null +++ b/x-pack/plugins/lists/server/services/utils/find_source_type.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { SearchEsListItemSchema, Type, type } from '../../../common/schemas'; + +export const findSourceType = ( + listItem: SearchEsListItemSchema, + types: string[] = Object.keys(type.keys) +): Type | null => { + const foundEntry = Object.entries(listItem).find( + ([key, value]) => types.includes(key) && value != null + ); + if (foundEntry != null && type.is(foundEntry[0])) { + return foundEntry[0]; + } else { + return null; + } +}; diff --git a/x-pack/plugins/lists/server/services/utils/find_source_value.test.ts b/x-pack/plugins/lists/server/services/utils/find_source_value.test.ts new file mode 100644 index 0000000000000..8763b181403c9 --- /dev/null +++ b/x-pack/plugins/lists/server/services/utils/find_source_value.test.ts @@ -0,0 +1,367 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchEsListItemSchema } from '../../../common/schemas'; +import { getSearchEsListItemsAsAllUndefinedMock } from '../../../common/schemas/elastic_response/search_es_list_item_schema.mock'; + +import { + DEFAULT_DATE_RANGE, + DEFAULT_GEO_POINT, + DEFAULT_LTE_GTE, + DEFAULT_VALUE, + deserializeValue, + findSourceValue, +} from './find_source_value'; + +describe('find_source_value', () => { + describe('findSourceValue', () => { + test('it returns a binary type', () => { + const listItem: SearchEsListItemSchema = { + ...getSearchEsListItemsAsAllUndefinedMock(), + binary: 'U29tZSBiaW5hcnkgYmxvYg==', + }; + const value = findSourceValue(listItem); + expect(value).toEqual('U29tZSBiaW5hcnkgYmxvYg=='); + }); + + test('it returns a boolean type', () => { + const listItem: SearchEsListItemSchema = { + ...getSearchEsListItemsAsAllUndefinedMock(), + boolean: 'true', + }; + const value = findSourceValue(listItem); + expect(value).toEqual('true'); + }); + + test('it returns a byte type', () => { + const listItem: SearchEsListItemSchema = { + ...getSearchEsListItemsAsAllUndefinedMock(), + byte: '1', + }; + const value = findSourceValue(listItem); + expect(value).toEqual('1'); + }); + + test('it returns a date type', () => { + const listItem: SearchEsListItemSchema = { + ...getSearchEsListItemsAsAllUndefinedMock(), + date: '2020-07-01T23:10:19.505Z', + }; + const value = findSourceValue(listItem); + expect(value).toEqual('2020-07-01T23:10:19.505Z'); + }); + + test('it returns a date_nanos type', () => { + const listItem: SearchEsListItemSchema = { + ...getSearchEsListItemsAsAllUndefinedMock(), + date_nanos: '2015-01-01T12:10:30.123456789Z', + }; + const value = findSourceValue(listItem); + expect(value).toEqual('2015-01-01T12:10:30.123456789Z'); + }); + + test('it returns a date_range type', () => { + const listItem: SearchEsListItemSchema = { + ...getSearchEsListItemsAsAllUndefinedMock(), + date_range: { gte: '2020-07-01T23:10:19.505Z', lte: '2020-07-01T23:10:19.505Z' }, + }; + const value = findSourceValue(listItem); + expect(value).toEqual('2020-07-01T23:10:19.505Z,2020-07-01T23:10:19.505Z'); + }); + + test('it returns a double type', () => { + const listItem: SearchEsListItemSchema = { + ...getSearchEsListItemsAsAllUndefinedMock(), + double: '1.1', + }; + const value = findSourceValue(listItem); + expect(value).toEqual('1.1'); + }); + + test('it returns a double_range type', () => { + const listItem: SearchEsListItemSchema = { + ...getSearchEsListItemsAsAllUndefinedMock(), + double_range: { gte: '1.1', lte: '1.2' }, + }; + const value = findSourceValue(listItem); + expect(value).toEqual('1.1-1.2'); + }); + + test('it returns a float type', () => { + const listItem: SearchEsListItemSchema = { + ...getSearchEsListItemsAsAllUndefinedMock(), + float: '1.1', + }; + const value = findSourceValue(listItem); + expect(value).toEqual('1.1'); + }); + + test('it returns a float_range type', () => { + const listItem: SearchEsListItemSchema = { + ...getSearchEsListItemsAsAllUndefinedMock(), + float_range: { gte: '1.1', lte: '2.1' }, + }; + const value = findSourceValue(listItem); + expect(value).toEqual('1.1-2.1'); + }); + + test('it returns a geo_point type', () => { + const listItem: SearchEsListItemSchema = { + ...getSearchEsListItemsAsAllUndefinedMock(), + geo_point: 'POINT (-71.34 41.12)', + }; + const value = findSourceValue(listItem); + expect(value).toEqual('POINT (-71.34 41.12)'); + }); + + test('it returns a geo_point type which has a lat lon', () => { + const listItem: SearchEsListItemSchema = { + ...getSearchEsListItemsAsAllUndefinedMock(), + geo_point: { lat: '41', lon: '-71' }, + }; + const value = findSourceValue(listItem); + expect(value).toEqual('41,-71'); + }); + + test('it returns a geo_shape type', () => { + const listItem: SearchEsListItemSchema = { + ...getSearchEsListItemsAsAllUndefinedMock(), + geo_shape: 'POINT (-71.34 41.12)', + }; + const value = findSourceValue(listItem); + expect(value).toEqual('POINT (-71.34 41.12)'); + }); + + test('it returns a half_float type', () => { + const listItem: SearchEsListItemSchema = { + ...getSearchEsListItemsAsAllUndefinedMock(), + half_float: '1.2', + }; + const value = findSourceValue(listItem); + expect(value).toEqual('1.2'); + }); + + test('it returns a integer type', () => { + const listItem: SearchEsListItemSchema = { + ...getSearchEsListItemsAsAllUndefinedMock(), + integer: '1', + }; + const value = findSourceValue(listItem); + expect(value).toEqual('1'); + }); + + test('it returns a integer_range type', () => { + const listItem: SearchEsListItemSchema = { + ...getSearchEsListItemsAsAllUndefinedMock(), + integer: '1-2', + }; + const value = findSourceValue(listItem); + expect(value).toEqual('1-2'); + }); + + test('it returns a ip type', () => { + const listItem: SearchEsListItemSchema = { + ...getSearchEsListItemsAsAllUndefinedMock(), + ip: '127.0.0.1', + }; + const value = findSourceValue(listItem); + expect(value).toEqual('127.0.0.1'); + }); + + test('it returns a ip_range type', () => { + const listItem: SearchEsListItemSchema = { + ...getSearchEsListItemsAsAllUndefinedMock(), + ip_range: '127.0.0.1-127.0.0.2', + }; + const value = findSourceValue(listItem); + expect(value).toEqual('127.0.0.1-127.0.0.2'); + }); + + test('it returns a keyword type', () => { + const listItem: SearchEsListItemSchema = { + ...getSearchEsListItemsAsAllUndefinedMock(), + keyword: 'www.example.com', + }; + const value = findSourceValue(listItem); + expect(value).toEqual('www.example.com'); + }); + + test('it returns a long type', () => { + const listItem: SearchEsListItemSchema = { + ...getSearchEsListItemsAsAllUndefinedMock(), + long: '1', + }; + const value = findSourceValue(listItem); + expect(value).toEqual('1'); + }); + + test('it returns a long_range type', () => { + const listItem: SearchEsListItemSchema = { + ...getSearchEsListItemsAsAllUndefinedMock(), + long_range: { gte: '1', lte: '2' }, + }; + const value = findSourceValue(listItem); + expect(value).toEqual('1-2'); + }); + + test('it returns a shape type', () => { + const listItem: SearchEsListItemSchema = { + ...getSearchEsListItemsAsAllUndefinedMock(), + shape: 'POINT (-71.34 41.12)', + }; + const value = findSourceValue(listItem); + expect(value).toEqual('POINT (-71.34 41.12)'); + }); + + test('it returns a short type', () => { + const listItem: SearchEsListItemSchema = { + ...getSearchEsListItemsAsAllUndefinedMock(), + short: '1', + }; + const value = findSourceValue(listItem); + expect(value).toEqual('1'); + }); + + test('it returns a text type', () => { + const listItem: SearchEsListItemSchema = { + ...getSearchEsListItemsAsAllUndefinedMock(), + text: 'www.example.com', + }; + const value = findSourceValue(listItem); + expect(value).toEqual('www.example.com'); + }); + + test('it returns null if the type is not found because the type was never specified', () => { + const listItem: SearchEsListItemSchema = { + ...getSearchEsListItemsAsAllUndefinedMock(), + }; + const value = findSourceValue(listItem); + expect(value).toEqual(null); + }); + + test('it will custom deserialize a single value with a custom deserializer', () => { + const listItem: SearchEsListItemSchema = { + ...getSearchEsListItemsAsAllUndefinedMock(), + deserializer: 'custom value: {{value}}', + text: 'www.example.com', + }; + const value = findSourceValue(listItem); + expect(value).toEqual('custom value: www.example.com'); + }); + + test('it will custom deserialize a text with a custom deserializer', () => { + const listItem: SearchEsListItemSchema = { + ...getSearchEsListItemsAsAllUndefinedMock(), + deserializer: 'custom value: {{value}}', + text: 'www.example.com', + }; + const value = findSourceValue(listItem); + expect(value).toEqual('custom value: www.example.com'); + }); + + test('it will custom deserialize a date_range with a custom deserializer', () => { + const listItem: SearchEsListItemSchema = { + ...getSearchEsListItemsAsAllUndefinedMock(), + date_range: { gte: '2020-07-01T23:10:19.505Z', lte: '2020-07-01T23:10:19.505Z' }, + deserializer: 'custom value: {{gte}} {{lte}}', + }; + const value = findSourceValue(listItem); + expect(value).toEqual('custom value: 2020-07-01T23:10:19.505Z 2020-07-01T23:10:19.505Z'); + }); + + test('it will custom deserialize a ip_range with a custom deserializer using lte, gte', () => { + const listItem: SearchEsListItemSchema = { + ...getSearchEsListItemsAsAllUndefinedMock(), + deserializer: 'custom value: {{gte}} {{lte}}', + ip_range: { gte: '127.0.0.1', lte: '127.0.0.2' }, + }; + const value = findSourceValue(listItem); + expect(value).toEqual('custom value: 127.0.0.1 127.0.0.2'); + }); + }); + + describe('deserializeValue', () => { + test('it deserializes a regular value', () => { + const deserialized = deserializeValue({ + defaultDeserializer: DEFAULT_VALUE, + defaultValueDeserializer: DEFAULT_VALUE, + deserializer: undefined, + value: 'some value', + }); + expect(deserialized).toEqual('some value'); + }); + + test('it deserializes a value using the defaultValueDeserializer if its default is a gte, lte but we only provide a value', () => { + const deserialized = deserializeValue({ + defaultDeserializer: DEFAULT_LTE_GTE, + defaultValueDeserializer: DEFAULT_VALUE, + deserializer: undefined, + value: 'some value', + }); + expect(deserialized).toEqual('some value'); + }); + + test('it deserializes a lte, gte value if given one', () => { + const deserialized = deserializeValue({ + defaultDeserializer: DEFAULT_LTE_GTE, + defaultValueDeserializer: DEFAULT_VALUE, + deserializer: undefined, + value: { gte: '1', lte: '2' }, + }); + expect(deserialized).toEqual('1-2'); + }); + + test('it deserializes a lte, gte value if given a custom deserializer', () => { + const deserialized = deserializeValue({ + defaultDeserializer: DEFAULT_LTE_GTE, + defaultValueDeserializer: DEFAULT_VALUE, + deserializer: '{{{gte}}},{{{lte}}}', + value: { gte: '1', lte: '2' }, + }); + expect(deserialized).toEqual('1,2'); + }); + + test('it deserializes using the default if given a lte, get value but the deserializer does not include gte and lte', () => { + const deserialized = deserializeValue({ + defaultDeserializer: DEFAULT_LTE_GTE, + defaultValueDeserializer: DEFAULT_VALUE, + deserializer: '{{{value}}}', + value: { gte: '1', lte: '2' }, + }); + expect(deserialized).toEqual('1-2'); + }); + + test('it deserializes a lat, lon value if given one', () => { + const deserialized = deserializeValue({ + defaultDeserializer: DEFAULT_GEO_POINT, + defaultValueDeserializer: DEFAULT_VALUE, + deserializer: undefined, + value: { lat: '1', lon: '2' }, + }); + expect(deserialized).toEqual('1,2'); + }); + + test('it deserializes a lat, lon value if given a custom deserializer', () => { + const deserialized = deserializeValue({ + defaultDeserializer: DEFAULT_GEO_POINT, + defaultValueDeserializer: DEFAULT_VALUE, + deserializer: '{{{lat}}}-{{{lon}}}', + value: { lat: '1', lon: '2' }, + }); + expect(deserialized).toEqual('1-2'); + }); + + test('it deserializes a lte, gte value with a comma if given a date range deserializer', () => { + const deserialized = deserializeValue({ + defaultDeserializer: DEFAULT_DATE_RANGE, + defaultValueDeserializer: DEFAULT_VALUE, + deserializer: undefined, + value: { gte: '1', lte: '2' }, + }); + expect(deserialized).toEqual('1,2'); + }); + }); +}); diff --git a/x-pack/plugins/lists/server/services/utils/find_source_value.ts b/x-pack/plugins/lists/server/services/utils/find_source_value.ts new file mode 100644 index 0000000000000..8a52818c3d207 --- /dev/null +++ b/x-pack/plugins/lists/server/services/utils/find_source_value.ts @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import Mustache from 'mustache'; + +import { + DeserializerOrUndefined, + SearchEsListItemSchema, + esDataTypeGeoPointRange, + esDataTypeRange, + type, +} from '../../../common/schemas'; + +export const DEFAULT_GEO_POINT = '{{{lat}}},{{{lon}}}'; +export const DEFAULT_DATE_RANGE = '{{{gte}}},{{{lte}}}'; +export const DEFAULT_LTE_GTE = '{{{gte}}}-{{{lte}}}'; +export const DEFAULT_VALUE = '{{{value}}}'; + +export const findSourceValue = ( + listItem: SearchEsListItemSchema, + types: string[] = Object.keys(type.keys) +): string | null => { + const foundEntry = Object.entries(listItem).find( + ([key, value]) => types.includes(key) && value != null + ); + if (foundEntry != null) { + const [foundType, value] = foundEntry; + switch (foundType) { + case 'shape': + case 'geo_shape': + case 'geo_point': { + return deserializeValue({ + defaultDeserializer: DEFAULT_GEO_POINT, + defaultValueDeserializer: DEFAULT_VALUE, + deserializer: listItem.deserializer, + value, + }); + } + case 'double_range': + case 'float_range': + case 'integer_range': + case 'long_range': + case 'ip_range': { + return deserializeValue({ + defaultDeserializer: DEFAULT_LTE_GTE, + defaultValueDeserializer: DEFAULT_VALUE, + deserializer: listItem.deserializer, + value, + }); + } + case 'date_range': { + return deserializeValue({ + defaultDeserializer: DEFAULT_DATE_RANGE, + defaultValueDeserializer: DEFAULT_VALUE, + deserializer: listItem.deserializer, + value, + }); + } + default: { + return deserializeValue({ + defaultDeserializer: DEFAULT_VALUE, + defaultValueDeserializer: DEFAULT_VALUE, + deserializer: listItem.deserializer, + value, + }); + } + } + } else { + return null; + } +}; + +export const deserializeValue = ({ + deserializer, + defaultValueDeserializer, + defaultDeserializer, + value, +}: { + deserializer: DeserializerOrUndefined; + defaultValueDeserializer: string; + defaultDeserializer: string; + value: string | object | undefined; +}): string | null => { + if (esDataTypeRange.is(value)) { + const template = + deserializer?.includes('gte') && deserializer?.includes('lte') + ? deserializer + : defaultDeserializer; + const variables = { gte: value.gte, lte: value.lte }; + return Mustache.render(template, variables); + } else if (esDataTypeGeoPointRange.is(value)) { + const template = + deserializer?.includes('lat') && deserializer?.includes('lon') + ? deserializer + : defaultDeserializer; + const variables = { lat: value.lat, lon: value.lon }; + return Mustache.render(template, variables); + } else if (typeof value === 'string') { + const template = deserializer?.includes('value') ? deserializer : defaultValueDeserializer; + const variables = { value }; + return Mustache.render(template, variables); + } else { + return null; + } +}; diff --git a/x-pack/plugins/lists/server/services/utils/get_query_filter.test.ts b/x-pack/plugins/lists/server/services/utils/get_query_filter.test.ts index 50c266eb5d573..27221398d4064 100644 --- a/x-pack/plugins/lists/server/services/utils/get_query_filter.test.ts +++ b/x-pack/plugins/lists/server/services/utils/get_query_filter.test.ts @@ -4,31 +4,103 @@ * you may not use this file except in compliance with the Elastic License. */ -import { getQueryFilter } from './get_query_filter'; +import { getQueryFilter, getQueryFilterWithListId } from './get_query_filter'; describe('get_query_filter', () => { - test('it should work with a basic kuery', () => { - const esQuery = getQueryFilter({ filter: 'type: ip' }); - expect(esQuery).toEqual({ - bool: { - filter: [ - { - bool: { - minimum_should_match: 1, - should: [ - { - match: { - type: 'ip', + describe('getQueryFilter', () => { + test('it should work with a basic kuery', () => { + const esQuery = getQueryFilter({ filter: 'type: ip' }); + expect(esQuery).toEqual({ + bool: { + filter: [ + { + bool: { + minimum_should_match: 1, + should: [ + { + match: { + type: 'ip', + }, }, - }, - ], + ], + }, }, - }, - ], - must: [], - must_not: [], - should: [], - }, + ], + must: [], + must_not: [], + should: [], + }, + }); + }); + }); + + describe('getQueryFilterWithListId', () => { + test('it returns a basic kuery with the list id added and an empty filter', () => { + const esQuery = getQueryFilterWithListId({ filter: '', listId: 'list-123' }); + expect(esQuery).toEqual({ + bool: { + filter: [ + { + bool: { + minimum_should_match: 1, + should: [ + { + match: { + list_id: 'list-123', + }, + }, + ], + }, + }, + ], + must: [], + must_not: [], + should: [], + }, + }); + }); + + test('it returns a basic kuery with the list id added and a filter', () => { + const esQuery = getQueryFilterWithListId({ filter: 'type: ip', listId: 'list-123' }); + expect(esQuery).toEqual({ + bool: { + filter: [ + { + bool: { + filter: [ + { + bool: { + minimum_should_match: 1, + should: [ + { + match: { + list_id: 'list-123', + }, + }, + ], + }, + }, + { + bool: { + minimum_should_match: 1, + should: [ + { + match: { + type: 'ip', + }, + }, + ], + }, + }, + ], + }, + }, + ], + must: [], + must_not: [], + should: [], + }, + }); }); }); }); diff --git a/x-pack/plugins/lists/server/services/utils/get_query_filter.ts b/x-pack/plugins/lists/server/services/utils/get_query_filter.ts index cf0dd5b6250e5..9d4af7761728c 100644 --- a/x-pack/plugins/lists/server/services/utils/get_query_filter.ts +++ b/x-pack/plugins/lists/server/services/utils/get_query_filter.ts @@ -12,6 +12,11 @@ export interface GetQueryFilterOptions { filter: string; } +export interface GetQueryFilterWithListIdOptions { + filter: string; + listId: string; +} + export interface GetQueryFilterReturn { bool: { must: DslQuery[]; filter: Filter[]; should: never[]; must_not: Filter[] }; } @@ -30,3 +35,12 @@ export const getQueryFilter = ({ filter }: GetQueryFilterOptions): GetQueryFilte return esQuery.buildEsQuery(undefined, kqlQuery, [], config); }; + +export const getQueryFilterWithListId = ({ + filter, + listId, +}: GetQueryFilterWithListIdOptions): GetQueryFilterReturn => { + const filterWithListId = + filter.trim() !== '' ? `list_id: ${listId} AND (${filter})` : `list_id: ${listId}`; + return getQueryFilter({ filter: filterWithListId }); +}; diff --git a/x-pack/plugins/lists/server/services/utils/get_query_filter_from_type_value.ts b/x-pack/plugins/lists/server/services/utils/get_query_filter_from_type_value.ts index 3f50efe0c6c56..3baba07aa9885 100644 --- a/x-pack/plugins/lists/server/services/utils/get_query_filter_from_type_value.ts +++ b/x-pack/plugins/lists/server/services/utils/get_query_filter_from_type_value.ts @@ -6,9 +6,10 @@ import { Type } from '../../../common/schemas'; -export type QueryFilterType = Array< - { term: { list_id: string } } | { terms: { ip: string[] } } | { terms: { keyword: string[] } } ->; +export type QueryFilterType = [ + { term: Record }, + { terms: Record } +]; export const getQueryFilterFromTypeValue = ({ type, @@ -18,16 +19,4 @@ export const getQueryFilterFromTypeValue = ({ type: Type; value: string[]; listId: string; - // We disable the consistent return since we want to use typescript for exhaustive type checks - // eslint-disable-next-line consistent-return -}): QueryFilterType => { - const filter: QueryFilterType = [{ term: { list_id: listId } }]; - switch (type) { - case 'ip': { - return [...filter, ...[{ terms: { ip: value } }]]; - } - case 'keyword': { - return [...filter, ...[{ terms: { keyword: value } }]]; - } - } -}; +}): QueryFilterType => [{ term: { list_id: listId } }, { terms: { [type]: value } }]; diff --git a/x-pack/plugins/lists/server/services/utils/index.ts b/x-pack/plugins/lists/server/services/utils/index.ts index 28bb3cea29e61..f7ed118ea5857 100644 --- a/x-pack/plugins/lists/server/services/utils/index.ts +++ b/x-pack/plugins/lists/server/services/utils/index.ts @@ -4,15 +4,17 @@ * you may not use this file except in compliance with the Elastic License. */ -export * from './derive_type_from_es_type'; +export * from './calculate_scroll_math'; export * from './encode_decode_cursor'; -export * from './get_query_filter_from_type_value'; +export * from './find_source_type'; +export * from './find_source_value'; export * from './get_query_filter'; +export * from './get_query_filter_from_type_value'; export * from './get_search_after_scroll'; export * from './get_search_after_with_tie_breaker'; export * from './get_sort_with_tie_breaker'; export * from './get_source_with_tie_breaker'; export * from './scroll_to_start_page'; -export * from './transform_elastic_to_list_item'; export * from './transform_elastic_to_list'; +export * from './transform_elastic_to_list_item'; export * from './transform_list_item_to_elastic_query'; diff --git a/x-pack/plugins/lists/server/services/utils/transform_elastic_to_list_item.test.ts b/x-pack/plugins/lists/server/services/utils/transform_elastic_to_list_item.test.ts index 8b32f09400719..8a5554c3865c5 100644 --- a/x-pack/plugins/lists/server/services/utils/transform_elastic_to_list_item.test.ts +++ b/x-pack/plugins/lists/server/services/utils/transform_elastic_to_list_item.test.ts @@ -43,28 +43,4 @@ describe('transform_elastic_to_list_item', () => { const expected: ListItemArraySchema = [listItemResponse]; expect(queryFilter).toEqual(expected); }); - - test('it does a throw if it cannot determine the list item type from "ip"', () => { - const response = getSearchListItemMock(); - response.hits.hits[0]._source.ip = undefined; - response.hits.hits[0]._source.keyword = 'host-name-example'; - expect(() => - transformElasticToListItem({ - response, - type: 'ip', - }) - ).toThrow('Was expecting ip to not be null/undefined'); - }); - - test('it does a throw if it cannot determine the list item type from "keyword"', () => { - const response = getSearchListItemMock(); - response.hits.hits[0]._source.ip = '127.0.0.1'; - response.hits.hits[0]._source.keyword = undefined; - expect(() => - transformElasticToListItem({ - response, - type: 'keyword', - }) - ).toThrow('Was expecting keyword to not be null/undefined'); - }); }); diff --git a/x-pack/plugins/lists/server/services/utils/transform_elastic_to_list_item.ts b/x-pack/plugins/lists/server/services/utils/transform_elastic_to_list_item.ts index f150a29c524a5..a59b3b383cd2a 100644 --- a/x-pack/plugins/lists/server/services/utils/transform_elastic_to_list_item.ts +++ b/x-pack/plugins/lists/server/services/utils/transform_elastic_to_list_item.ts @@ -9,6 +9,8 @@ import { SearchResponse } from 'elasticsearch'; import { ListItemArraySchema, SearchEsListItemSchema, Type } from '../../../common/schemas'; import { ErrorWithStatusCode } from '../../error_with_status_code'; +import { findSourceValue } from './find_source_value'; + export interface TransformElasticToListItemOptions { response: SearchResponse; type: Type; @@ -23,53 +25,34 @@ export const transformElasticToListItem = ({ _id, _source: { created_at, + deserializer, + serializer, updated_at, updated_by, created_by, list_id, tie_breaker_id, - ip, - keyword, meta, }, } = hit; - - const baseTypes = { - created_at, - created_by, - id: _id, - list_id, - meta, - tie_breaker_id, - type, - updated_at, - updated_by, - }; - - switch (type) { - case 'ip': { - if (ip == null) { - throw new ErrorWithStatusCode('Was expecting ip to not be null/undefined', 400); - } - return { - ...baseTypes, - value: ip, - }; - } - case 'keyword': { - if (keyword == null) { - throw new ErrorWithStatusCode('Was expecting keyword to not be null/undefined', 400); - } - return { - ...baseTypes, - value: keyword, - }; - } + const value = findSourceValue(hit._source); + if (value == null) { + throw new ErrorWithStatusCode(`Was expected ${type} to not be null/undefined`, 400); + } else { + return { + created_at, + created_by, + deserializer, + id: _id, + list_id, + meta, + serializer, + tie_breaker_id, + type, + updated_at, + updated_by, + value, + }; } - return assertUnreachable(); }); }; - -const assertUnreachable = (): never => { - throw new Error('Unknown type in elastic_to_list_items'); -}; diff --git a/x-pack/plugins/lists/server/services/utils/transform_list_item_to_elastic_query.test.ts b/x-pack/plugins/lists/server/services/utils/transform_list_item_to_elastic_query.test.ts index 217cad30bfdbb..0a3860d033f64 100644 --- a/x-pack/plugins/lists/server/services/utils/transform_list_item_to_elastic_query.test.ts +++ b/x-pack/plugins/lists/server/services/utils/transform_list_item_to_elastic_query.test.ts @@ -4,9 +4,20 @@ * you may not use this file except in compliance with the Elastic License. */ -import { EsDataTypeUnion, Type } from '../../../common/schemas'; +import { EsDataTypeUnion } from '../../../common/schemas'; -import { transformListItemToElasticQuery } from './transform_list_item_to_elastic_query'; +import { + DEFAULT_DATE_REGEX, + DEFAULT_GEO_REGEX, + DEFAULT_LTE_GTE_REGEX, + DEFAULT_SINGLE_REGEX, + serializeGeoPoint, + serializeGeoShape, + serializeIpRange, + serializeRanges, + serializeSingleValue, + transformListItemToElasticQuery, +} from './transform_list_item_to_elastic_query'; describe('transform_elastic_to_elastic_query', () => { beforeEach(() => { @@ -17,31 +28,567 @@ describe('transform_elastic_to_elastic_query', () => { jest.clearAllMocks(); }); - test('it transforms a ip type and value to a union', () => { - const elasticQuery = transformListItemToElasticQuery({ - type: 'ip', - value: '127.0.0.1', + describe('transformListItemToElasticQuery', () => { + test('it transforms a shape to a union when it is a WKT', () => { + const elasticQuery = transformListItemToElasticQuery({ + serializer: undefined, + type: 'shape', + value: 'POINT (-77.03653 38.897676)', + }); + const expected: EsDataTypeUnion = { shape: 'POINT (-77.03653 38.897676)' }; + expect(elasticQuery).toEqual(expected); + }); + + test('it transforms a shape to a union when it is a lat,lon', () => { + const elasticQuery = transformListItemToElasticQuery({ + serializer: undefined, + type: 'shape', + value: '38.897676,-77.03653', + }); + const expected: EsDataTypeUnion = { shape: 'POINT (-77.03653 38.897676)' }; + expect(elasticQuery).toEqual(expected); + }); + + test('it transforms a geo_shape to a union when it is a WKT', () => { + const elasticQuery = transformListItemToElasticQuery({ + serializer: undefined, + type: 'geo_shape', + value: 'POINT (-77.03653 38.897676)', + }); + const expected: EsDataTypeUnion = { geo_shape: 'POINT (-77.03653 38.897676)' }; + expect(elasticQuery).toEqual(expected); + }); + + test('it transforms a geo_shape to a union when it is a lat,lon', () => { + const elasticQuery = transformListItemToElasticQuery({ + serializer: undefined, + type: 'geo_shape', + value: '38.897676,-77.03653', + }); + const expected: EsDataTypeUnion = { geo_shape: 'POINT (-77.03653 38.897676)' }; + expect(elasticQuery).toEqual(expected); + }); + + test('it transforms a geo_point to a union when it is a WKT', () => { + const elasticQuery = transformListItemToElasticQuery({ + serializer: undefined, + type: 'geo_point', + value: 'POINT (-77.03653 38.897676)', + }); + const expected: EsDataTypeUnion = { geo_point: 'POINT (-77.03653 38.897676)' }; + expect(elasticQuery).toEqual(expected); + }); + + test('it transforms a geo_point to a union when it is a lat,lon', () => { + const elasticQuery = transformListItemToElasticQuery({ + serializer: undefined, + type: 'geo_point', + value: '38.897676, -77.03653', + }); + const expected: EsDataTypeUnion = { geo_point: { lat: '38.897676', lon: '-77.03653' } }; + expect(elasticQuery).toEqual(expected); + }); + + test('it transforms a ip_range to a union', () => { + const elasticQuery = transformListItemToElasticQuery({ + serializer: undefined, + type: 'ip_range', + value: '127.0.0.1-127.0.0.2', + }); + const expected: EsDataTypeUnion = { + ip_range: { gte: '127.0.0.1', lte: '127.0.0.2' }, + }; + expect(elasticQuery).toEqual(expected); + }); + + test('it transforms a ip CIDR to a union', () => { + const elasticQuery = transformListItemToElasticQuery({ + serializer: undefined, + type: 'ip_range', + value: '127.0.0.1/16', + }); + const expected: EsDataTypeUnion = { + ip_range: '127.0.0.1/16', + }; + expect(elasticQuery).toEqual(expected); + }); + + test('it transforms a ip_range to a union even if only a single value is found', () => { + const elasticQuery = transformListItemToElasticQuery({ + serializer: undefined, + type: 'ip_range', + value: '127.0.0.1', + }); + const expected: EsDataTypeUnion = { + ip_range: { gte: '127.0.0.1', lte: '127.0.0.1' }, + }; + expect(elasticQuery).toEqual(expected); + }); + + test('it transforms a ip_range to a union using a custom serializer', () => { + const elasticQuery = transformListItemToElasticQuery({ + serializer: '(?.+),(?.+)|(?.+)', + type: 'ip_range', + value: '127.0.0.1,127.0.0.2', + }); + const expected: EsDataTypeUnion = { + ip_range: { gte: '127.0.0.1', lte: '127.0.0.2' }, + }; + expect(elasticQuery).toEqual(expected); + }); + + test('it transforms a date_range to a union', () => { + const elasticQuery = transformListItemToElasticQuery({ + serializer: undefined, + type: 'date_range', + value: '2020-06-02T06:19:51.434Z,2020-07-02T06:19:51.434Z', + }); + const expected: EsDataTypeUnion = { + date_range: { gte: '2020-06-02T06:19:51.434Z', lte: '2020-07-02T06:19:51.434Z' }, + }; + expect(elasticQuery).toEqual(expected); + }); + + test('it transforms a date_range to a union even if only one date is found', () => { + const elasticQuery = transformListItemToElasticQuery({ + serializer: undefined, + type: 'date_range', + value: '2020-06-02T06:19:51.434Z', + }); + const expected: EsDataTypeUnion = { + date_range: { gte: '2020-06-02T06:19:51.434Z', lte: '2020-06-02T06:19:51.434Z' }, + }; + expect(elasticQuery).toEqual(expected); + }); + + test('it transforms a double_range to a union', () => { + const elasticQuery = transformListItemToElasticQuery({ + serializer: undefined, + type: 'double_range', + value: '1.1-1.2', + }); + const expected: EsDataTypeUnion = { + double_range: { gte: '1.1', lte: '1.2' }, + }; + expect(elasticQuery).toEqual(expected); + }); + + test('it transforms a float_range to a union', () => { + const elasticQuery = transformListItemToElasticQuery({ + serializer: undefined, + type: 'float_range', + value: '1.1-1.2', + }); + const expected: EsDataTypeUnion = { + float_range: { gte: '1.1', lte: '1.2' }, + }; + expect(elasticQuery).toEqual(expected); + }); + + test('it transforms a integer_range to a union', () => { + const elasticQuery = transformListItemToElasticQuery({ + serializer: undefined, + type: 'integer_range', + value: '1.1-1.2', + }); + const expected: EsDataTypeUnion = { + integer_range: { gte: '1.1', lte: '1.2' }, + }; + expect(elasticQuery).toEqual(expected); + }); + + test('it transforms a integer_range to a union even if only one is found', () => { + const elasticQuery = transformListItemToElasticQuery({ + serializer: undefined, + type: 'integer_range', + value: '1.1', + }); + const expected: EsDataTypeUnion = { + integer_range: { gte: '1.1', lte: '1.1' }, + }; + expect(elasticQuery).toEqual(expected); + }); + + test('it transforms a long_range to a union', () => { + const elasticQuery = transformListItemToElasticQuery({ + serializer: undefined, + type: 'long_range', + value: '1.1-1.2', + }); + const expected: EsDataTypeUnion = { + long_range: { gte: '1.1', lte: '1.2' }, + }; + expect(elasticQuery).toEqual(expected); + }); + + test('it transforms a ip type and value to a union', () => { + const elasticQuery = transformListItemToElasticQuery({ + serializer: undefined, + type: 'ip', + value: '127.0.0.1', + }); + const expected: EsDataTypeUnion = { ip: '127.0.0.1' }; + expect(elasticQuery).toEqual(expected); + }); + + test('it transforms a keyword type and value to a union', () => { + const elasticQuery = transformListItemToElasticQuery({ + serializer: undefined, + type: 'keyword', + value: 'host-name', + }); + const expected: EsDataTypeUnion = { keyword: 'host-name' }; + expect(elasticQuery).toEqual(expected); + }); + + test('it transforms a text type and value to a union', () => { + const elasticQuery = transformListItemToElasticQuery({ + serializer: undefined, + type: 'text', + value: 'host-name', + }); + const expected: EsDataTypeUnion = { text: 'host-name' }; + expect(elasticQuery).toEqual(expected); + }); + + test('it transforms a binary type and value to a union', () => { + const elasticQuery = transformListItemToElasticQuery({ + serializer: undefined, + type: 'binary', + value: 'U29tZSBiaW5hcnkgYmxvYg==', + }); + const expected: EsDataTypeUnion = { binary: 'U29tZSBiaW5hcnkgYmxvYg==' }; + expect(elasticQuery).toEqual(expected); + }); + + test('it transforms a boolean type and value to a union', () => { + const elasticQuery = transformListItemToElasticQuery({ + serializer: undefined, + type: 'boolean', + value: 'true', + }); + const expected: EsDataTypeUnion = { boolean: 'true' }; + expect(elasticQuery).toEqual(expected); + }); + + test('it transforms a byte type and value to a union', () => { + const elasticQuery = transformListItemToElasticQuery({ + serializer: undefined, + type: 'byte', + value: '1', + }); + const expected: EsDataTypeUnion = { byte: '1' }; + expect(elasticQuery).toEqual(expected); + }); + + test('it transforms a date type and value to a union', () => { + const elasticQuery = transformListItemToElasticQuery({ + serializer: undefined, + type: 'date', + value: '2020-07-02T06:19:51.434Z', + }); + const expected: EsDataTypeUnion = { date: '2020-07-02T06:19:51.434Z' }; + expect(elasticQuery).toEqual(expected); + }); + + test('it transforms a date_nanos type and value to a union', () => { + const elasticQuery = transformListItemToElasticQuery({ + serializer: undefined, + type: 'date_nanos', + value: '2015-01-01T12:10:30.123456789Z', + }); + const expected: EsDataTypeUnion = { date_nanos: '2015-01-01T12:10:30.123456789Z' }; + expect(elasticQuery).toEqual(expected); + }); + + test('it transforms a double type and value to a union', () => { + const elasticQuery = transformListItemToElasticQuery({ + serializer: undefined, + type: 'double', + value: '1.1', + }); + const expected: EsDataTypeUnion = { double: '1.1' }; + expect(elasticQuery).toEqual(expected); + }); + + test('it transforms a float type and value to a union', () => { + const elasticQuery = transformListItemToElasticQuery({ + serializer: undefined, + type: 'float', + value: '1.1', + }); + const expected: EsDataTypeUnion = { float: '1.1' }; + expect(elasticQuery).toEqual(expected); + }); + + test('it transforms a integer type and value to a union', () => { + const elasticQuery = transformListItemToElasticQuery({ + serializer: undefined, + type: 'integer', + value: '1', + }); + const expected: EsDataTypeUnion = { integer: '1' }; + expect(elasticQuery).toEqual(expected); + }); + + test('it transforms a long type and value to a union', () => { + const elasticQuery = transformListItemToElasticQuery({ + serializer: undefined, + type: 'long', + value: '1', + }); + const expected: EsDataTypeUnion = { long: '1' }; + expect(elasticQuery).toEqual(expected); }); - const expected: EsDataTypeUnion = { ip: '127.0.0.1' }; - expect(elasticQuery).toEqual(expected); }); - test('it transforms a keyword type and value to a union', () => { - const elasticQuery = transformListItemToElasticQuery({ - type: 'keyword', - value: 'host-name', + describe('serializeGeoShape', () => { + test('it transforms a shape to a union when it is a WKT', () => { + const elasticQuery = serializeGeoShape({ + defaultSerializer: DEFAULT_GEO_REGEX, + serializer: undefined, + type: 'shape', + value: 'POINT (-77.03653 38.897676)', + }); + const expected: EsDataTypeUnion = { shape: 'POINT (-77.03653 38.897676)' }; + expect(elasticQuery).toEqual(expected); + }); + + test('it trims extra spaces', () => { + const elasticQuery = serializeGeoShape({ + defaultSerializer: DEFAULT_GEO_REGEX, + serializer: undefined, + type: 'shape', + value: ' POINT (-77.03653 38.897676) ', + }); + const expected: EsDataTypeUnion = { shape: 'POINT (-77.03653 38.897676)' }; + expect(elasticQuery).toEqual(expected); + }); + + test('it transforms a shape to a union when it is a lat,lon', () => { + const elasticQuery = serializeGeoShape({ + defaultSerializer: DEFAULT_GEO_REGEX, + serializer: undefined, + type: 'shape', + value: '38.897676,-77.03653', + }); + const expected: EsDataTypeUnion = { shape: 'POINT (-77.03653 38.897676)' }; + expect(elasticQuery).toEqual(expected); + }); + + test('it transforms a shape to a union when it is a lat,lon with a custom serializer', () => { + const elasticQuery = serializeGeoShape({ + defaultSerializer: DEFAULT_GEO_REGEX, + serializer: '(?.+)/(?.+)', + type: 'shape', + value: '38.897676/-77.03653', + }); + const expected: EsDataTypeUnion = { shape: 'POINT (-77.03653 38.897676)' }; + expect(elasticQuery).toEqual(expected); }); - const expected: EsDataTypeUnion = { keyword: 'host-name' }; - expect(elasticQuery).toEqual(expected); }); - test('it throws if the type is not known', () => { - const type: Type = 'made-up' as Type; - expect(() => - transformListItemToElasticQuery({ - type, - value: 'some-value', - }) - ).toThrow('Unknown type: "made-up" in transformListItemToElasticQuery'); + describe('serializeGeoPoint', () => { + test('it transforms a geo_point to a union when it is a WKT', () => { + const elasticQuery = serializeGeoPoint({ + defaultSerializer: DEFAULT_GEO_REGEX, + serializer: undefined, + value: 'POINT (-77.03653 38.897676)', + }); + const expected: EsDataTypeUnion = { geo_point: 'POINT (-77.03653 38.897676)' }; + expect(elasticQuery).toEqual(expected); + }); + + test('it trims extra spaces', () => { + const elasticQuery = serializeGeoPoint({ + defaultSerializer: DEFAULT_GEO_REGEX, + serializer: undefined, + value: ' POINT (-77.03653 38.897676) ', + }); + const expected: EsDataTypeUnion = { geo_point: 'POINT (-77.03653 38.897676)' }; + expect(elasticQuery).toEqual(expected); + }); + + test('it transforms a geo_point to a union when it is a lat,lon', () => { + const elasticQuery = serializeGeoPoint({ + defaultSerializer: DEFAULT_GEO_REGEX, + serializer: undefined, + value: '38.897676, -77.03653', + }); + const expected: EsDataTypeUnion = { geo_point: { lat: '38.897676', lon: '-77.03653' } }; + expect(elasticQuery).toEqual(expected); + }); + + test('it transforms a geo_point to a union when it is a lat,lon with a custom serializer', () => { + const elasticQuery = serializeGeoPoint({ + defaultSerializer: DEFAULT_GEO_REGEX, + serializer: '(?.+)/(?.+)', + value: '38.897676/-77.03653', + }); + const expected: EsDataTypeUnion = { geo_point: { lat: '38.897676', lon: '-77.03653' } }; + expect(elasticQuery).toEqual(expected); + }); + }); + + describe('serializeIpRange', () => { + test('it transforms a ip_range to a union', () => { + const elasticQuery = serializeIpRange({ + defaultSerializer: DEFAULT_LTE_GTE_REGEX, + serializer: undefined, + value: '127.0.0.1-127.0.0.2', + }); + const expected: EsDataTypeUnion = { + ip_range: { gte: '127.0.0.1', lte: '127.0.0.2' }, + }; + expect(elasticQuery).toEqual(expected); + }); + + test('it transforms a ip CIDR to a union', () => { + const elasticQuery = serializeIpRange({ + defaultSerializer: DEFAULT_LTE_GTE_REGEX, + serializer: undefined, + value: '127.0.0.1/16', + }); + const expected: EsDataTypeUnion = { + ip_range: '127.0.0.1/16', + }; + expect(elasticQuery).toEqual(expected); + }); + + test('it trims extras spaces', () => { + const elasticQuery = serializeIpRange({ + defaultSerializer: DEFAULT_LTE_GTE_REGEX, + serializer: undefined, + value: ' 127.0.0.1 - 127.0.0.2 ', + }); + const expected: EsDataTypeUnion = { + ip_range: { gte: '127.0.0.1', lte: '127.0.0.2' }, + }; + expect(elasticQuery).toEqual(expected); + }); + + test('it transforms a ip_range to a union even if only a single value is found', () => { + const elasticQuery = serializeIpRange({ + defaultSerializer: DEFAULT_LTE_GTE_REGEX, + serializer: undefined, + value: '127.0.0.1', + }); + const expected: EsDataTypeUnion = { + ip_range: { gte: '127.0.0.1', lte: '127.0.0.1' }, + }; + expect(elasticQuery).toEqual(expected); + }); + + test('it transforms a ip_range to a union using a custom serializer', () => { + const elasticQuery = serializeIpRange({ + defaultSerializer: DEFAULT_LTE_GTE_REGEX, + serializer: '(?.+),(?.+)|(?.+)', + value: '127.0.0.1,127.0.0.2', + }); + const expected: EsDataTypeUnion = { + ip_range: { gte: '127.0.0.1', lte: '127.0.0.2' }, + }; + expect(elasticQuery).toEqual(expected); + }); + }); + + describe('serializeRanges', () => { + test('it transforms a date_range to a union', () => { + const elasticQuery = serializeRanges({ + defaultSerializer: DEFAULT_DATE_REGEX, + serializer: undefined, + type: 'date_range', + value: '2020-06-02T06:19:51.434Z,2020-07-02T06:19:51.434Z', + }); + const expected: EsDataTypeUnion = { + date_range: { gte: '2020-06-02T06:19:51.434Z', lte: '2020-07-02T06:19:51.434Z' }, + }; + expect(elasticQuery).toEqual(expected); + }); + + test('it trims extra spaces', () => { + const elasticQuery = serializeRanges({ + defaultSerializer: DEFAULT_DATE_REGEX, + serializer: undefined, + type: 'date_range', + value: ' 2020-06-02T06:19:51.434Z , 2020-07-02T06:19:51.434Z ', + }); + const expected: EsDataTypeUnion = { + date_range: { gte: '2020-06-02T06:19:51.434Z', lte: '2020-07-02T06:19:51.434Z' }, + }; + expect(elasticQuery).toEqual(expected); + }); + + test('it transforms a date_range to a union even if only one date is found', () => { + const elasticQuery = serializeRanges({ + defaultSerializer: DEFAULT_DATE_REGEX, + serializer: undefined, + type: 'date_range', + value: '2020-06-02T06:19:51.434Z', + }); + const expected: EsDataTypeUnion = { + date_range: { gte: '2020-06-02T06:19:51.434Z', lte: '2020-06-02T06:19:51.434Z' }, + }; + expect(elasticQuery).toEqual(expected); + }); + + test('it transforms a long_range to a union with a custom serializer', () => { + const elasticQuery = serializeRanges({ + defaultSerializer: DEFAULT_DATE_REGEX, + serializer: '(?.+)/(?.+)|(?.+)', + type: 'long_range', + value: '1/2', + }); + const expected: EsDataTypeUnion = { + long_range: { gte: '1', lte: '2' }, + }; + expect(elasticQuery).toEqual(expected); + }); + }); + + describe('serializeSingleValue', () => { + test('it transforms a ip type and value to a union', () => { + const elasticQuery = serializeSingleValue({ + defaultSerializer: DEFAULT_SINGLE_REGEX, + serializer: undefined, + type: 'ip', + value: '127.0.0.1', + }); + const expected: EsDataTypeUnion = { ip: '127.0.0.1' }; + expect(elasticQuery).toEqual(expected); + }); + + test('it trims extra spaces', () => { + const elasticQuery = serializeSingleValue({ + defaultSerializer: DEFAULT_SINGLE_REGEX, + serializer: undefined, + type: 'ip', + value: ' 127.0.0.1 ', + }); + const expected: EsDataTypeUnion = { ip: '127.0.0.1' }; + expect(elasticQuery).toEqual(expected); + }); + + test('it transforms a ip type and value to a union with a custom serializer', () => { + const elasticQuery = serializeSingleValue({ + defaultSerializer: DEFAULT_SINGLE_REGEX, + serializer: 'junk-(?.+)', + type: 'ip', + value: 'junk-127.0.0.1', + }); + const expected: EsDataTypeUnion = { ip: '127.0.0.1' }; + expect(elasticQuery).toEqual(expected); + }); + + test('it returns the value as is if it does not match the custom serializer', () => { + const elasticQuery = serializeSingleValue({ + defaultSerializer: DEFAULT_SINGLE_REGEX, + serializer: 'junk-(?garbage)', + type: 'ip', + value: 'junk-127.0.0.1', + }); + const expected: EsDataTypeUnion = { ip: 'junk-127.0.0.1' }; + expect(elasticQuery).toEqual(expected); + }); }); }); diff --git a/x-pack/plugins/lists/server/services/utils/transform_list_item_to_elastic_query.ts b/x-pack/plugins/lists/server/services/utils/transform_list_item_to_elastic_query.ts index 051802cc41b5b..a87380176e6ec 100644 --- a/x-pack/plugins/lists/server/services/utils/transform_list_item_to_elastic_query.ts +++ b/x-pack/plugins/lists/server/services/utils/transform_list_item_to_elastic_query.ts @@ -4,30 +4,249 @@ * you may not use this file except in compliance with the Elastic License. */ -import { EsDataTypeUnion, Type } from '../../../common/schemas'; +import { + EsDataTypeGeoPoint, + EsDataTypeGeoShape, + EsDataTypeRangeTerm, + EsDataTypeSingle, + EsDataTypeUnion, + SerializerOrUndefined, + Type, + esDataTypeGeoShape, + esDataTypeRangeTerm, + esDataTypeSingle, +} from '../../../common/schemas'; + +export const DEFAULT_DATE_REGEX = RegExp('(?.+),(?.+)|(?.+)'); +export const DEFAULT_LTE_GTE_REGEX = RegExp('(?.+)-(?.+)|(?.+)'); +export const DEFAULT_GEO_REGEX = RegExp('(?.+),(?.+)'); +export const DEFAULT_SINGLE_REGEX = RegExp('(?.+)'); export const transformListItemToElasticQuery = ({ + serializer, type, value, }: { type: Type; value: string; -}): EsDataTypeUnion => { + serializer: SerializerOrUndefined; +}): EsDataTypeUnion | null => { switch (type) { - case 'ip': { + case 'shape': + case 'geo_shape': { + return serializeGeoShape({ + defaultSerializer: DEFAULT_GEO_REGEX, + serializer, + type, + value, + }); + } + case 'geo_point': { + return serializeGeoPoint({ defaultSerializer: DEFAULT_GEO_REGEX, serializer, value }); + } + case 'ip_range': { + return serializeIpRange({ + defaultSerializer: DEFAULT_LTE_GTE_REGEX, + serializer, + value, + }); + } + case 'date_range': { + return serializeRanges({ + defaultSerializer: DEFAULT_DATE_REGEX, + serializer, + type, + value, + }); + } + case 'double_range': + case 'float_range': + case 'integer_range': + case 'long_range': { + return serializeRanges({ + defaultSerializer: DEFAULT_LTE_GTE_REGEX, + serializer, + type, + value, + }); + } + default: { + return serializeSingleValue({ + defaultSerializer: DEFAULT_SINGLE_REGEX, + serializer, + type, + value, + }); + } + } +}; + +export const serializeGeoShape = ({ + defaultSerializer, + serializer, + value, + type, +}: { + value: string; + serializer: SerializerOrUndefined; + defaultSerializer: RegExp; + type: 'geo_shape' | 'shape'; +}): EsDataTypeGeoShape | null => { + const regExpSerializer = serializer != null ? RegExp(serializer) : defaultSerializer; + const parsed = regExpSerializer.exec(value.trim()); + + // we only support lat/lon for point and represent it as Well Known Text (WKT) + if (parsed?.groups?.lat != null && parsed?.groups?.lon != null) { + const unionType = { [type]: `POINT (${parsed.groups.lon.trim()} ${parsed.groups.lat.trim()})` }; + if (esDataTypeGeoShape.is(unionType)) { + return unionType; + } else { + return null; + } + } else { + // This should be in Well Known Text (WKT) at this point so let's return it as is + const unionType = { [type]: value.trim() }; + if (esDataTypeGeoShape.is(unionType)) { + return unionType; + } else { + return null; + } + } +}; + +export const serializeGeoPoint = ({ + defaultSerializer, + serializer, + value, +}: { + value: string; + serializer: SerializerOrUndefined; + defaultSerializer: RegExp; +}): EsDataTypeGeoPoint | null => { + const regExpSerializer = serializer != null ? RegExp(serializer) : defaultSerializer; + const parsed = regExpSerializer.exec(value.trim()); + + if (parsed?.groups?.lat != null && parsed?.groups?.lon != null) { + return { + geo_point: { lat: parsed.groups.lat.trim(), lon: parsed.groups.lon.trim() }, + }; + } else { + // This might be in Well Known Text (WKT) so let's return it as is + return { geo_point: value.trim() }; + } +}; + +export const serializeIpRange = ({ + defaultSerializer, + serializer, + value, +}: { + value: string; + serializer: SerializerOrUndefined; + defaultSerializer: RegExp; +}): EsDataTypeRangeTerm | null => { + const regExpSerializer = serializer != null ? RegExp(serializer) : defaultSerializer; + const parsed = regExpSerializer.exec(value.trim()); + + if (parsed?.groups?.lte != null && parsed?.groups?.gte != null) { + return { + ip_range: { gte: parsed.groups.gte.trim(), lte: parsed.groups.lte.trim() }, + }; + } else if (parsed?.groups?.value != null) { + // This is a CIDR string based on the serializer involving value such as (?.+) + if (parsed.groups.value.includes('/')) { return { - ip: value, + ip_range: parsed.groups.value.trim(), }; - } - case 'keyword': { + } else { return { - keyword: value, + ip_range: { gte: parsed.groups.value.trim(), lte: parsed.groups.value.trim() }, }; } + } else { + return null; } - return assertUnreachable(type); }; -const assertUnreachable = (type: string): never => { - throw new Error(`Unknown type: "${type}" in transformListItemToElasticQuery`); +export const serializeRanges = ({ + type, + serializer, + value, + defaultSerializer, +}: { + type: 'long_range' | 'date_range' | 'double_range' | 'float_range' | 'integer_range'; + value: string; + serializer: SerializerOrUndefined; + defaultSerializer: RegExp; +}): EsDataTypeRangeTerm | null => { + const regExpSerializer = serializer != null ? RegExp(serializer) : defaultSerializer; + const parsed = regExpSerializer.exec(value.trim()); + + if (parsed?.groups?.lte != null && parsed?.groups?.gte != null) { + const unionType = { + [type]: { gte: parsed.groups.gte.trim(), lte: parsed.groups.lte.trim() }, + }; + if (esDataTypeRangeTerm.is(unionType)) { + return unionType; + } else { + return null; + } + } else if (parsed?.groups?.value != null) { + const unionType = { + [type]: { gte: parsed.groups.value.trim(), lte: parsed.groups.value.trim() }, + }; + if (esDataTypeRangeTerm.is(unionType)) { + return unionType; + } else { + return null; + } + } else { + return null; + } +}; + +export const serializeSingleValue = ({ + serializer, + value, + defaultSerializer, + type, +}: { + value: string; + serializer: SerializerOrUndefined; + type: + | 'binary' + | 'boolean' + | 'byte' + | 'date' + | 'date_nanos' + | 'double' + | 'float' + | 'half_float' + | 'integer' + | 'ip' + | 'long' + | 'shape' + | 'short' + | 'text' + | 'keyword'; + defaultSerializer: RegExp; +}): EsDataTypeSingle | null => { + const regExpSerializer = serializer != null ? RegExp(serializer) : defaultSerializer; + const parsed = regExpSerializer.exec(value.trim()); + + if (parsed?.groups?.value != null) { + const unionType = { [type]: `${parsed.groups.value.trim()}` }; + if (esDataTypeSingle.is(unionType)) { + return unionType; + } else { + return null; + } + } else { + const unionType = { [type]: value }; + if (esDataTypeSingle.is(unionType)) { + return unionType; + } else { + return null; + } + } };